@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 9d9072a069
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

Implement prompt logprobs & Batched topk for computing logprobs (#1328)

Co-authored-by: Yunmo Chen <16273544+wanmok@users.noreply.github.com>

Zhuohan Li committed Oct 16, 2023 at 10:56 UTC 9d9072a069202e7892a40ef94e9085019e73f370
14 files changed +371 -132
examples/llm_engine_example.py
+1 -1
@@ -11,7 +11,7 @@ def main(args: argparse.Namespace):
11 # Test the following prompts.
12 test_prompts = [
13 ("A robot may not injure a human being",
14 - SamplingParams(temperature=0.0)),
14 + SamplingParams(temperature=0.0, logprobs=1, prompt_logprobs=1)),
15 ("To be or not to be,",
16 SamplingParams(temperature=0.8, top_k=5, presence_penalty=0.2)),
17 ("What is the meaning of life?",
tests/async_engine/test_request_tracker.py
+1 -1
@@ -64,7 +64,7 @@ def test_request_tracker():
64 stream_5 = tracker.add_request("5")
65 assert tracker.new_requests_event.flag
66 tracker.process_request_output(
67 - RequestOutput("2", "output", [], [], finished=True))
67 + RequestOutput("2", "output", [], [], [], finished=True))
68 new, finished = tracker.get_new_and_finished_requests()
69 assert not tracker.new_requests_event.flag
70 assert len(finished) == 1
tests/conftest.py
+33
@@ -107,6 +107,39 @@ class HfRunner:
107 outputs[i] = (output_ids, output_str)
108 return outputs
109
110 + def generate_greedy_logprobs(
111 + self,
112 + prompts: List[str],
113 + max_tokens: int,
114 + ) -> List[List[torch.Tensor]]:
115 + all_logprobs = []
116 + for prompt in prompts:
117 + input_ids = self.tokenizer(prompt, return_tensors="pt").input_ids
118 + output = self.model.generate(
119 + input_ids.cuda(),
120 + use_cache=True,
121 + do_sample=False,
122 + max_new_tokens=max_tokens,
123 + output_hidden_states=True,
124 + return_dict_in_generate=True,
125 + )
126 + seq_logprobs = []
127 + for hidden_states in output.hidden_states:
128 + last_hidden_states = hidden_states[-1][0]
129 + logits = torch.matmul(
130 + last_hidden_states,
131 + self.model.get_output_embeddings().weight.t(),
132 + )
133 + if self.model.get_output_embeddings().bias is not None:
134 + logits += self.model.get_output_embeddings(
135 + ).bias.unsqueeze(0)
136 + logprobs = torch.nn.functional.log_softmax(logits,
137 + dim=-1,
138 + dtype=torch.float32)
139 + seq_logprobs.append(logprobs)
140 + all_logprobs.append(seq_logprobs)
141 + return all_logprobs
142 +
143
144 @pytest.fixture
145 def hf_runner():
tests/samplers/test_logprobs.py new
+55
@@ -0,0 +1,55 @@
1 +import pytest
2 +import torch
3 +
4 +from vllm import SamplingParams
5 +
6 +MODELS = ["facebook/opt-125m"]
7 +
8 +
9 +@pytest.mark.parametrize("model", MODELS)
10 +@pytest.mark.parametrize("dtype", ["half"])
11 +def test_get_prompt_logprobs(
12 + hf_runner,
13 + vllm_runner,
14 + model,
15 + dtype,
16 + example_prompts,
17 +):
18 + max_tokens = 5
19 + hf_model = hf_runner(model, dtype=dtype)
20 + hf_logprobs = hf_model.generate_greedy_logprobs(
21 + example_prompts,
22 + max_tokens=max_tokens,
23 + )
24 + del hf_model
25 +
26 + vllm_model = vllm_runner(model, dtype=dtype)
27 + vllm_sampling_params = SamplingParams(max_tokens=max_tokens,
28 + logprobs=5,
29 + prompt_logprobs=5,
30 + temperature=0.0)
31 + vllm_results = vllm_model.model.generate(
32 + example_prompts, sampling_params=vllm_sampling_params)
33 +
34 + # Test whether logprobs are included in the results.
35 + for result in vllm_results:
36 + assert result.prompt_logprobs is not None
37 + assert result.outputs[0].logprobs is not None
38 +
39 + # Test whether prompt logprobs are consistent with HF
40 + for vllm_result, hf_logprob in zip(vllm_results, hf_logprobs):
41 + # Check prompt logprobs
42 + vllm_prompt_logprobs = vllm_result.prompt_logprobs[1:]
43 + for i, vllm_prompt_logprob_dict in enumerate(vllm_prompt_logprobs):
44 + for token_id, logprob in vllm_prompt_logprob_dict.items():
45 + torch.testing.assert_close(logprob,
46 + hf_logprob[0][i][token_id].item(),
47 + atol=1e-2,
48 + rtol=1e-2)
49 + vllm_sample_logprobs = vllm_result.outputs[0].logprobs
50 + for i, vllm_sample_logprob_dict in enumerate(vllm_sample_logprobs):
51 + for token_id, logprob in vllm_sample_logprob_dict.items():
52 + torch.testing.assert_close(logprob,
53 + hf_logprob[i][-1][token_id].item(),
54 + atol=1e-2,
55 + rtol=1e-2)
vllm/config.py
+1 -1
@@ -143,7 +143,7 @@ class ModelConfig:
143 def get_num_kv_heads(self, parallel_config: "ParallelConfig") -> int:
144 """Returns the number of KV heads per GPU worker."""
145 # For GPTBigCode & Falcon:
146 - # Note: for falcon, when new_decoder_architecture is True, the
146 + # NOTE: for falcon, when new_decoder_architecture is True, the
147 # multi_query flag is ignored and we use n_head_kv for the number of
148 # KV heads.
149 falcon_model_types = ["falcon", "RefinedWeb", "RefinedWebModel"]
vllm/engine/llm_engine.py
+13 -7
@@ -12,8 +12,8 @@ from vllm.logger import init_logger
12 from vllm.outputs import RequestOutput
13 from vllm.sampling_params import SamplingParams
14 from vllm.sequence import (SamplerOutput, Sequence, SequenceGroup,
15 - SequenceGroupMetadata, SequenceOutputs,
16 - SequenceStatus)
15 + SequenceGroupMetadata, SequenceGroupOutputs,
16 + SequenceOutputs, SequenceStatus)
17 from vllm.transformers_utils.tokenizer import (detokenize_incrementally,
18 get_tokenizer)
19 from vllm.utils import Counter
@@ -350,9 +350,15 @@ class LLMEngine:
350 eos_token_id=self.tokenizer.eos_token_id))
351 return current_worst_score >= highest_attainable_score
352
353 - def _process_sequence_group_samples(
354 - self, seq_group: SequenceGroup,
355 - samples: List[SequenceOutputs]) -> None:
353 + def _process_sequence_group_outputs(self, seq_group: SequenceGroup,
354 + outputs: SequenceGroupOutputs) -> None:
355 + # Process prompt logprobs
356 + prompt_logprobs = outputs.prompt_logprobs
357 + if prompt_logprobs is not None:
358 + seq_group.prompt_logprobs = prompt_logprobs
359 +
360 + # Process samples
361 + samples = outputs.samples
362 parent_seqs = seq_group.get_seqs(status=SequenceStatus.RUNNING)
363 existing_finished_seqs = seq_group.get_finished_seqs()
364 parent_child_dict = {
@@ -520,8 +526,8 @@ class LLMEngine:
526 scheduler_outputs: SchedulerOutputs) -> List[RequestOutput]:
527 # Update the scheduled sequence groups with the model outputs.
528 scheduled_seq_groups = scheduler_outputs.scheduled_seq_groups
523 - for seq_group, samples in zip(scheduled_seq_groups, output):
524 - self._process_sequence_group_samples(seq_group, samples)
529 + for seq_group, outputs in zip(scheduled_seq_groups, output):
530 + self._process_sequence_group_outputs(seq_group, outputs)
531
532 # Free the finished sequence groups.
533 self.scheduler.free_finished_seq_groups()
vllm/model_executor/layers/attention.py
+1 -1
@@ -420,7 +420,7 @@ class PagedAttentionWithALiBi(PagedAttention):
420 # Generates ALiBi mask for each prompt.
421 for prompt_len in input_metadata.prompt_lens:
422 bias = torch.arange(prompt_len, dtype=dtype)
423 - # Note(zhuohan): HF uses
423 + # NOTE(zhuohan): HF uses
424 # `bias = bias[None, :].repeat(prompt_len, 1)`
425 # here. We find that both biases give the same results, but
426 # the bias below more accurately follows the original ALiBi
vllm/model_executor/layers/sampler.py
+203 -107
@@ -8,7 +8,8 @@ from vllm.model_executor.input_metadata import InputMetadata
8 from vllm.model_executor.parallel_utils.communication_op import (
9 tensor_model_parallel_all_gather)
10 from vllm.sampling_params import SamplingParams, SamplingType
11 -from vllm.sequence import SamplerOutput, SequenceData, SequenceOutputs
11 +from vllm.sequence import (PromptLogprobs, SampleLogprobs, SamplerOutput,
12 + SequenceData, SequenceGroupOutputs, SequenceOutputs)
13
14 _SAMPLING_EPS = 1e-5
15
@@ -82,7 +83,12 @@ class Sampler(nn.Module):
83 logprobs = torch.log_softmax(logits, dim=-1, dtype=torch.float)
84
85 # Sample the next tokens.
85 - return _sample(probs, logprobs, input_metadata)
86 + sample_results = _sample(probs, logprobs, input_metadata)
87 + # Get the logprobs query results.
88 + prompt_logprobs, sample_logprobs = _get_logprobs(
89 + logprobs, input_metadata, sample_results)
90 + return _build_sampler_output(sample_results, input_metadata,
91 + prompt_logprobs, sample_logprobs)
92
93
94 def _get_logits(hidden_states: torch.Tensor, embedding: torch.Tensor,
@@ -102,24 +108,28 @@ def _prune_hidden_states(
108 hidden_states: torch.Tensor,
109 input_metadata: InputMetadata,
110 ) -> torch.Tensor:
105 - last_token_indices = []
111 + selected_token_indices: List[int] = []
112 start_idx = 0
113 for i, seq_group in enumerate(input_metadata.seq_groups):
108 - seq_ids, _ = seq_group
114 + seq_ids, sampling_params = seq_group
115 if i < input_metadata.num_prompts:
116 assert len(seq_ids) == 1, "Prompt input should have only one seq."
117 prompt_len = input_metadata.prompt_lens[i]
112 - last_token_indices.append(start_idx + prompt_len - 1)
118 + if sampling_params.prompt_logprobs is not None:
119 + selected_token_indices.extend(
120 + range(start_idx, start_idx + prompt_len - 1))
121 + selected_token_indices.append(start_idx + prompt_len - 1)
122 start_idx += prompt_len
123 else:
124 num_seqs = len(seq_ids)
116 - last_token_indices.extend(range(start_idx, start_idx + num_seqs))
125 + selected_token_indices.extend(
126 + range(start_idx, start_idx + num_seqs))
127 start_idx += num_seqs
128
119 - last_token_indices = torch.tensor(last_token_indices,
120 - dtype=torch.long,
121 - device=hidden_states.device)
122 - return hidden_states.index_select(0, last_token_indices)
129 + selected_token_indices = torch.tensor(selected_token_indices,
130 + dtype=torch.long,
131 + device=hidden_states.device)
132 + return hidden_states.index_select(0, selected_token_indices)
133
134
135 def _get_penalties(
@@ -127,10 +137,17 @@ def _get_penalties(
137 # Collect the presence and frequency penalties.
138 presence_penalties: List[float] = []
139 frequency_penalties: List[float] = []
130 - for seq_group in input_metadata.seq_groups:
140 + for i, seq_group in enumerate(input_metadata.seq_groups):
141 seq_ids, sampling_params = seq_group
142 p = sampling_params.presence_penalty
143 f = sampling_params.frequency_penalty
144 + if (i < input_metadata.num_prompts
145 + and sampling_params.prompt_logprobs is not None):
146 + # NOTE: We do not apply presence and frequency penalties for the
147 + # prompt token positions where we don't sample new tokens.
148 + prompt_len = input_metadata.prompt_lens[i]
149 + presence_penalties += [0] * (prompt_len - 1)
150 + frequency_penalties += [0] * (prompt_len - 1)
151 presence_penalties += [p] * len(seq_ids)
152 frequency_penalties += [f] * len(seq_ids)
153 return presence_penalties, frequency_penalties
@@ -138,8 +155,14 @@ def _get_penalties(
155
156 def _get_output_tokens(input_metadata: InputMetadata) -> List[List[int]]:
157 output_tokens: List[List[int]] = []
141 - for seq_group in input_metadata.seq_groups:
142 - seq_ids, _ = seq_group
158 + for i, seq_group in enumerate(input_metadata.seq_groups):
159 + seq_ids, sampling_params = seq_group
160 + if (i < input_metadata.num_prompts
161 + and sampling_params.prompt_logprobs is not None):
162 + # NOTE: prompt token positions do not need output tokens to
163 + # compute penalties.
164 + prompt_len = input_metadata.prompt_lens[i]
165 + output_tokens.extend([] for _ in range(prompt_len - 1))
166 for seq_id in seq_ids:
167 seq_data = input_metadata.seq_data[seq_id]
168 output_tokens.append(seq_data.output_token_ids)
@@ -200,7 +223,7 @@ def _apply_penalties(
223 def _get_temperatures(input_metadata: InputMetadata) -> List[float]:
224 # Collect the temperatures for the logits.
225 temperatures: List[float] = []
203 - for seq_group in input_metadata.seq_groups:
226 + for i, seq_group in enumerate(input_metadata.seq_groups):
227 seq_ids, sampling_params = seq_group
228 temperature = sampling_params.temperature
229 if temperature < _SAMPLING_EPS:
@@ -208,6 +231,10 @@ def _get_temperatures(input_metadata: InputMetadata) -> List[float]:
231 # (i.e., greedy sampling or beam search).
232 # Set the temperature to 1 to avoid division by zero.
233 temperature = 1.0
234 + if (i < input_metadata.num_prompts
235 + and sampling_params.prompt_logprobs is not None):
236 + prompt_len = input_metadata.prompt_lens[i]
237 + temperatures += [temperature] * (prompt_len - 1)
238 temperatures += [temperature] * len(seq_ids)
239 return temperatures
240
@@ -218,13 +245,18 @@ def _get_top_p_top_k(
245 ) -> Tuple[List[float], List[int]]:
246 top_ps: List[float] = []
247 top_ks: List[int] = []
221 - for seq_group in input_metadata.seq_groups:
248 + for i, seq_group in enumerate(input_metadata.seq_groups):
249 seq_ids, sampling_params = seq_group
250 top_p = sampling_params.top_p
251 # k should not be greater than the vocab size.
252 top_k = min(sampling_params.top_k, vocab_size)
253 # k=-1 means no truncation.
254 top_k = vocab_size if top_k == -1 else top_k
255 + if (i < input_metadata.num_prompts
256 + and sampling_params.prompt_logprobs is not None):
257 + prompt_len = input_metadata.prompt_lens[i]
258 + top_ps += [top_p] * (prompt_len - 1)
259 + top_ks += [top_k] * (prompt_len - 1)
260 top_ps += [top_p] * len(seq_ids)
261 top_ks += [top_k] * len(seq_ids)
262 return top_ps, top_ks
@@ -259,49 +291,6 @@ def _apply_top_p_top_k(
291 return logits
292
293
262 -def _get_topk_logprobs(
263 - logprobs: torch.Tensor,
264 - num_logprobs: Optional[int],
265 -) -> List[Dict[int, float]]:
266 - num_seqs = logprobs.size(0)
267 - if num_logprobs is None or num_logprobs == 0:
268 - return [{} for _ in range(num_seqs)]
269 -
270 - all_topk_logprobs, all_topk_ids = torch.topk(logprobs,
271 - num_logprobs,
272 - dim=-1)
273 - all_topk_logprobs = all_topk_logprobs.cpu()
274 - all_topk_ids = all_topk_ids.cpu()
275 - all_token_to_logprob = []
276 - for topk_logprobs, topk_ids in zip(all_topk_logprobs, all_topk_ids):
277 - token_to_logprob: Dict[int, float] = {}
278 - for token_id, logprob in zip(topk_ids, topk_logprobs):
279 - token_to_logprob[token_id.item()] = logprob.item()
280 - all_token_to_logprob.append(token_to_logprob)
281 - return all_token_to_logprob
282 -
283 -
284 -def _build_sequence_outputs(
285 - parent_ids: List[int],
286 - next_token_ids: List[int],
287 - selected_token_logprobs: List[float],
288 - parent_seq_ids: List[int],
289 - parent_logprobs: torch.Tensor,
290 - num_output_logprobs: Optional[int],
291 -) -> List[SequenceOutputs]:
292 - # Get top-k log probabilities for the next tokens.
293 - next_logprobs = _get_topk_logprobs(parent_logprobs, num_output_logprobs)
294 - seq_outputs: List[SequenceOutputs] = []
295 - for parent_id, next_token_id, token_logprob in zip(
296 - parent_ids, next_token_ids, selected_token_logprobs):
297 - output_logprobs = next_logprobs[parent_id].copy()
298 - output_logprobs[next_token_id] = token_logprob
299 - seq_outputs.append(
300 - SequenceOutputs(parent_seq_ids[parent_id], next_token_id,
301 - output_logprobs))
302 - return seq_outputs
303 -
304 -
294 def _greedy_sample(
295 selected_seq_groups: List[Tuple[List[int], SamplingParams]],
296 logprobs: torch.Tensor,
@@ -372,7 +361,7 @@ def _beam_search_sample(
361 # for details. See also HF reference:
362 # https://github.com/huggingface/transformers/blob/a4dd53d88e4852f023332d284ff07a01afcd5681/src/transformers/generation/utils.py#L3063-L3065
363 #
375 - # Note: Beam search is not vectorized, so its speed can be slower than
364 + # NOTE: Beam search is not vectorized, so its speed can be slower than
365 # other sampling methods.
366 sample_idx = 0
367 results = []
@@ -416,79 +405,186 @@ def _sample(
405 probs: torch.Tensor,
406 logprobs: torch.Tensor,
407 input_metadata: InputMetadata,
419 -) -> SamplerOutput:
408 +) -> List[Tuple[List[int], List[int]]]:
409 categorized_seq_group_ids = {t: [] for t in SamplingType}
410 + categorized_sample_indices = {t: [] for t in SamplingType}
411 start_idx = 0
422 - categorized_seq_ids = {t: [] for t in SamplingType}
412 for i, seq_group in enumerate(input_metadata.seq_groups):
413 seq_ids, sampling_params = seq_group
414 sampling_type = sampling_params.sampling_type
415 + if (i < input_metadata.num_prompts
416 + and sampling_params.prompt_logprobs is not None):
417 + # NOTE: prompt token positions do not need sample, skip
418 + prompt_len = input_metadata.prompt_lens[i]
419 + start_idx += prompt_len - 1
420 categorized_seq_group_ids[sampling_type].append(i)
421 num_seqs = len(seq_ids)
428 - categorized_seq_ids[sampling_type].extend(
422 + categorized_sample_indices[sampling_type].extend(
423 range(start_idx, start_idx + num_seqs))
424 start_idx += num_seqs
431 - seq_outputs_dict: Dict[int, List[SequenceOutputs]] = {}
425 +
426 + sample_results_dict: Dict[int, Tuple[List[int], List[int]]] = {}
427 for sampling_type in SamplingType:
428 seq_group_ids = categorized_seq_group_ids[sampling_type]
429 seq_groups = [input_metadata.seq_groups[i] for i in seq_group_ids]
430 is_prompts = [i < input_metadata.num_prompts for i in seq_group_ids]
436 - num_tokens = len(categorized_seq_ids[sampling_type])
431 + sample_indices = categorized_sample_indices[sampling_type]
432 + num_tokens = len(sample_indices)
433 if num_tokens == 0:
434 continue
439 - category_logprobs = logprobs[categorized_seq_ids[sampling_type]]
440 - category_probs = probs[categorized_seq_ids[sampling_type]]
435 if sampling_type == SamplingType.GREEDY:
436 + category_logprobs = logprobs[sample_indices]
437 sample_results = _greedy_sample(seq_groups, category_logprobs)
438 elif sampling_type == SamplingType.RANDOM:
439 + category_probs = probs[sample_indices]
440 sample_results = _random_sample(seq_groups, is_prompts,
441 category_probs)
442 elif sampling_type == SamplingType.BEAM:
443 + category_logprobs = logprobs[sample_indices]
444 sample_results = _beam_search_sample(seq_groups, is_prompts,
445 input_metadata.seq_data,
446 category_logprobs)
447 else:
448 raise ValueError(f"Unsupported sampling type: {sampling_type}")
449 + sample_results_dict.update(zip(seq_group_ids, sample_results))
450
453 - # Batched query for logprobs of selected token
454 - batched_logprobs_query_seq_indices: List[int] = []
455 - batched_logprobs_query_token_indices: List[int] = []
456 - sample_idx = 0
457 - for seq_group_id, seq_group, sample_result in zip(
458 - seq_group_ids, seq_groups, sample_results):
459 - seq_ids, sampling_params = seq_group
460 - next_token_ids, parent_ids = sample_result
461 - num_parent_seqs = len(seq_ids)
451 + sample_results = [
452 + sample_results_dict[i] for i in range(len(input_metadata.seq_groups))
453 + ]
454 + return sample_results
455 +
456 +
457 +def _get_logprobs(
458 + logprobs: torch.Tensor,
459 + input_metadata: InputMetadata,
460 + sample_results: List[Tuple[List[int], List[int]]],
461 +) -> Tuple[List[Optional[List[Optional[Dict[int, float]]]]], List[List[Dict[
462 + int, float]]]]:
463 + # Prepare query indices
464 + batched_logprobs_query_seq_indices: List[int] = []
465 + batched_logprobs_query_token_indices: List[int] = []
466 + largest_num_logprobs = 0
467 + sample_idx = 0
468 + for i, (seq_group, sample_result) in enumerate(
469 + zip(input_metadata.seq_groups, sample_results)):
470 + seq_ids, sampling_params = seq_group
471 + next_token_ids, parent_ids = sample_result
472 + num_parent_seqs = len(seq_ids)
473 + if (i < input_metadata.num_prompts
474 + and sampling_params.prompt_logprobs is not None):
475 + largest_num_logprobs = max(largest_num_logprobs,
476 + sampling_params.prompt_logprobs)
477 + prompt_len = input_metadata.prompt_lens[i]
478 + prompt_tokens = input_metadata.seq_data[
479 + seq_ids[0]].prompt_token_ids
480 batched_logprobs_query_seq_indices.extend(
463 - [sample_idx + parent_id for parent_id in parent_ids])
464 - batched_logprobs_query_token_indices.extend(next_token_ids)
465 - sample_idx += num_parent_seqs
466 - assert sample_idx == num_tokens
467 - batched_logprobs_query_result = category_logprobs[[
468 - batched_logprobs_query_seq_indices,
469 - batched_logprobs_query_token_indices
470 - ]].tolist()
471 -
472 - # Build the sequence outputs.
473 - sample_idx = 0
474 - result_idx = 0
475 - for seq_group_id, seq_group, sample_result in zip(
476 - seq_group_ids, seq_groups, sample_results):
477 - seq_ids, sampling_params = seq_group
478 - next_token_ids, parent_ids = sample_result
479 - num_results = len(next_token_ids)
480 - num_parent_seqs = len(seq_ids)
481 - parent_logprobs = category_logprobs[sample_idx:sample_idx +
482 - num_parent_seqs]
483 - selected_token_logprobs = batched_logprobs_query_result[
484 - result_idx:result_idx + num_results]
485 - seq_output = _build_sequence_outputs(parent_ids, next_token_ids,
486 - selected_token_logprobs,
487 - seq_ids, parent_logprobs,
488 - sampling_params.logprobs)
489 - seq_outputs_dict[seq_group_id] = seq_output
490 - sample_idx += num_parent_seqs
491 - result_idx += num_results
492 - assert sample_idx == num_tokens
493 -
494 - return [seq_outputs_dict[i] for i in range(len(input_metadata.seq_groups))]
481 + sample_idx + j for j in range(prompt_len - 1))
482 + batched_logprobs_query_token_indices.extend(
483 + token_id for token_id in prompt_tokens[1:])
484 + sample_idx += prompt_len - 1
485 + batched_logprobs_query_seq_indices.extend(
486 + [sample_idx + parent_id for parent_id in parent_ids])
487 + batched_logprobs_query_token_indices.extend(next_token_ids)
488 + if sampling_params.logprobs is not None:
489 + largest_num_logprobs = max(largest_num_logprobs,
490 + sampling_params.logprobs)
491 + sample_idx += num_parent_seqs
492 + assert sample_idx == logprobs.size(0)
493 +
494 + # Batched query for logprobs of selected token
495 + batched_logprobs_query_result = logprobs[[
496 + batched_logprobs_query_seq_indices,
497 + batched_logprobs_query_token_indices
498 + ]].cpu()
499 +
500 + # Batched query for logprobs of topk tokens
501 + if largest_num_logprobs > 0:
502 + top_logprobs, top_token_ids = torch.topk(logprobs,
503 + largest_num_logprobs,
504 + dim=-1)
505 + top_logprobs = top_logprobs.cpu()
506 + top_token_ids = top_token_ids.cpu()
507 + else:
508 + top_logprobs, top_token_ids = None, None
509 +
510 + # Gather results
511 + result_prompt_logprobs: List[Optional[PromptLogprobs]] = []
512 + result_sample_logprobs: List[SampleLogprobs] = []
513 + sample_idx = 0
514 + query_result_idx = 0
515 + for i, (seq_group, sample_result) in enumerate(
516 + zip(input_metadata.seq_groups, sample_results)):
517 + seq_ids, sampling_params = seq_group
518 + next_token_ids, parent_ids = sample_result
519 +
520 + # Prompt logprobs
521 + if (i < input_metadata.num_prompts
522 + and sampling_params.prompt_logprobs is not None):
523 + num_logprobs = sampling_params.prompt_logprobs
524 + prompt_len = input_metadata.prompt_lens[i]
525 + prompt_tokens = input_metadata.seq_data[
526 + seq_ids[0]].prompt_token_ids
527 + group_prompt_logprobs: PromptLogprobs = [None]
528 + for token_id in prompt_tokens[1:]:
529 + prompt_logprobs_dict = {
530 + token_id:
531 + batched_logprobs_query_result[query_result_idx].item()
532 + }
533 + if num_logprobs > 0:
534 + prompt_logprobs_dict.update(
535 + zip(top_token_ids[sample_idx, :num_logprobs].tolist(),
536 + top_logprobs[sample_idx, :num_logprobs].tolist()))
537 + group_prompt_logprobs.append(prompt_logprobs_dict)
538 + sample_idx += 1
539 + query_result_idx += 1
540 + result_prompt_logprobs.append(group_prompt_logprobs)
541 + else:
542 + result_prompt_logprobs.append(None)
543 +
544 + # Sample logprobs
545 + num_logprobs = sampling_params.logprobs
546 + if num_logprobs is None:
547 + num_logprobs = 0
548 + group_sample_logprobs: SampleLogprobs = []
549 + for next_token_id, parent_id in zip(next_token_ids, parent_ids):
550 + sample_logprobs_dict = {
551 + next_token_id:
552 + batched_logprobs_query_result[query_result_idx].item()
553 + }
554 + query_result_idx += 1
555 + if num_logprobs > 0:
556 + sample_logprobs_dict.update(
557 + zip(
558 + top_token_ids[sample_idx +
559 + parent_id, :num_logprobs].tolist(),
560 + top_logprobs[sample_idx +
561 + parent_id, :num_logprobs].tolist()))
562 + group_sample_logprobs.append(sample_logprobs_dict)
563 + result_sample_logprobs.append(group_sample_logprobs)
564 + sample_idx += len(seq_ids)
565 +
566 + return result_prompt_logprobs, result_sample_logprobs
567 +
568 +
569 +def _build_sampler_output(
570 + sample_results: List[Tuple[List[int], List[int]]],
571 + input_metadata: InputMetadata,
572 + prompt_logprobs: List[Optional[PromptLogprobs]],
573 + sample_logprobs: List[SampleLogprobs],
574 +) -> SamplerOutput:
575 + sampler_output = []
576 + for (seq_group, sample_result, group_prompt_logprobs,
577 + group_sample_logprobs) in zip(input_metadata.seq_groups,
578 + sample_results, prompt_logprobs,
579 + sample_logprobs):
580 + seq_ids, _ = seq_group
581 + next_token_ids, parent_ids = sample_result
582 + seq_outputs = []
583 + for parent_id, next_token_id, logprobs in zip(parent_ids,
584 + next_token_ids,
585 + group_sample_logprobs):
586 + seq_outputs.append(
587 + SequenceOutputs(seq_ids[parent_id], next_token_id, logprobs))
588 + sampler_output.append(
589 + SequenceGroupOutputs(seq_outputs, group_prompt_logprobs))
590 + return sampler_output
vllm/model_executor/parallel_utils/communication_op.py
+1 -1
@@ -9,7 +9,7 @@ from vllm.model_executor.parallel_utils.parallel_state import (
9 def tensor_model_parallel_all_reduce(input_):
10 """All-reduce the input tensor across model parallel group.
11
12 - Note: This operation is applied in-place on the input tensor.
12 + NOTE: This operation is applied in-place on the input tensor.
13 """
14 # Bypass the function if we are using only 1 GPU.
15 if get_tensor_model_parallel_world_size() == 1:
vllm/model_executor/parallel_utils/layers.py
+1 -1
@@ -133,7 +133,7 @@ class ColumnParallelLinear(torch.nn.Module):
133 params_dtype = torch.get_default_dtype()
134
135 # Parameters.
136 - # Note: torch.nn.functional.linear performs XA^T + b and as a result
136 + # NOTE: torch.nn.functional.linear performs XA^T + b and as a result
137 # we allocate the transpose.
138 self.create_weights(params_dtype)
139
vllm/model_executor/parallel_utils/utils.py
+1 -1
@@ -41,7 +41,7 @@ def split_tensor_along_last_dim(
41 last_dim_size = divide(tensor.size()[last_dim], num_partitions)
42 # Split.
43 tensor_list = torch.split(tensor, last_dim_size, dim=last_dim)
44 - # Note: torch.split does not create contiguous tensors by default.
44 + # NOTE: torch.split does not create contiguous tensors by default.
45 if contiguous_split_chunks:
46 return tuple(chunk.contiguous() for chunk in tensor_list)
47
vllm/outputs.py
+13 -8
@@ -1,6 +1,7 @@
1 -from typing import Dict, List, Optional
1 +from typing import List, Optional
2
3 -from vllm.sequence import SequenceGroup, SequenceStatus
3 +from vllm.sequence import (PromptLogprobs, SampleLogprobs, SequenceGroup,
4 + SequenceStatus)
5
6
7 class CompletionOutput:
@@ -23,7 +24,7 @@ class CompletionOutput:
24 text: str,
25 token_ids: List[int],
26 cumulative_logprob: float,
26 - logprobs: Optional[List[Dict[int, float]]],
27 + logprobs: Optional[SampleLogprobs],
28 finish_reason: Optional[str] = None,
29 ) -> None:
30 self.index = index
@@ -61,12 +62,14 @@ class RequestOutput:
62 request_id: str,
63 prompt: str,
64 prompt_token_ids: List[int],
65 + prompt_logprobs: Optional[PromptLogprobs],
66 outputs: List[CompletionOutput],
67 finished: bool,
68 ) -> None:
69 self.request_id = request_id
70 self.prompt = prompt
71 self.prompt_token_ids = prompt_token_ids
72 + self.prompt_logprobs = prompt_logprobs
73 self.outputs = outputs
74 self.finished = finished
75
@@ -91,7 +94,7 @@ class RequestOutput:
94 # NOTE: We need to take care of this case because the sequence
95 # always has the logprobs of the sampled tokens even if the
96 # logprobs are not requested.
94 - logprobs = {}
97 + logprobs = None
98 finshed_reason = SequenceStatus.get_finished_reason(seq.status)
99 output = CompletionOutput(seqs.index(seq), seq.output_text,
100 seq.get_output_token_ids(),
@@ -100,15 +103,17 @@ class RequestOutput:
103 outputs.append(output)
104
105 # Every sequence in the sequence group should have the same prompt.
103 - prompt = top_n_seqs[0].prompt
104 - prompt_token_ids = top_n_seqs[0].data.prompt_token_ids
106 + prompt = seq_group.prompt
107 + prompt_token_ids = seq_group.prompt_token_ids
108 + prompt_logprobs = seq_group.prompt_logprobs
109 finished = seq_group.is_finished()
106 - return cls(seq_group.request_id, prompt, prompt_token_ids, outputs,
107 - finished)
110 + return cls(seq_group.request_id, prompt, prompt_token_ids,
111 + prompt_logprobs, outputs, finished)
112
113 def __repr__(self) -> str:
114 return (f"RequestOutput(request_id={self.request_id}, "
115 f"prompt={self.prompt!r}, "
116 f"prompt_token_ids={self.prompt_token_ids}, "
117 + f"prompt_logprobs={self.prompt_logprobs}, "
118 f"outputs={self.outputs}, "
119 f"finished={self.finished})")
vllm/sampling_params.py
+12
@@ -60,6 +60,12 @@ class SamplingParams:
60 tokens after the EOS token is generated.
61 max_tokens: Maximum number of tokens to generate per output sequence.
62 logprobs: Number of log probabilities to return per output token.
63 + Note that the implementation follows the OpenAI API: The return
64 + result includes the log probabilities on the `logprobs` most likely
65 + tokens, as well the chosen tokens. The API will always return the
66 + log probability of the sampled token, so there may be up to
67 + `logprobs+1` elements in the response.
68 + prompt_logprobs: Number of log probabilities to return per prompt token.
69 skip_special_tokens: Whether to skip special tokens in the output.
70 """
71
@@ -80,6 +86,7 @@ class SamplingParams:
86 ignore_eos: bool = False,
87 max_tokens: int = 16,
88 logprobs: Optional[int] = None,
89 + prompt_logprobs: Optional[int] = None,
90 skip_special_tokens: bool = True,
91 ) -> None:
92 self.n = n
@@ -105,6 +112,7 @@ class SamplingParams:
112 self.ignore_eos = ignore_eos
113 self.max_tokens = max_tokens
114 self.logprobs = logprobs
115 + self.prompt_logprobs = prompt_logprobs
116 self.skip_special_tokens = skip_special_tokens
117
118 self._verify_args()
@@ -142,6 +150,9 @@ class SamplingParams:
150 if self.logprobs is not None and self.logprobs < 0:
151 raise ValueError(
152 f"logprobs must be non-negative, got {self.logprobs}.")
153 + if self.prompt_logprobs is not None and self.prompt_logprobs < 0:
154 + raise ValueError(f"prompt_logprobs must be non-negative, got "
155 + f"{self.prompt_logprobs}.")
156
157 def _verify_beam_search(self) -> None:
158 if self.best_of == 1:
@@ -200,4 +211,5 @@ class SamplingParams:
211 f"ignore_eos={self.ignore_eos}, "
212 f"max_tokens={self.max_tokens}, "
213 f"logprobs={self.logprobs}, "
214 + f"prompt_logprobs={self.prompt_logprobs}, "
215 f"skip_special_tokens={self.skip_special_tokens})")
vllm/sequence.py
+35 -3
@@ -6,6 +6,9 @@ from typing import Dict, List, Optional, Union
6 from vllm.block import LogicalTokenBlock
7 from vllm.sampling_params import SamplingParams
8
9 +PromptLogprobs = List[Optional[Dict[int, float]]]
10 +SampleLogprobs = List[Dict[int, float]]
11 +
12
13 class SequenceStatus(enum.Enum):
14 """Status of a sequence."""
@@ -116,7 +119,7 @@ class Sequence:
119 self.block_size = block_size
120
121 self.data = SequenceData(prompt_token_ids)
119 - self.output_logprobs: List[Dict[int, float]] = []
122 + self.output_logprobs: SampleLogprobs = []
123 self.output_text = ""
124
125 self.logical_token_blocks: List[LogicalTokenBlock] = []
@@ -196,7 +199,7 @@ class Sequence:
199 """
200 if seq_len is None:
201 seq_len = self.get_len()
199 - # Note: HF implementation does not count the EOS token
202 + # NOTE: HF implementation does not count the EOS token
203 # towards the length, we align with that here for testing.
204 if (eos_token_id is not None
205 and self.get_last_token_id() == eos_token_id):
@@ -238,6 +241,19 @@ class SequenceGroup:
241 self.seqs_dict = {seq.seq_id: seq for seq in seqs}
242 self.sampling_params = sampling_params
243 self.arrival_time = arrival_time
244 + self.prompt_logprobs: Optional[PromptLogprobs] = None
245 +
246 + @property
247 + def prompt(self) -> str:
248 + # All sequences in the group should have the same prompt.
249 + # We use the prompt of an arbitrary sequence.
250 + return next(iter(self.seqs_dict.values())).prompt
251 +
252 + @property
253 + def prompt_token_ids(self) -> List[int]:
254 + # All sequences in the group should have the same prompt.
255 + # We use the prompt of an arbitrary sequence.
256 + return next(iter(self.seqs_dict.values())).data.prompt_token_ids
257
258 def get_max_num_running_seqs(self) -> int:
259 """The maximum number of sequences running in parallel in the remaining
@@ -370,6 +386,22 @@ class SequenceOutputs:
386 and self.logprobs == other.logprobs)
387
388
389 +class SequenceGroupOutputs:
390 + """The model outputs associated with a sequence group."""
391 +
392 + def __init__(
393 + self,
394 + samples: List[SequenceOutputs],
395 + prompt_logprobs: Optional[PromptLogprobs],
396 + ) -> None:
397 + self.samples = samples
398 + self.prompt_logprobs = prompt_logprobs
399 +
400 + def __repr__(self) -> str:
401 + return (f"SequenceGroupOutputs(samples={self.samples}, "
402 + f"prompt_logprobs={self.prompt_logprobs})")
403 +
404 +
405 # For each sequence group, we generate a list of SequenceOutputs object,
406 # each of which contains one possible candidate for the next token.
375 -SamplerOutput = List[List[SequenceOutputs]]
407 +SamplerOutput = List[SequenceGroupOutputs]