@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 91fce82c6f
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

change the timing of sorting logits (#1309)

yhlskt23 committed Oct 11, 2023 at 11:37 UTC 91fce82c6f3e3efd705faa0edd3aa64d328c3c77
1 file changed +16 -24
vllm/model_executor/layers/sampler.py
+16 -24
@@ -102,30 +102,24 @@ def _prune_hidden_states(
102 hidden_states: torch.Tensor,
103 input_metadata: InputMetadata,
104 ) -> torch.Tensor:
105 - last_token_indices = {t: [] for t in SamplingType}
105 + last_token_indices = []
106 start_idx = 0
107 for i, seq_group in enumerate(input_metadata.seq_groups):
108 - seq_ids, sampling_params = seq_group
109 - sampling_type = sampling_params.sampling_type
108 + seq_ids, _ = seq_group
109 if i < input_metadata.num_prompts:
110 assert len(seq_ids) == 1, "Prompt input should have only one seq."
111 prompt_len = input_metadata.prompt_lens[i]
113 - last_token_indices[sampling_type].append(start_idx + prompt_len -
114 - 1)
112 + last_token_indices.append(start_idx + prompt_len - 1)
113 start_idx += prompt_len
114 else:
115 num_seqs = len(seq_ids)
118 - last_token_indices[sampling_type].extend(
119 - range(start_idx, start_idx + num_seqs))
116 + last_token_indices.extend(range(start_idx, start_idx + num_seqs))
117 start_idx += num_seqs
118
122 - all_last_token_indices = []
123 - for sampling_type in SamplingType:
124 - all_last_token_indices.extend(last_token_indices[sampling_type])
125 - all_last_token_indices = torch.tensor(all_last_token_indices,
126 - dtype=torch.long,
127 - device=hidden_states.device)
128 - return hidden_states.index_select(0, all_last_token_indices)
119 + last_token_indices = torch.tensor(last_token_indices,
120 + dtype=torch.long,
121 + device=hidden_states.device)
122 + return hidden_states.index_select(0, last_token_indices)
123
124
125 def _get_penalties(
@@ -424,27 +418,26 @@ def _sample(
418 input_metadata: InputMetadata,
419 ) -> SamplerOutput:
420 categorized_seq_group_ids = {t: [] for t in SamplingType}
427 - category_num_tokens = {t: 0 for t in SamplingType}
421 + start_idx = 0
422 + categorized_seq_ids = {t: [] for t in SamplingType}
423 for i, seq_group in enumerate(input_metadata.seq_groups):
424 seq_ids, sampling_params = seq_group
425 sampling_type = sampling_params.sampling_type
426 categorized_seq_group_ids[sampling_type].append(i)
427 num_seqs = len(seq_ids)
433 - category_num_tokens[sampling_type] += num_seqs
434 -
428 + categorized_seq_ids[sampling_type].extend(
429 + range(start_idx, start_idx + num_seqs))
430 + start_idx += num_seqs
431 seq_outputs_dict: Dict[int, List[SequenceOutputs]] = {}
436 - category_start_idx = 0
432 for sampling_type in SamplingType:
433 seq_group_ids = categorized_seq_group_ids[sampling_type]
434 seq_groups = [input_metadata.seq_groups[i] for i in seq_group_ids]
435 is_prompts = [i < input_metadata.num_prompts for i in seq_group_ids]
441 - num_tokens = category_num_tokens[sampling_type]
436 + num_tokens = len(categorized_seq_ids[sampling_type])
437 if num_tokens == 0:
438 continue
444 - category_logprobs = logprobs[category_start_idx:category_start_idx +
445 - num_tokens]
446 - category_probs = probs[category_start_idx:category_start_idx +
447 - num_tokens]
439 + category_logprobs = logprobs[categorized_seq_ids[sampling_type]]
440 + category_probs = probs[categorized_seq_ids[sampling_type]]
441 if sampling_type == SamplingType.GREEDY:
442 sample_results = _greedy_sample(seq_groups, category_logprobs)
443 elif sampling_type == SamplingType.RANDOM:
@@ -497,6 +490,5 @@ def _sample(
490 sample_idx += num_parent_seqs
491 result_idx += num_results
492 assert sample_idx == num_tokens
500 - category_start_idx += num_tokens
493
494 return [seq_outputs_dict[i] for i in range(len(input_metadata.seq_groups))]