A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 08:00 UTC
Improve _prune_hidden_states micro-benchmark (#707)
Aman Gupta Karmani committed
Aug 31, 2023 at 00:28 UTC
28873a2799ddfdd0624edd4619e6fbeeb49cd02c
1 file changed
+2
-1
vllm/model_executor/layers/sampler.py
+2
-1
@@ -100,7 +100,8 @@ def _prune_hidden_states(
100
start_idx += prompt_len
101
last_token_indicies.extend(
102
range(start_idx, start_idx + input_metadata.num_generation_tokens))
103
- return hidden_states[last_token_indicies]
103
+ return hidden_states.index_select(
104
+ 0, torch.tensor(last_token_indicies, device=hidden_states.device))
105
106
107
def _get_penalties(