@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 28873a2799
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 08:00 UTC

Improve _prune_hidden_states micro-benchmark (#707)

Aman Gupta Karmani committed Aug 31, 2023 at 00:28 UTC 28873a2799ddfdd0624edd4619e6fbeeb49cd02c
1 file changed +2 -1
vllm/model_executor/layers/sampler.py
+2 -1
@@ -100,7 +100,8 @@ def _prune_hidden_states(
100 start_idx += prompt_len
101 last_token_indicies.extend(
102 range(start_idx, start_idx + input_metadata.num_generation_tokens))
103 - return hidden_states[last_token_indicies]
103 + return hidden_states.index_select(
104 + 0, torch.tensor(last_token_indicies, device=hidden_states.device))
105
106
107 def _get_penalties(