A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
Fix 1D query issue from `_prune_hidden_states` (#3539)
SangBin Cho committed
Mar 21, 2024 at 17:49 UTC
3bbff9e5ab964cf04897cebfc5e886a1113fef01
1 file changed
-1
vllm/model_executor/layers/logits_processor.py
-1
@@ -77,7 +77,6 @@ def _prune_hidden_states(
77
hidden_states: torch.Tensor,
78
sampling_metadata: SamplingMetadata,
79
) -> torch.Tensor:
80
- hidden_states = hidden_states.view(-1, hidden_states.shape[-1])
80
return hidden_states.index_select(0,
81
sampling_metadata.selected_token_indices)
82