A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 07:00 UTC
[Model] Remove redundant softmax when using PoolingType.STEP (#10415)
Maybewuss committed
Nov 18, 2024 at 18:05 UTC
01aae1cc68d6013dd91e87418a6d82fa02c58457
1 file changed
+1
-2
vllm/model_executor/layers/pooler.py
+1
-2
@@ -118,14 +118,13 @@ class Pooler(nn.Module):
118
if returned_token_ids is not None and len(returned_token_ids) > 0:
119
hidden_states = hidden_states[:, returned_token_ids]
120
121
- logits = hidden_states.softmax(dim=-1)
121
step_tag_id = self.step_tag_id
122
123
offset = 0
124
pooled_data_lst = []
125
for prompt_len, seq_data_i in zip(
126
prompt_lens, pooling_metadata.seq_data.values()):
128
- pooled_data_i = logits[offset:offset + prompt_len]
127
+ pooled_data_i = hidden_states[offset:offset + prompt_len]
128
if step_tag_id is not None:
129
token_ids = torch.tensor(seq_data_i.prompt_token_ids)
130
pooled_data_i = pooled_data_i[token_ids == step_tag_id]