@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 01aae1cc68
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 07:00 UTC

[Model] Remove redundant softmax when using PoolingType.STEP (#10415)

Maybewuss committed Nov 18, 2024 at 18:05 UTC 01aae1cc68d6013dd91e87418a6d82fa02c58457
1 file changed +1 -2
vllm/model_executor/layers/pooler.py
+1 -2
@@ -118,14 +118,13 @@ class Pooler(nn.Module):
118 if returned_token_ids is not None and len(returned_token_ids) > 0:
119 hidden_states = hidden_states[:, returned_token_ids]
120
121 - logits = hidden_states.softmax(dim=-1)
121 step_tag_id = self.step_tag_id
122
123 offset = 0
124 pooled_data_lst = []
125 for prompt_len, seq_data_i in zip(
126 prompt_lens, pooling_metadata.seq_data.values()):
128 - pooled_data_i = logits[offset:offset + prompt_len]
127 + pooled_data_i = hidden_states[offset:offset + prompt_len]
128 if step_tag_id is not None:
129 token_ids = torch.tensor(seq_data_i.prompt_token_ids)
130 pooled_data_i = pooled_data_i[token_ids == step_tag_id]