@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 6806998bf9
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 23:15 UTC

[Bugfix] Fix embedding to support 2D inputs (#5829)

Woosuk Kwon committed Jun 26, 2024 at 00:15 UTC 6806998bf9c7f24d710d9017c901e9e9a30757d5
1 file changed +2 -2
vllm/model_executor/layers/vocab_parallel_embedding.py
+2 -2
@@ -306,11 +306,11 @@ class VocabParallelEmbedding(torch.nn.Module):
306 self.shard_indices.added_vocab_end_index)
307 else:
308 masked_input = input_
309 - # Get the embeddings.
309 + # Get the embeddings.
310 output_parallel = F.embedding(masked_input.long(), self.weight)
311 # Mask the output embedding.
312 if self.tp_size > 1:
313 - output_parallel.masked_fill_(input_mask.unsqueeze(1), 0)
313 + output_parallel.masked_fill_(input_mask.unsqueeze(-1), 0)
314 # Reduce across all the model parallel GPUs.
315 output = tensor_model_parallel_all_reduce(output_parallel)
316 return output