A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
[BugFix] tensor.get_device() -> tensor.device (#3604)
Kunshang Ji committed
Mar 25, 2024 at 10:01 UTC
6d93d35308fc34bf76e2456c25be297b095ae902
1 file changed
+2
-2
vllm/model_executor/layers/rotary_embedding.py
+2
-2
@@ -108,7 +108,7 @@ class RotaryEmbedding(nn.Module):
108
query_pass = query[..., self.rotary_dim:]
109
key_pass = key[..., self.rotary_dim:]
110
111
- self.cos_sin_cache = self.cos_sin_cache.to(positions.get_device())
111
+ self.cos_sin_cache = self.cos_sin_cache.to(positions.device)
112
cos_sin = self.cos_sin_cache[torch.add(positions, offsets)
113
if offsets is not None else positions]
114
cos, sin = cos_sin.chunk(2, dim=-1)
@@ -142,7 +142,7 @@ class RotaryEmbedding(nn.Module):
142
key: torch.Tensor,
143
offsets: Optional[torch.Tensor] = None,
144
) -> Tuple[torch.Tensor, torch.Tensor]:
145
- self.cos_sin_cache = self.cos_sin_cache.to(positions.get_device())
145
+ self.cos_sin_cache = self.cos_sin_cache.to(positions.device)
146
# ops.rotary_embedding()/batched_rotary_embedding()
147
# are in-place operations that update the query and key tensors.
148
if offsets is not None: