@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 6d93d35308
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

[BugFix] tensor.get_device() -> tensor.device (#3604)

Kunshang Ji committed Mar 25, 2024 at 10:01 UTC 6d93d35308fc34bf76e2456c25be297b095ae902
1 file changed +2 -2
vllm/model_executor/layers/rotary_embedding.py
+2 -2
@@ -108,7 +108,7 @@ class RotaryEmbedding(nn.Module):
108 query_pass = query[..., self.rotary_dim:]
109 key_pass = key[..., self.rotary_dim:]
110
111 - self.cos_sin_cache = self.cos_sin_cache.to(positions.get_device())
111 + self.cos_sin_cache = self.cos_sin_cache.to(positions.device)
112 cos_sin = self.cos_sin_cache[torch.add(positions, offsets)
113 if offsets is not None else positions]
114 cos, sin = cos_sin.chunk(2, dim=-1)
@@ -142,7 +142,7 @@ class RotaryEmbedding(nn.Module):
142 key: torch.Tensor,
143 offsets: Optional[torch.Tensor] = None,
144 ) -> Tuple[torch.Tensor, torch.Tensor]:
145 - self.cos_sin_cache = self.cos_sin_cache.to(positions.get_device())
145 + self.cos_sin_cache = self.cos_sin_cache.to(positions.device)
146 # ops.rotary_embedding()/batched_rotary_embedding()
147 # are in-place operations that update the query and key tensors.
148 if offsets is not None: