A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 03:15 UTC
[v1][bugfix] fix cudagraph with inplace buffer assignment (#11596)
Signed-off-by: youkaichao <youkaichao@gmail.com>
youkaichao committed
Dec 29, 2024 at 17:03 UTC
dba4d9dec606da028fbb28240e99cabd5a761e6a
2 files changed
+10
-11
vllm/compilation/wrapper.py
+9
-1
@@ -28,11 +28,12 @@ class TorchCompileWrapperWithCustomDispatcher:
28
compiled_callable: Optional[Callable] = None,
29
compilation_level: int = 0):
30
31
+ vllm_config = get_current_vllm_config()
32
+ self.vllm_config = vllm_config
33
if compiled_callable is None:
34
# default compilation settings
35
# compiling the forward method
36
35
- vllm_config = get_current_vllm_config()
37
backend = vllm_config.compilation_config.init_backend(vllm_config)
38
39
compiled_callable = torch.compile(
@@ -82,6 +83,13 @@ class TorchCompileWrapperWithCustomDispatcher:
83
84
self.compiled_codes.append(new_code)
85
86
+ if self.vllm_config.compilation_config.use_cudagraph and \
87
+ "update" in new_code.co_names:
88
+ import depyf
89
+ src = depyf.decompile(new_code)
90
+ msg = "Assigning / modifying buffers of nn.Module during forward pass is not allowed when using cudagraph inside the compiler because it will cause silent errors. Please use eager mode or fix the code. The following code contains clues about which buffer is being modified (please search for the usage of the function `update`):\n" + src # noqa
91
+ raise RuntimeError(msg)
92
+
93
@contextmanager
94
def dispatch_to_code(self, index: int):
95
"""Context manager to dispatch to the compiled code.
vllm/model_executor/layers/rotary_embedding.py
+1
-10
@@ -541,19 +541,12 @@ class Phi3LongRoPEScaledRotaryEmbedding(nn.Module):
541
short_cache = self._compute_cos_sin_cache(
542
original_max_position_embeddings, short_factor, short_mscale)
543
short_cache = short_cache.to(dtype)
544
- self.register_buffer("short_cos_sin_cache",
545
- short_cache,
546
- persistent=False)
544
545
long_cache = self._compute_cos_sin_cache(max_position_embeddings,
546
long_factor, long_mscale)
547
long_cache = long_cache.to(dtype)
551
- self.register_buffer("long_cos_sin_cache",
552
- long_cache,
553
- persistent=False)
548
555
- long_short_cache = torch.cat(
556
- [self.short_cos_sin_cache, self.long_cos_sin_cache], dim=0)
549
+ long_short_cache = torch.cat([short_cache, long_cache], dim=0)
550
self.register_buffer("long_short_cos_sin_cache",
551
long_short_cache,
552
persistent=False)
@@ -593,8 +586,6 @@ class Phi3LongRoPEScaledRotaryEmbedding(nn.Module):
586
torch.full_like(positions, k)).long()
587
idx = (torch.add(positions, long_prompt_offset)
588
if long_prompt_offset is not None else positions)
596
- self.long_short_cos_sin_cache: torch.Tensor = (
597
- self.long_short_cos_sin_cache.to(idx.device))
589
idx = torch.add(idx, offsets) if offsets is not None else idx
590
cos_sin = torch.index_select(self.long_short_cos_sin_cache, 0, idx)
591