@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / dba4d9dec6
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 03:15 UTC

[v1][bugfix] fix cudagraph with inplace buffer assignment (#11596)

Signed-off-by: youkaichao <youkaichao@gmail.com>

youkaichao committed Dec 29, 2024 at 17:03 UTC dba4d9dec606da028fbb28240e99cabd5a761e6a
2 files changed +10 -11
vllm/compilation/wrapper.py
+9 -1
@@ -28,11 +28,12 @@ class TorchCompileWrapperWithCustomDispatcher:
28 compiled_callable: Optional[Callable] = None,
29 compilation_level: int = 0):
30
31 + vllm_config = get_current_vllm_config()
32 + self.vllm_config = vllm_config
33 if compiled_callable is None:
34 # default compilation settings
35 # compiling the forward method
36
35 - vllm_config = get_current_vllm_config()
37 backend = vllm_config.compilation_config.init_backend(vllm_config)
38
39 compiled_callable = torch.compile(
@@ -82,6 +83,13 @@ class TorchCompileWrapperWithCustomDispatcher:
83
84 self.compiled_codes.append(new_code)
85
86 + if self.vllm_config.compilation_config.use_cudagraph and \
87 + "update" in new_code.co_names:
88 + import depyf
89 + src = depyf.decompile(new_code)
90 + msg = "Assigning / modifying buffers of nn.Module during forward pass is not allowed when using cudagraph inside the compiler because it will cause silent errors. Please use eager mode or fix the code. The following code contains clues about which buffer is being modified (please search for the usage of the function `update`):\n" + src # noqa
91 + raise RuntimeError(msg)
92 +
93 @contextmanager
94 def dispatch_to_code(self, index: int):
95 """Context manager to dispatch to the compiled code.
vllm/model_executor/layers/rotary_embedding.py
+1 -10
@@ -541,19 +541,12 @@ class Phi3LongRoPEScaledRotaryEmbedding(nn.Module):
541 short_cache = self._compute_cos_sin_cache(
542 original_max_position_embeddings, short_factor, short_mscale)
543 short_cache = short_cache.to(dtype)
544 - self.register_buffer("short_cos_sin_cache",
545 - short_cache,
546 - persistent=False)
544
545 long_cache = self._compute_cos_sin_cache(max_position_embeddings,
546 long_factor, long_mscale)
547 long_cache = long_cache.to(dtype)
551 - self.register_buffer("long_cos_sin_cache",
552 - long_cache,
553 - persistent=False)
548
555 - long_short_cache = torch.cat(
556 - [self.short_cos_sin_cache, self.long_cos_sin_cache], dim=0)
549 + long_short_cache = torch.cat([short_cache, long_cache], dim=0)
550 self.register_buffer("long_short_cos_sin_cache",
551 long_short_cache,
552 persistent=False)
@@ -593,8 +586,6 @@ class Phi3LongRoPEScaledRotaryEmbedding(nn.Module):
586 torch.full_like(positions, k)).long()
587 idx = (torch.add(positions, long_prompt_offset)
588 if long_prompt_offset is not None else positions)
596 - self.long_short_cos_sin_cache: torch.Tensor = (
597 - self.long_short_cos_sin_cache.to(idx.device))
589 idx = torch.add(idx, offsets) if offsets is not None else idx
590 cos_sin = torch.index_select(self.long_short_cos_sin_cache, 0, idx)
591