@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 708989341e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[misc] add a flag to enable compile (#7092)

youkaichao committed Aug 2, 2024 at 16:18 UTC 708989341ef6361a5981d890a0e2f1b794323458
2 files changed +10
vllm/envs.py
+4
@@ -174,6 +174,10 @@ environment_variables: Dict[str, Callable[[], Any]] = {
174 lambda: (os.environ.get("VLLM_USE_TRITON_FLASH_ATTN", "True").lower() in
175 ("true", "1")),
176
177 + # Internal flag to enable Dynamo graph capture
178 + "VLLM_TEST_DYNAMO_GRAPH_CAPTURE":
179 + lambda: int(os.environ.get("VLLM_TEST_DYNAMO_GRAPH_CAPTURE", "0")),
180 +
181 # local rank of the process in the distributed setting, used to determine
182 # the GPU device id
183 "LOCAL_RANK":
vllm/worker/model_runner.py
+6
@@ -23,6 +23,7 @@ except ImportError:
23 BatchPrefillWithPagedKVCacheWrapper = None
24 FLASHINFER_WORKSPACE_BUFFER_SIZE = 0
25
26 +import vllm.envs as envs
27 from vllm.attention import AttentionMetadata, get_attn_backend
28 from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
29 ModelConfig, MultiModalConfig, ParallelConfig,
@@ -786,6 +787,11 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
787 "provided. Defaulting to scaling factors of 1.0. "
788 "This may lead to less accurate results!")
789
790 + if envs.VLLM_TEST_DYNAMO_GRAPH_CAPTURE:
791 + self.model = torch.compile(self.model,
792 + fullgraph=True,
793 + backend="eager")
794 +
795 def save_sharded_state(
796 self,
797 path: str,