A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[misc] add a flag to enable compile (#7092)
youkaichao committed
Aug 2, 2024 at 16:18 UTC
708989341ef6361a5981d890a0e2f1b794323458
2 files changed
+10
vllm/envs.py
+4
@@ -174,6 +174,10 @@ environment_variables: Dict[str, Callable[[], Any]] = {
174
lambda: (os.environ.get("VLLM_USE_TRITON_FLASH_ATTN", "True").lower() in
175
("true", "1")),
176
177
+ # Internal flag to enable Dynamo graph capture
178
+ "VLLM_TEST_DYNAMO_GRAPH_CAPTURE":
179
+ lambda: int(os.environ.get("VLLM_TEST_DYNAMO_GRAPH_CAPTURE", "0")),
180
+
181
# local rank of the process in the distributed setting, used to determine
182
# the GPU device id
183
"LOCAL_RANK":
vllm/worker/model_runner.py
+6
@@ -23,6 +23,7 @@ except ImportError:
23
BatchPrefillWithPagedKVCacheWrapper = None
24
FLASHINFER_WORKSPACE_BUFFER_SIZE = 0
25
26
+import vllm.envs as envs
27
from vllm.attention import AttentionMetadata, get_attn_backend
28
from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
29
ModelConfig, MultiModalConfig, ParallelConfig,
@@ -786,6 +787,11 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
787
"provided. Defaulting to scaling factors of 1.0. "
788
"This may lead to less accurate results!")
789
790
+ if envs.VLLM_TEST_DYNAMO_GRAPH_CAPTURE:
791
+ self.model = torch.compile(self.model,
792
+ fullgraph=True,
793
+ backend="eager")
794
+
795
def save_sharded_state(
796
self,
797
path: str,