A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
[Core/Bugfix] pass VLLM_ATTENTION_BACKEND to ray workers (#8172)
William Lin committed
Sep 10, 2024 at 16:46 UTC
1d5e397aa4d94d0ccc1c9dbad533afa5cb60bb69
1 file changed
+3
vllm/executor/ray_gpu_executor.py
+3
@@ -242,6 +242,9 @@ class RayGPUExecutor(DistributedGPUExecutor):
242
VLLM_INSTANCE_ID,
243
"VLLM_TRACE_FUNCTION":
244
str(envs.VLLM_TRACE_FUNCTION),
245
+ **({
246
+ "VLLM_ATTENTION_BACKEND": envs.VLLM_ATTENTION_BACKEND
247
+ } if envs.VLLM_ATTENTION_BACKEND is not None else {})
248
}, ) for (node_id, _) in worker_node_and_gpu_ids]
249
250
self._env_vars_for_all_workers = (