@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 1d5e397aa4
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 17:00 UTC

[Core/Bugfix] pass VLLM_ATTENTION_BACKEND to ray workers (#8172)

William Lin committed Sep 10, 2024 at 16:46 UTC 1d5e397aa4d94d0ccc1c9dbad533afa5cb60bb69
1 file changed +3
vllm/executor/ray_gpu_executor.py
+3
@@ -242,6 +242,9 @@ class RayGPUExecutor(DistributedGPUExecutor):
242 VLLM_INSTANCE_ID,
243 "VLLM_TRACE_FUNCTION":
244 str(envs.VLLM_TRACE_FUNCTION),
245 + **({
246 + "VLLM_ATTENTION_BACKEND": envs.VLLM_ATTENTION_BACKEND
247 + } if envs.VLLM_ATTENTION_BACKEND is not None else {})
248 }, ) for (node_id, _) in worker_node_and_gpu_ids]
249
250 self._env_vars_for_all_workers = (