@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / eeab52a4ff
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

[FIX] Simpler fix for async engine running on ray (#3371)

Zhuohan Li committed Mar 13, 2024 at 14:18 UTC eeab52a4ff02e15f970880a689df2861ad173770
1 file changed +1 -2
vllm/executor/ray_gpu_executor.py
+1 -2
@@ -430,8 +430,7 @@ class RayGPUExecutorAsync(RayGPUExecutor, ExecutorAsyncBase):
430 "blocks_to_swap_in": blocks_to_swap_in,
431 "blocks_to_swap_out": blocks_to_swap_out,
432 "blocks_to_copy": blocks_to_copy,
433 - },
434 - use_ray_compiled_dag=USE_RAY_COMPILED_DAG)
433 + })
434
435 # Only the driver worker returns the sampling results.
436 output = all_outputs[0]