A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
[FIX] Simpler fix for async engine running on ray (#3371)
Zhuohan Li committed
Mar 13, 2024 at 14:18 UTC
eeab52a4ff02e15f970880a689df2861ad173770
1 file changed
+1
-2
vllm/executor/ray_gpu_executor.py
+1
-2
@@ -430,8 +430,7 @@ class RayGPUExecutorAsync(RayGPUExecutor, ExecutorAsyncBase):
430
"blocks_to_swap_in": blocks_to_swap_in,
431
"blocks_to_swap_out": blocks_to_swap_out,
432
"blocks_to_copy": blocks_to_copy,
433
- },
434
- use_ray_compiled_dag=USE_RAY_COMPILED_DAG)
433
+ })
434
435
# Only the driver worker returns the sampling results.
436
output = all_outputs[0]