A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
[Core] Fix engine-use-ray broken (#4105)
SangBin Cho committed
Apr 16, 2024 at 14:24 UTC
4e7ee664e201442e24e2298a36a5264b98691626
2 files changed
+16
-8
tests/async_engine/test_api_server.py
+13
-4
@@ -25,21 +25,30 @@ def _query_server_long(prompt: str) -> dict:
25
26
27
@pytest.fixture
28
-def api_server(tokenizer_pool_size: int):
28
+def api_server(tokenizer_pool_size: int, engine_use_ray: bool,
29
+ worker_use_ray: bool):
30
script_path = Path(__file__).parent.joinpath(
31
"api_server_async_engine.py").absolute()
31
- uvicorn_process = subprocess.Popen([
32
+ commands = [
33
sys.executable, "-u",
34
str(script_path), "--model", "facebook/opt-125m", "--host",
35
"127.0.0.1", "--tokenizer-pool-size",
36
str(tokenizer_pool_size)
36
- ])
37
+ ]
38
+ if engine_use_ray:
39
+ commands.append("--engine-use-ray")
40
+ if worker_use_ray:
41
+ commands.append("--worker-use-ray")
42
+ uvicorn_process = subprocess.Popen(commands)
43
yield
44
uvicorn_process.terminate()
45
46
47
@pytest.mark.parametrize("tokenizer_pool_size", [0, 2])
42
-def test_api_server(api_server, tokenizer_pool_size: int):
48
+@pytest.mark.parametrize("worker_use_ray", [False, True])
49
+@pytest.mark.parametrize("engine_use_ray", [False, True])
50
+def test_api_server(api_server, tokenizer_pool_size: int, worker_use_ray: bool,
51
+ engine_use_ray: bool):
52
"""
53
Run the API server and test it.
54
vllm/engine/async_llm_engine.py
+3
-4
@@ -333,8 +333,7 @@ class AsyncLLMEngine:
333
if engine_config.device_config.device_type == "neuron":
334
raise NotImplementedError("Neuron is not supported for "
335
"async engine yet.")
336
- elif (engine_config.parallel_config.worker_use_ray
337
- or engine_args.engine_use_ray):
336
+ elif engine_config.parallel_config.worker_use_ray:
337
initialize_ray_cluster(engine_config.parallel_config)
338
from vllm.executor.ray_gpu_executor import RayGPUExecutorAsync
339
executor_class = RayGPUExecutorAsync
@@ -410,8 +409,8 @@ class AsyncLLMEngine:
409
else:
410
# FIXME(woosuk): This is a bit hacky. Be careful when changing the
411
# order of the arguments.
413
- cache_config = args[1]
414
- parallel_config = args[2]
412
+ cache_config = kwargs["cache_config"]
413
+ parallel_config = kwargs["parallel_config"]
414
if parallel_config.tensor_parallel_size == 1:
415
num_gpus = cache_config.gpu_memory_utilization
416
else: