@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 4e7ee664e2
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 03:00 UTC

[Core] Fix engine-use-ray broken (#4105)

SangBin Cho committed Apr 16, 2024 at 14:24 UTC 4e7ee664e201442e24e2298a36a5264b98691626
2 files changed +16 -8
tests/async_engine/test_api_server.py
+13 -4
@@ -25,21 +25,30 @@ def _query_server_long(prompt: str) -> dict:
25
26
27 @pytest.fixture
28 -def api_server(tokenizer_pool_size: int):
28 +def api_server(tokenizer_pool_size: int, engine_use_ray: bool,
29 + worker_use_ray: bool):
30 script_path = Path(__file__).parent.joinpath(
31 "api_server_async_engine.py").absolute()
31 - uvicorn_process = subprocess.Popen([
32 + commands = [
33 sys.executable, "-u",
34 str(script_path), "--model", "facebook/opt-125m", "--host",
35 "127.0.0.1", "--tokenizer-pool-size",
36 str(tokenizer_pool_size)
36 - ])
37 + ]
38 + if engine_use_ray:
39 + commands.append("--engine-use-ray")
40 + if worker_use_ray:
41 + commands.append("--worker-use-ray")
42 + uvicorn_process = subprocess.Popen(commands)
43 yield
44 uvicorn_process.terminate()
45
46
47 @pytest.mark.parametrize("tokenizer_pool_size", [0, 2])
42 -def test_api_server(api_server, tokenizer_pool_size: int):
48 +@pytest.mark.parametrize("worker_use_ray", [False, True])
49 +@pytest.mark.parametrize("engine_use_ray", [False, True])
50 +def test_api_server(api_server, tokenizer_pool_size: int, worker_use_ray: bool,
51 + engine_use_ray: bool):
52 """
53 Run the API server and test it.
54
vllm/engine/async_llm_engine.py
+3 -4
@@ -333,8 +333,7 @@ class AsyncLLMEngine:
333 if engine_config.device_config.device_type == "neuron":
334 raise NotImplementedError("Neuron is not supported for "
335 "async engine yet.")
336 - elif (engine_config.parallel_config.worker_use_ray
337 - or engine_args.engine_use_ray):
336 + elif engine_config.parallel_config.worker_use_ray:
337 initialize_ray_cluster(engine_config.parallel_config)
338 from vllm.executor.ray_gpu_executor import RayGPUExecutorAsync
339 executor_class = RayGPUExecutorAsync
@@ -410,8 +409,8 @@ class AsyncLLMEngine:
409 else:
410 # FIXME(woosuk): This is a bit hacky. Be careful when changing the
411 # order of the arguments.
413 - cache_config = args[1]
414 - parallel_config = args[2]
412 + cache_config = kwargs["cache_config"]
413 + parallel_config = kwargs["parallel_config"]
414 if parallel_config.tensor_parallel_size == 1:
415 num_gpus = cache_config.gpu_memory_utilization
416 else: