@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 41708e5034
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 22:00 UTC

[ci] try to add multi-node tests (#6280)

Signed-off-by: Muralidhar Andoorveedu <muralidhar.andoorveedu@centml.ai> Co-authored-by: Muralidhar Andoorveedu <muralidhar.andoorveedu@centml.ai>

youkaichao committed Jul 12, 2024 at 21:51 UTC 41708e50341c82668fd25ebc7777470cba6f5303
13 files changed +230 -275
.buildkite/run-multi-node-test.sh
+40 -12
@@ -2,16 +2,17 @@
2
3 set -euox pipefail
4
5 -if [[ $# -lt 3 ]]; then
6 - echo "Please provide the number of nodes and GPU per node."
5 +if [[ $# -lt 4 ]]; then
6 + echo "Usage: .buildkite/run-multi-node-test.sh WORKING_DIR NUM_NODES NUM_GPUS DOCKER_IMAGE COMMAND1 COMMAND2 ... COMMANDN"
7 exit 1
8 fi
9
10 -NUM_NODES=$1
11 -NUM_GPUS=$2
12 -DOCKER_IMAGE=$3
10 +WORKING_DIR=$1
11 +NUM_NODES=$2
12 +NUM_GPUS=$3
13 +DOCKER_IMAGE=$4
14
14 -shift 3
15 +shift 4
16 COMMANDS=("$@")
17 if [ ${#COMMANDS[@]} -ne $NUM_NODES ]; then
18 echo "The number of commands must be equal to the number of nodes."
@@ -40,13 +41,40 @@ start_nodes() {
41 fi
42 done
43 GPU_DEVICES+='"'
43 - # echo "Starting node$node with GPU devices: $GPU_DEVICES"
44 - docker run -d --gpus "$GPU_DEVICES" --name node$node --network docker-net --ip 192.168.10.$((10 + $node)) --rm $DOCKER_IMAGE tail -f /dev/null
44 +
45 + # start the container in detached mode
46 + # things to note:
47 + # 1. --shm-size=10.24gb is required. don't use --ipc=host
48 + # 2. pass HF_TOKEN to the container
49 + # 3. map the huggingface cache directory to the container
50 + # 3. assign ip addresses to the containers (head node: 192.168.10.10, worker nodes:
51 + # starting from 192.168.10.11)
52 + docker run -d --gpus "$GPU_DEVICES" --shm-size=10.24gb -e HF_TOKEN -v ~/.cache/huggingface:/root/.cache/huggingface --name node$node --network docker-net --ip 192.168.10.$((10 + $node)) --rm $DOCKER_IMAGE /bin/bash -c "tail -f /dev/null"
53 +
54 + # organize containers into a ray cluster
55 + if [ $node -eq 0 ]; then
56 + # start the ray head node
57 + docker exec -d node$node /bin/bash -c "ray start --head --port=6379 --block"
58 + # wait for the head node to be ready
59 + sleep 10
60 + else
61 + # start the ray worker nodes, and connect them to the head node
62 + docker exec -d node$node /bin/bash -c "ray start --address=192.168.10.10:6379 --block"
63 + fi
64 done
65 +
66 + # wait for the cluster to be ready
67 + sleep 10
68 +
69 + # print the cluster status
70 + docker exec node0 /bin/bash -c "ray status"
71 }
72
73 run_nodes() {
49 - for node in $(seq 0 $(($NUM_NODES-1))); do
74 + # important: iterate in reverse order to start the head node last
75 + # we start the worker nodes first, in detached mode, and then start the head node
76 + # in the foreground, so that the output of the head node is visible in the buildkite logs
77 + for node in $(seq $(($NUM_NODES - 1)) -1 0); do
78 GPU_DEVICES='"device='
79 for node_gpu in $(seq 0 $(($NUM_GPUS - 1))); do
80 DEVICE_NUM=$(($node * $NUM_GPUS + $node_gpu))
@@ -57,10 +85,10 @@ run_nodes() {
85 done
86 GPU_DEVICES+='"'
87 echo "Running node$node with GPU devices: $GPU_DEVICES"
60 - if [ $node -lt $(($NUM_NODES - 1)) ]; then
61 - docker exec -d node$node /bin/bash -c "${COMMANDS[$node]}"
88 + if [ $node -ne 0 ]; then
89 + docker exec -d node$node /bin/bash -c "cd $WORKING_DIR ; ${COMMANDS[$node]}"
90 else
63 - docker exec node$node /bin/bash -c "${COMMANDS[$node]}"
91 + docker exec node$node /bin/bash -c "cd $WORKING_DIR ; ${COMMANDS[$node]}"
92 fi
93 done
94 }
.buildkite/test-pipeline.yaml
+15 -1
@@ -68,6 +68,17 @@ steps:
68 - pytest -v -s distributed/test_comm_ops.py
69 - pytest -v -s distributed/test_shm_broadcast.py
70
71 +- label: 2 Node Tests (4 GPUs in total)
72 + working_dir: "/vllm-workspace/tests"
73 + num_gpus: 2
74 + num_nodes: 2
75 + commands:
76 + - # the following commands are for the first node, with ip 192.168.10.10 (ray environment already set up)
77 + - VLLM_TEST_SAME_HOST=0 torchrun --nnodes 2 --nproc-per-node=2 --rdzv_backend=c10d --rdzv_endpoint=192.168.10.10 distributed/test_same_node.py
78 + - TP_SIZE=2 PP_SIZE=2 EAGER_MODE=1 CHUNKED_PREFILL=0 pytest -v -s distributed/test_pipeline_parallel.py
79 + - # the following commands are for the second node, with ip 192.168.10.11 (ray environment already set up)
80 + - VLLM_TEST_SAME_HOST=0 torchrun --nnodes 2 --nproc-per-node=2 --rdzv_backend=c10d --rdzv_endpoint=192.168.10.10 distributed/test_same_node.py
81 +
82 - label: Distributed Tests (2 GPUs)
83 mirror_hardwares: [amd]
84 working_dir: "/vllm-workspace/tests"
@@ -213,7 +224,10 @@ steps:
224
225 - label: Tensorizer Test
226 #mirror_hardwares: [amd]
216 - command: apt-get install curl libsodium23 && pytest -v -s tensorizer_loader
227 + commands:
228 + - apt-get install curl libsodium23
229 + - export VLLM_WORKER_MULTIPROC_METHOD=spawn
230 + - pytest -v -s tensorizer_loader
231
232 - label: Metrics Test
233 mirror_hardwares: [amd]
tests/async_engine/test_openapi_server_ray.py
+14 -23
@@ -1,35 +1,26 @@
1 import openai # use the official client for correctness check
2 import pytest
3 -# using Ray for overall ease of process management, parallel requests,
4 -# and debugging.
5 -import ray
3
7 -from ..utils import VLLM_PATH, RemoteOpenAIServer
4 +from ..utils import RemoteOpenAIServer
5
6 # any model with a chat template should work here
7 MODEL_NAME = "facebook/opt-125m"
8
9
10 @pytest.fixture(scope="module")
14 -def ray_ctx():
15 - ray.init(runtime_env={"working_dir": VLLM_PATH})
16 - yield
17 - ray.shutdown()
18 -
19 -
20 -@pytest.fixture(scope="module")
21 -def server(ray_ctx):
22 - return RemoteOpenAIServer([
23 - "--model",
24 - MODEL_NAME,
25 - # use half precision for speed and memory savings in CI environment
26 - "--dtype",
27 - "float16",
28 - "--max-model-len",
29 - "2048",
30 - "--enforce-eager",
31 - "--engine-use-ray"
32 - ])
11 +def server():
12 + with RemoteOpenAIServer([
13 + "--model",
14 + MODEL_NAME,
15 + # use half precision for speed and memory savings in CI environment
16 + "--dtype",
17 + "float16",
18 + "--max-model-len",
19 + "2048",
20 + "--enforce-eager",
21 + "--engine-use-ray"
22 + ]) as remote_server:
23 + yield remote_server
24
25
26 @pytest.fixture(scope="module")
tests/distributed/test_pipeline_parallel.py
+4 -13
@@ -2,11 +2,8 @@ import os
2
3 import openai # use the official client for correctness check
4 import pytest
5 -# using Ray for overall ease of process management, parallel requests,
6 -# and debugging.
7 -import ray
5
9 -from ..utils import VLLM_PATH, RemoteOpenAIServer
6 +from ..utils import RemoteOpenAIServer
7
8 # downloading lora to test lora requests
9
@@ -21,14 +18,7 @@ pytestmark = pytest.mark.asyncio
18
19
20 @pytest.fixture(scope="module")
24 -def ray_ctx():
25 - ray.init(runtime_env={"working_dir": VLLM_PATH})
26 - yield
27 - ray.shutdown()
28 -
29 -
30 -@pytest.fixture(scope="module")
31 -def server(ray_ctx):
21 +def server():
22 args = [
23 "--model",
24 MODEL_NAME,
@@ -50,7 +40,8 @@ def server(ray_ctx):
40 args += [
41 "--enforce-eager",
42 ]
53 - return RemoteOpenAIServer(args, num_gpus=PP_SIZE * TP_SIZE)
43 + with RemoteOpenAIServer(args) as remote_server:
44 + yield remote_server
45
46
47 @pytest.fixture(scope="module")
tests/distributed/test_same_node.py
+1
@@ -10,3 +10,4 @@ test_result = all(
10
11 expected = os.environ.get("VLLM_TEST_SAME_HOST", "1") == "1"
12 assert test_result == expected, f"Expected {expected}, got {test_result}"
13 +print("Same node test passed!")
tests/entrypoints/openai/test_chat.py
+24 -33
@@ -6,15 +6,12 @@ from typing import List
6 import jsonschema
7 import openai # use the official client for correctness check
8 import pytest
9 -# using Ray for overall ease of process management, parallel requests,
10 -# and debugging.
11 -import ray
9 import torch
10 # downloading lora to test lora requests
11 from huggingface_hub import snapshot_download
12 from openai import BadRequestError
13
17 -from ...utils import VLLM_PATH, RemoteOpenAIServer
14 +from ...utils import RemoteOpenAIServer
15
16 # any model with a chat template should work here
17 MODEL_NAME = "HuggingFaceH4/zephyr-7b-beta"
@@ -29,35 +26,29 @@ def zephyr_lora_files():
26
27
28 @pytest.fixture(scope="module")
32 -def ray_ctx():
33 - ray.init(runtime_env={"working_dir": VLLM_PATH})
34 - yield
35 - ray.shutdown()
36 -
37 -
38 -@pytest.fixture(scope="module")
39 -def server(zephyr_lora_files, ray_ctx):
40 - return RemoteOpenAIServer([
41 - "--model",
42 - MODEL_NAME,
43 - # use half precision for speed and memory savings in CI environment
44 - "--dtype",
45 - "bfloat16",
46 - "--max-model-len",
47 - "8192",
48 - "--enforce-eager",
49 - # lora config below
50 - "--enable-lora",
51 - "--lora-modules",
52 - f"zephyr-lora={zephyr_lora_files}",
53 - f"zephyr-lora2={zephyr_lora_files}",
54 - "--max-lora-rank",
55 - "64",
56 - "--max-cpu-loras",
57 - "2",
58 - "--max-num-seqs",
59 - "128",
60 - ])
29 +def server(zephyr_lora_files):
30 + with RemoteOpenAIServer([
31 + "--model",
32 + MODEL_NAME,
33 + # use half precision for speed and memory savings in CI environment
34 + "--dtype",
35 + "bfloat16",
36 + "--max-model-len",
37 + "8192",
38 + "--enforce-eager",
39 + # lora config below
40 + "--enable-lora",
41 + "--lora-modules",
42 + f"zephyr-lora={zephyr_lora_files}",
43 + f"zephyr-lora2={zephyr_lora_files}",
44 + "--max-lora-rank",
45 + "64",
46 + "--max-cpu-loras",
47 + "2",
48 + "--max-num-seqs",
49 + "128",
50 + ]) as remote_server:
51 + yield remote_server
52
53
54 @pytest.fixture(scope="module")
tests/entrypoints/openai/test_completion.py
+24 -33
@@ -6,9 +6,6 @@ from typing import List
6 import jsonschema
7 import openai # use the official client for correctness check
8 import pytest
9 -# using Ray for overall ease of process management, parallel requests,
10 -# and debugging.
11 -import ray
9 import requests
10 # downloading lora to test lora requests
11 from huggingface_hub import snapshot_download
@@ -16,7 +13,7 @@ from openai import BadRequestError
13
14 from vllm.transformers_utils.tokenizer import get_tokenizer
15
19 -from ...utils import VLLM_PATH, RemoteOpenAIServer
16 +from ...utils import RemoteOpenAIServer
17
18 # any model with a chat template should work here
19 MODEL_NAME = "HuggingFaceH4/zephyr-7b-beta"
@@ -31,35 +28,29 @@ def zephyr_lora_files():
28
29
30 @pytest.fixture(scope="module")
34 -def ray_ctx():
35 - ray.init(runtime_env={"working_dir": VLLM_PATH})
36 - yield
37 - ray.shutdown()
38 -
39 -
40 -@pytest.fixture(scope="module")
41 -def server(zephyr_lora_files, ray_ctx):
42 - return RemoteOpenAIServer([
43 - "--model",
44 - MODEL_NAME,
45 - # use half precision for speed and memory savings in CI environment
46 - "--dtype",
47 - "bfloat16",
48 - "--max-model-len",
49 - "8192",
50 - "--enforce-eager",
51 - # lora config below
52 - "--enable-lora",
53 - "--lora-modules",
54 - f"zephyr-lora={zephyr_lora_files}",
55 - f"zephyr-lora2={zephyr_lora_files}",
56 - "--max-lora-rank",
57 - "64",
58 - "--max-cpu-loras",
59 - "2",
60 - "--max-num-seqs",
61 - "128",
62 - ])
31 +def server(zephyr_lora_files):
32 + with RemoteOpenAIServer([
33 + "--model",
34 + MODEL_NAME,
35 + # use half precision for speed and memory savings in CI environment
36 + "--dtype",
37 + "bfloat16",
38 + "--max-model-len",
39 + "8192",
40 + "--enforce-eager",
41 + # lora config below
42 + "--enable-lora",
43 + "--lora-modules",
44 + f"zephyr-lora={zephyr_lora_files}",
45 + f"zephyr-lora2={zephyr_lora_files}",
46 + "--max-lora-rank",
47 + "64",
48 + "--max-cpu-loras",
49 + "2",
50 + "--max-num-seqs",
51 + "128",
52 + ]) as remote_server:
53 + yield remote_server
54
55
56 @pytest.fixture(scope="module")
tests/entrypoints/openai/test_embedding.py
+14 -21
@@ -3,33 +3,26 @@ import base64
3 import numpy as np
4 import openai
5 import pytest
6 -import ray
6
8 -from ...utils import VLLM_PATH, RemoteOpenAIServer
7 +from ...utils import RemoteOpenAIServer
8
9 EMBEDDING_MODEL_NAME = "intfloat/e5-mistral-7b-instruct"
10
11
12 @pytest.fixture(scope="module")
14 -def ray_ctx():
15 - ray.init(runtime_env={"working_dir": VLLM_PATH})
16 - yield
17 - ray.shutdown()
18 -
19 -
20 -@pytest.fixture(scope="module")
21 -def embedding_server(ray_ctx):
22 - return RemoteOpenAIServer([
23 - "--model",
24 - EMBEDDING_MODEL_NAME,
25 - # use half precision for speed and memory savings in CI environment
26 - "--dtype",
27 - "bfloat16",
28 - "--enforce-eager",
29 - "--max-model-len",
30 - "8192",
31 - "--enforce-eager",
32 - ])
13 +def embedding_server():
14 + with RemoteOpenAIServer([
15 + "--model",
16 + EMBEDDING_MODEL_NAME,
17 + # use half precision for speed and memory savings in CI environment
18 + "--dtype",
19 + "bfloat16",
20 + "--enforce-eager",
21 + "--max-model-len",
22 + "8192",
23 + "--enforce-eager",
24 + ]) as remote_server:
25 + yield remote_server
26
27
28 @pytest.mark.asyncio
tests/entrypoints/openai/test_models.py
+24 -33
@@ -1,12 +1,9 @@
1 import openai # use the official client for correctness check
2 import pytest
3 -# using Ray for overall ease of process management, parallel requests,
4 -# and debugging.
5 -import ray
3 # downloading lora to test lora requests
4 from huggingface_hub import snapshot_download
5
9 -from ...utils import VLLM_PATH, RemoteOpenAIServer
6 +from ...utils import RemoteOpenAIServer
7
8 # any model with a chat template should work here
9 MODEL_NAME = "HuggingFaceH4/zephyr-7b-beta"
@@ -21,35 +18,29 @@ def zephyr_lora_files():
18
19
20 @pytest.fixture(scope="module")
24 -def ray_ctx():
25 - ray.init(runtime_env={"working_dir": VLLM_PATH})
26 - yield
27 - ray.shutdown()
28 -
29 -
30 -@pytest.fixture(scope="module")
31 -def server(zephyr_lora_files, ray_ctx):
32 - return RemoteOpenAIServer([
33 - "--model",
34 - MODEL_NAME,
35 - # use half precision for speed and memory savings in CI environment
36 - "--dtype",
37 - "bfloat16",
38 - "--max-model-len",
39 - "8192",
40 - "--enforce-eager",
41 - # lora config below
42 - "--enable-lora",
43 - "--lora-modules",
44 - f"zephyr-lora={zephyr_lora_files}",
45 - f"zephyr-lora2={zephyr_lora_files}",
46 - "--max-lora-rank",
47 - "64",
48 - "--max-cpu-loras",
49 - "2",
50 - "--max-num-seqs",
51 - "128",
52 - ])
21 +def server(zephyr_lora_files):
22 + with RemoteOpenAIServer([
23 + "--model",
24 + MODEL_NAME,
25 + # use half precision for speed and memory savings in CI environment
26 + "--dtype",
27 + "bfloat16",
28 + "--max-model-len",
29 + "8192",
30 + "--enforce-eager",
31 + # lora config below
32 + "--enable-lora",
33 + "--lora-modules",
34 + f"zephyr-lora={zephyr_lora_files}",
35 + f"zephyr-lora2={zephyr_lora_files}",
36 + "--max-lora-rank",
37 + "64",
38 + "--max-cpu-loras",
39 + "2",
40 + "--max-num-seqs",
41 + "128",
42 + ]) as remote_server:
43 + yield remote_server
44
45
46 @pytest.fixture(scope="module")
tests/entrypoints/openai/test_vision.py
+13 -20
@@ -3,7 +3,6 @@ from typing import Dict, List
3 import openai
4 import pytest
5 import pytest_asyncio
6 -import ray
6
7 from vllm.multimodal.utils import ImageFetchAiohttp, encode_image_base64
8
@@ -23,25 +22,19 @@ TEST_IMAGE_URLS = [
22
23
24 @pytest.fixture(scope="module")
26 -def ray_ctx():
27 - ray.init(runtime_env={"working_dir": VLLM_PATH})
28 - yield
29 - ray.shutdown()
30 -
31 -
32 -@pytest.fixture(scope="module")
33 -def server(ray_ctx):
34 - return RemoteOpenAIServer([
35 - "--model",
36 - MODEL_NAME,
37 - "--dtype",
38 - "bfloat16",
39 - "--max-model-len",
40 - "4096",
41 - "--enforce-eager",
42 - "--chat-template",
43 - str(LLAVA_CHAT_TEMPLATE),
44 - ])
25 +def server():
26 + with RemoteOpenAIServer([
27 + "--model",
28 + MODEL_NAME,
29 + "--dtype",
30 + "bfloat16",
31 + "--max-model-len",
32 + "4096",
33 + "--enforce-eager",
34 + "--chat-template",
35 + str(LLAVA_CHAT_TEMPLATE),
36 + ]) as remote_server:
37 + yield remote_server
38
39
40 @pytest.fixture(scope="module")
tests/tensorizer_loader/test_tensorizer.py
+14 -19
@@ -6,7 +6,6 @@ from unittest.mock import MagicMock, patch
6
7 import openai
8 import pytest
9 -import ray
9 import torch
10 from tensorizer import EncryptionParams
11
@@ -22,7 +21,7 @@ from vllm.model_executor.model_loader.tensorizer import (TensorizerConfig,
21 tensorize_vllm_model)
22
23 from ..conftest import VllmRunner, cleanup
25 -from ..utils import VLLM_PATH, RemoteOpenAIServer
24 +from ..utils import RemoteOpenAIServer
25
26 # yapf conflicts with isort for this docstring
27
@@ -220,23 +219,21 @@ def test_openai_apiserver_with_tensorizer(vllm_runner, tmp_path):
219 json.dumps(model_loader_extra_config),
220 ]
221
223 - ray.init(runtime_env={"working_dir": VLLM_PATH})
222 + with RemoteOpenAIServer(openai_args) as server:
223 + print("Server ready.")
224
225 - server = RemoteOpenAIServer(openai_args)
226 - print("Server ready.")
225 + client = server.get_client()
226 + completion = client.completions.create(model=model_ref,
227 + prompt="Hello, my name is",
228 + max_tokens=5,
229 + temperature=0.0)
230
228 - client = server.get_client()
229 - completion = client.completions.create(model=model_ref,
230 - prompt="Hello, my name is",
231 - max_tokens=5,
232 - temperature=0.0)
233 -
234 - assert completion.id is not None
235 - assert len(completion.choices) == 1
236 - assert len(completion.choices[0].text) >= 5
237 - assert completion.choices[0].finish_reason == "length"
238 - assert completion.usage == openai.types.CompletionUsage(
239 - completion_tokens=5, prompt_tokens=6, total_tokens=11)
231 + assert completion.id is not None
232 + assert len(completion.choices) == 1
233 + assert len(completion.choices[0].text) >= 5
234 + assert completion.choices[0].finish_reason == "length"
235 + assert completion.usage == openai.types.CompletionUsage(
236 + completion_tokens=5, prompt_tokens=6, total_tokens=11)
237
238
239 def test_raise_value_error_on_invalid_load_format(vllm_runner):
@@ -282,7 +279,6 @@ def test_deserialized_encrypted_vllm_model_with_tp_has_same_outputs(vllm_runner,
279 base_model.model.llm_engine.model_executor.shutdown()
280 del base_model
281 cleanup()
285 - ray.shutdown()
282
283 # load model with two shards and serialize with encryption
284 model_path = str(tmp_path / (model_ref + "-%02d.tensors"))
@@ -305,7 +301,6 @@ def test_deserialized_encrypted_vllm_model_with_tp_has_same_outputs(vllm_runner,
301 assert os.path.isfile(model_path % 0), "Serialization subprocess failed"
302 assert os.path.isfile(model_path % 1), "Serialization subprocess failed"
303 cleanup()
308 - ray.shutdown()
304
305 loaded_vllm_model = vllm_runner(
306 model_ref,
tests/utils.py
+36 -57
@@ -49,53 +49,7 @@ class RemoteOpenAIServer:
49 DUMMY_API_KEY = "token-abc123" # vLLM's OpenAI server does not need API key
50 MAX_SERVER_START_WAIT_S = 600 # wait for server to start for 60 seconds
51
52 - class _RemoteRunner:
53 -
54 - def __init__(self, cli_args: List[str], *, wait_url: str,
55 - wait_timeout: float) -> None:
56 - env = os.environ.copy()
57 - env["PYTHONUNBUFFERED"] = "1"
58 - self.proc = subprocess.Popen(
59 - [
60 - sys.executable, "-m", "vllm.entrypoints.openai.api_server",
61 - *cli_args
62 - ],
63 - env=env,
64 - stdout=sys.stdout,
65 - stderr=sys.stderr,
66 - )
67 -
68 - self._wait_for_server(url=wait_url, timeout=wait_timeout)
69 -
70 - def ready(self):
71 - return True
72 -
73 - def _wait_for_server(self, *, url: str, timeout: float):
74 - # run health check
75 - start = time.time()
76 - while True:
77 - try:
78 - if requests.get(url).status_code == 200:
79 - break
80 - except Exception as err:
81 - if self.proc.poll() is not None:
82 - raise RuntimeError(
83 - "Server exited unexpectedly.") from err
84 -
85 - time.sleep(0.5)
86 - if time.time() - start > timeout:
87 - raise RuntimeError(
88 - "Server failed to start in time.") from err
89 -
90 - def __del__(self):
91 - if hasattr(self, "proc"):
92 - self.proc.terminate()
93 -
94 - def __init__(self,
95 - cli_args: List[str],
96 - *,
97 - auto_port: bool = True,
98 - num_gpus: int = 1) -> None:
52 + def __init__(self, cli_args: List[str], *, auto_port: bool = True) -> None:
53 if auto_port:
54 if "-p" in cli_args or "--port" in cli_args:
55 raise ValueError("You have manually specified the port"
@@ -108,13 +62,41 @@ class RemoteOpenAIServer:
62 self.host = str(args.host or 'localhost')
63 self.port = int(args.port)
64
111 - self._runner = ray.remote(num_gpus=num_gpus)(
112 - self._RemoteRunner).remote(
113 - cli_args,
114 - wait_url=self.url_for("health"),
115 - wait_timeout=self.MAX_SERVER_START_WAIT_S)
116 -
117 - self._wait_until_ready()
65 + env = os.environ.copy()
66 + # the current process might initialize cuda,
67 + # to be safe, we should use spawn method
68 + env['VLLM_WORKER_MULTIPROC_METHOD'] = 'spawn'
69 + self.proc = subprocess.Popen(
70 + [sys.executable, "-m", "vllm.entrypoints.openai.api_server"] +
71 + cli_args,
72 + env=env,
73 + stdout=sys.stdout,
74 + stderr=sys.stderr)
75 + self._wait_for_server(url=self.url_for("health"),
76 + timeout=self.MAX_SERVER_START_WAIT_S)
77 +
78 + def __enter__(self):
79 + return self
80 +
81 + def __exit__(self, exc_type, exc_value, traceback):
82 + self.proc.terminate()
83 +
84 + def _wait_for_server(self, *, url: str, timeout: float):
85 + # run health check
86 + start = time.time()
87 + while True:
88 + try:
89 + if requests.get(url).status_code == 200:
90 + break
91 + except Exception as err:
92 + result = self.proc.poll()
93 + if result is not None and result != 0:
94 + raise RuntimeError("Server exited unexpectedly.") from err
95 +
96 + time.sleep(0.5)
97 + if time.time() - start > timeout:
98 + raise RuntimeError(
99 + "Server failed to start in time.") from err
100
101 @property
102 def url_root(self) -> str:
@@ -123,9 +105,6 @@ class RemoteOpenAIServer:
105 def url_for(self, *parts: str) -> str:
106 return self.url_root + "/" + "/".join(parts)
107
126 - def _wait_until_ready(self) -> None:
127 - ray.get(self._runner.ready.remote())
128 -
108 def get_client(self):
109 return openai.OpenAI(
110 base_url=self.url_for("v1"),
vllm/executor/ray_gpu_executor.py
+7 -10
@@ -224,16 +224,13 @@ class RayGPUExecutor(DistributedGPUExecutor):
224 # broadcasted to.
225 self.non_driver_workers: List[RayWorkerWrapper] = []
226
227 - for pp_rank in range(self.parallel_config.pipeline_parallel_size):
228 - for tp_rank in range(self.parallel_config.tensor_parallel_size):
229 - rank = (pp_rank *
230 - self.parallel_config.tensor_parallel_size) + tp_rank
231 - if rank == 0:
232 - pass
233 - elif rank % self.parallel_config.tensor_parallel_size == 0:
234 - self.tp_driver_workers.append(self.workers[rank - 1])
235 - else:
236 - self.non_driver_workers.append(self.workers[rank - 1])
227 + for idx, rank in enumerate(worker_ranks[1:]):
228 + # We need to skip the driver worker, which we
229 + # do by skipping worker_ranks[0] which is always 0.
230 + if rank % self.parallel_config.tensor_parallel_size == 0:
231 + self.tp_driver_workers.append(self.workers[idx])
232 + else:
233 + self.non_driver_workers.append(self.workers[idx])
234
235 def _driver_execute_model(
236 self, execute_model_req: Optional[ExecuteModelRequest]