A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 22:00 UTC
[ci] try to add multi-node tests (#6280)
Signed-off-by: Muralidhar Andoorveedu <muralidhar.andoorveedu@centml.ai> Co-authored-by: Muralidhar Andoorveedu <muralidhar.andoorveedu@centml.ai>
youkaichao committed
Jul 12, 2024 at 21:51 UTC
41708e50341c82668fd25ebc7777470cba6f5303
13 files changed
+230
-275
.buildkite/run-multi-node-test.sh
+40
-12
@@ -2,16 +2,17 @@
2
3
set -euox pipefail
4
5
-if [[ $# -lt 3 ]]; then
6
- echo "Please provide the number of nodes and GPU per node."
5
+if [[ $# -lt 4 ]]; then
6
+ echo "Usage: .buildkite/run-multi-node-test.sh WORKING_DIR NUM_NODES NUM_GPUS DOCKER_IMAGE COMMAND1 COMMAND2 ... COMMANDN"
7
exit 1
8
fi
9
10
-NUM_NODES=$1
11
-NUM_GPUS=$2
12
-DOCKER_IMAGE=$3
10
+WORKING_DIR=$1
11
+NUM_NODES=$2
12
+NUM_GPUS=$3
13
+DOCKER_IMAGE=$4
14
14
-shift 3
15
+shift 4
16
COMMANDS=("$@")
17
if [ ${#COMMANDS[@]} -ne $NUM_NODES ]; then
18
echo "The number of commands must be equal to the number of nodes."
@@ -40,13 +41,40 @@ start_nodes() {
41
fi
42
done
43
GPU_DEVICES+='"'
43
- # echo "Starting node$node with GPU devices: $GPU_DEVICES"
44
- docker run -d --gpus "$GPU_DEVICES" --name node$node --network docker-net --ip 192.168.10.$((10 + $node)) --rm $DOCKER_IMAGE tail -f /dev/null
44
+
45
+ # start the container in detached mode
46
+ # things to note:
47
+ # 1. --shm-size=10.24gb is required. don't use --ipc=host
48
+ # 2. pass HF_TOKEN to the container
49
+ # 3. map the huggingface cache directory to the container
50
+ # 3. assign ip addresses to the containers (head node: 192.168.10.10, worker nodes:
51
+ # starting from 192.168.10.11)
52
+ docker run -d --gpus "$GPU_DEVICES" --shm-size=10.24gb -e HF_TOKEN -v ~/.cache/huggingface:/root/.cache/huggingface --name node$node --network docker-net --ip 192.168.10.$((10 + $node)) --rm $DOCKER_IMAGE /bin/bash -c "tail -f /dev/null"
53
+
54
+ # organize containers into a ray cluster
55
+ if [ $node -eq 0 ]; then
56
+ # start the ray head node
57
+ docker exec -d node$node /bin/bash -c "ray start --head --port=6379 --block"
58
+ # wait for the head node to be ready
59
+ sleep 10
60
+ else
61
+ # start the ray worker nodes, and connect them to the head node
62
+ docker exec -d node$node /bin/bash -c "ray start --address=192.168.10.10:6379 --block"
63
+ fi
64
done
65
+
66
+ # wait for the cluster to be ready
67
+ sleep 10
68
+
69
+ # print the cluster status
70
+ docker exec node0 /bin/bash -c "ray status"
71
}
72
73
run_nodes() {
49
- for node in $(seq 0 $(($NUM_NODES-1))); do
74
+ # important: iterate in reverse order to start the head node last
75
+ # we start the worker nodes first, in detached mode, and then start the head node
76
+ # in the foreground, so that the output of the head node is visible in the buildkite logs
77
+ for node in $(seq $(($NUM_NODES - 1)) -1 0); do
78
GPU_DEVICES='"device='
79
for node_gpu in $(seq 0 $(($NUM_GPUS - 1))); do
80
DEVICE_NUM=$(($node * $NUM_GPUS + $node_gpu))
@@ -57,10 +85,10 @@ run_nodes() {
85
done
86
GPU_DEVICES+='"'
87
echo "Running node$node with GPU devices: $GPU_DEVICES"
60
- if [ $node -lt $(($NUM_NODES - 1)) ]; then
61
- docker exec -d node$node /bin/bash -c "${COMMANDS[$node]}"
88
+ if [ $node -ne 0 ]; then
89
+ docker exec -d node$node /bin/bash -c "cd $WORKING_DIR ; ${COMMANDS[$node]}"
90
else
63
- docker exec node$node /bin/bash -c "${COMMANDS[$node]}"
91
+ docker exec node$node /bin/bash -c "cd $WORKING_DIR ; ${COMMANDS[$node]}"
92
fi
93
done
94
}
.buildkite/test-pipeline.yaml
+15
-1
@@ -68,6 +68,17 @@ steps:
68
- pytest -v -s distributed/test_comm_ops.py
69
- pytest -v -s distributed/test_shm_broadcast.py
70
71
+- label: 2 Node Tests (4 GPUs in total)
72
+ working_dir: "/vllm-workspace/tests"
73
+ num_gpus: 2
74
+ num_nodes: 2
75
+ commands:
76
+ - # the following commands are for the first node, with ip 192.168.10.10 (ray environment already set up)
77
+ - VLLM_TEST_SAME_HOST=0 torchrun --nnodes 2 --nproc-per-node=2 --rdzv_backend=c10d --rdzv_endpoint=192.168.10.10 distributed/test_same_node.py
78
+ - TP_SIZE=2 PP_SIZE=2 EAGER_MODE=1 CHUNKED_PREFILL=0 pytest -v -s distributed/test_pipeline_parallel.py
79
+ - # the following commands are for the second node, with ip 192.168.10.11 (ray environment already set up)
80
+ - VLLM_TEST_SAME_HOST=0 torchrun --nnodes 2 --nproc-per-node=2 --rdzv_backend=c10d --rdzv_endpoint=192.168.10.10 distributed/test_same_node.py
81
+
82
- label: Distributed Tests (2 GPUs)
83
mirror_hardwares: [amd]
84
working_dir: "/vllm-workspace/tests"
@@ -213,7 +224,10 @@ steps:
224
225
- label: Tensorizer Test
226
#mirror_hardwares: [amd]
216
- command: apt-get install curl libsodium23 && pytest -v -s tensorizer_loader
227
+ commands:
228
+ - apt-get install curl libsodium23
229
+ - export VLLM_WORKER_MULTIPROC_METHOD=spawn
230
+ - pytest -v -s tensorizer_loader
231
232
- label: Metrics Test
233
mirror_hardwares: [amd]
tests/async_engine/test_openapi_server_ray.py
+14
-23
@@ -1,35 +1,26 @@
1
import openai # use the official client for correctness check
2
import pytest
3
-# using Ray for overall ease of process management, parallel requests,
4
-# and debugging.
5
-import ray
3
7
-from ..utils import VLLM_PATH, RemoteOpenAIServer
4
+from ..utils import RemoteOpenAIServer
5
6
# any model with a chat template should work here
7
MODEL_NAME = "facebook/opt-125m"
8
9
10
@pytest.fixture(scope="module")
14
-def ray_ctx():
15
- ray.init(runtime_env={"working_dir": VLLM_PATH})
16
- yield
17
- ray.shutdown()
18
-
19
-
20
-@pytest.fixture(scope="module")
21
-def server(ray_ctx):
22
- return RemoteOpenAIServer([
23
- "--model",
24
- MODEL_NAME,
25
- # use half precision for speed and memory savings in CI environment
26
- "--dtype",
27
- "float16",
28
- "--max-model-len",
29
- "2048",
30
- "--enforce-eager",
31
- "--engine-use-ray"
32
- ])
11
+def server():
12
+ with RemoteOpenAIServer([
13
+ "--model",
14
+ MODEL_NAME,
15
+ # use half precision for speed and memory savings in CI environment
16
+ "--dtype",
17
+ "float16",
18
+ "--max-model-len",
19
+ "2048",
20
+ "--enforce-eager",
21
+ "--engine-use-ray"
22
+ ]) as remote_server:
23
+ yield remote_server
24
25
26
@pytest.fixture(scope="module")
tests/distributed/test_pipeline_parallel.py
+4
-13
@@ -2,11 +2,8 @@ import os
2
3
import openai # use the official client for correctness check
4
import pytest
5
-# using Ray for overall ease of process management, parallel requests,
6
-# and debugging.
7
-import ray
5
9
-from ..utils import VLLM_PATH, RemoteOpenAIServer
6
+from ..utils import RemoteOpenAIServer
7
8
# downloading lora to test lora requests
9
@@ -21,14 +18,7 @@ pytestmark = pytest.mark.asyncio
18
19
20
@pytest.fixture(scope="module")
24
-def ray_ctx():
25
- ray.init(runtime_env={"working_dir": VLLM_PATH})
26
- yield
27
- ray.shutdown()
28
-
29
-
30
-@pytest.fixture(scope="module")
31
-def server(ray_ctx):
21
+def server():
22
args = [
23
"--model",
24
MODEL_NAME,
@@ -50,7 +40,8 @@ def server(ray_ctx):
40
args += [
41
"--enforce-eager",
42
]
53
- return RemoteOpenAIServer(args, num_gpus=PP_SIZE * TP_SIZE)
43
+ with RemoteOpenAIServer(args) as remote_server:
44
+ yield remote_server
45
46
47
@pytest.fixture(scope="module")
tests/distributed/test_same_node.py
+1
@@ -10,3 +10,4 @@ test_result = all(
10
11
expected = os.environ.get("VLLM_TEST_SAME_HOST", "1") == "1"
12
assert test_result == expected, f"Expected {expected}, got {test_result}"
13
+print("Same node test passed!")
tests/entrypoints/openai/test_chat.py
+24
-33
@@ -6,15 +6,12 @@ from typing import List
6
import jsonschema
7
import openai # use the official client for correctness check
8
import pytest
9
-# using Ray for overall ease of process management, parallel requests,
10
-# and debugging.
11
-import ray
9
import torch
10
# downloading lora to test lora requests
11
from huggingface_hub import snapshot_download
12
from openai import BadRequestError
13
17
-from ...utils import VLLM_PATH, RemoteOpenAIServer
14
+from ...utils import RemoteOpenAIServer
15
16
# any model with a chat template should work here
17
MODEL_NAME = "HuggingFaceH4/zephyr-7b-beta"
@@ -29,35 +26,29 @@ def zephyr_lora_files():
26
27
28
@pytest.fixture(scope="module")
32
-def ray_ctx():
33
- ray.init(runtime_env={"working_dir": VLLM_PATH})
34
- yield
35
- ray.shutdown()
36
-
37
-
38
-@pytest.fixture(scope="module")
39
-def server(zephyr_lora_files, ray_ctx):
40
- return RemoteOpenAIServer([
41
- "--model",
42
- MODEL_NAME,
43
- # use half precision for speed and memory savings in CI environment
44
- "--dtype",
45
- "bfloat16",
46
- "--max-model-len",
47
- "8192",
48
- "--enforce-eager",
49
- # lora config below
50
- "--enable-lora",
51
- "--lora-modules",
52
- f"zephyr-lora={zephyr_lora_files}",
53
- f"zephyr-lora2={zephyr_lora_files}",
54
- "--max-lora-rank",
55
- "64",
56
- "--max-cpu-loras",
57
- "2",
58
- "--max-num-seqs",
59
- "128",
60
- ])
29
+def server(zephyr_lora_files):
30
+ with RemoteOpenAIServer([
31
+ "--model",
32
+ MODEL_NAME,
33
+ # use half precision for speed and memory savings in CI environment
34
+ "--dtype",
35
+ "bfloat16",
36
+ "--max-model-len",
37
+ "8192",
38
+ "--enforce-eager",
39
+ # lora config below
40
+ "--enable-lora",
41
+ "--lora-modules",
42
+ f"zephyr-lora={zephyr_lora_files}",
43
+ f"zephyr-lora2={zephyr_lora_files}",
44
+ "--max-lora-rank",
45
+ "64",
46
+ "--max-cpu-loras",
47
+ "2",
48
+ "--max-num-seqs",
49
+ "128",
50
+ ]) as remote_server:
51
+ yield remote_server
52
53
54
@pytest.fixture(scope="module")
tests/entrypoints/openai/test_completion.py
+24
-33
@@ -6,9 +6,6 @@ from typing import List
6
import jsonschema
7
import openai # use the official client for correctness check
8
import pytest
9
-# using Ray for overall ease of process management, parallel requests,
10
-# and debugging.
11
-import ray
9
import requests
10
# downloading lora to test lora requests
11
from huggingface_hub import snapshot_download
@@ -16,7 +13,7 @@ from openai import BadRequestError
13
14
from vllm.transformers_utils.tokenizer import get_tokenizer
15
19
-from ...utils import VLLM_PATH, RemoteOpenAIServer
16
+from ...utils import RemoteOpenAIServer
17
18
# any model with a chat template should work here
19
MODEL_NAME = "HuggingFaceH4/zephyr-7b-beta"
@@ -31,35 +28,29 @@ def zephyr_lora_files():
28
29
30
@pytest.fixture(scope="module")
34
-def ray_ctx():
35
- ray.init(runtime_env={"working_dir": VLLM_PATH})
36
- yield
37
- ray.shutdown()
38
-
39
-
40
-@pytest.fixture(scope="module")
41
-def server(zephyr_lora_files, ray_ctx):
42
- return RemoteOpenAIServer([
43
- "--model",
44
- MODEL_NAME,
45
- # use half precision for speed and memory savings in CI environment
46
- "--dtype",
47
- "bfloat16",
48
- "--max-model-len",
49
- "8192",
50
- "--enforce-eager",
51
- # lora config below
52
- "--enable-lora",
53
- "--lora-modules",
54
- f"zephyr-lora={zephyr_lora_files}",
55
- f"zephyr-lora2={zephyr_lora_files}",
56
- "--max-lora-rank",
57
- "64",
58
- "--max-cpu-loras",
59
- "2",
60
- "--max-num-seqs",
61
- "128",
62
- ])
31
+def server(zephyr_lora_files):
32
+ with RemoteOpenAIServer([
33
+ "--model",
34
+ MODEL_NAME,
35
+ # use half precision for speed and memory savings in CI environment
36
+ "--dtype",
37
+ "bfloat16",
38
+ "--max-model-len",
39
+ "8192",
40
+ "--enforce-eager",
41
+ # lora config below
42
+ "--enable-lora",
43
+ "--lora-modules",
44
+ f"zephyr-lora={zephyr_lora_files}",
45
+ f"zephyr-lora2={zephyr_lora_files}",
46
+ "--max-lora-rank",
47
+ "64",
48
+ "--max-cpu-loras",
49
+ "2",
50
+ "--max-num-seqs",
51
+ "128",
52
+ ]) as remote_server:
53
+ yield remote_server
54
55
56
@pytest.fixture(scope="module")
tests/entrypoints/openai/test_embedding.py
+14
-21
@@ -3,33 +3,26 @@ import base64
3
import numpy as np
4
import openai
5
import pytest
6
-import ray
6
8
-from ...utils import VLLM_PATH, RemoteOpenAIServer
7
+from ...utils import RemoteOpenAIServer
8
9
EMBEDDING_MODEL_NAME = "intfloat/e5-mistral-7b-instruct"
10
11
12
@pytest.fixture(scope="module")
14
-def ray_ctx():
15
- ray.init(runtime_env={"working_dir": VLLM_PATH})
16
- yield
17
- ray.shutdown()
18
-
19
-
20
-@pytest.fixture(scope="module")
21
-def embedding_server(ray_ctx):
22
- return RemoteOpenAIServer([
23
- "--model",
24
- EMBEDDING_MODEL_NAME,
25
- # use half precision for speed and memory savings in CI environment
26
- "--dtype",
27
- "bfloat16",
28
- "--enforce-eager",
29
- "--max-model-len",
30
- "8192",
31
- "--enforce-eager",
32
- ])
13
+def embedding_server():
14
+ with RemoteOpenAIServer([
15
+ "--model",
16
+ EMBEDDING_MODEL_NAME,
17
+ # use half precision for speed and memory savings in CI environment
18
+ "--dtype",
19
+ "bfloat16",
20
+ "--enforce-eager",
21
+ "--max-model-len",
22
+ "8192",
23
+ "--enforce-eager",
24
+ ]) as remote_server:
25
+ yield remote_server
26
27
28
@pytest.mark.asyncio
tests/entrypoints/openai/test_models.py
+24
-33
@@ -1,12 +1,9 @@
1
import openai # use the official client for correctness check
2
import pytest
3
-# using Ray for overall ease of process management, parallel requests,
4
-# and debugging.
5
-import ray
3
# downloading lora to test lora requests
4
from huggingface_hub import snapshot_download
5
9
-from ...utils import VLLM_PATH, RemoteOpenAIServer
6
+from ...utils import RemoteOpenAIServer
7
8
# any model with a chat template should work here
9
MODEL_NAME = "HuggingFaceH4/zephyr-7b-beta"
@@ -21,35 +18,29 @@ def zephyr_lora_files():
18
19
20
@pytest.fixture(scope="module")
24
-def ray_ctx():
25
- ray.init(runtime_env={"working_dir": VLLM_PATH})
26
- yield
27
- ray.shutdown()
28
-
29
-
30
-@pytest.fixture(scope="module")
31
-def server(zephyr_lora_files, ray_ctx):
32
- return RemoteOpenAIServer([
33
- "--model",
34
- MODEL_NAME,
35
- # use half precision for speed and memory savings in CI environment
36
- "--dtype",
37
- "bfloat16",
38
- "--max-model-len",
39
- "8192",
40
- "--enforce-eager",
41
- # lora config below
42
- "--enable-lora",
43
- "--lora-modules",
44
- f"zephyr-lora={zephyr_lora_files}",
45
- f"zephyr-lora2={zephyr_lora_files}",
46
- "--max-lora-rank",
47
- "64",
48
- "--max-cpu-loras",
49
- "2",
50
- "--max-num-seqs",
51
- "128",
52
- ])
21
+def server(zephyr_lora_files):
22
+ with RemoteOpenAIServer([
23
+ "--model",
24
+ MODEL_NAME,
25
+ # use half precision for speed and memory savings in CI environment
26
+ "--dtype",
27
+ "bfloat16",
28
+ "--max-model-len",
29
+ "8192",
30
+ "--enforce-eager",
31
+ # lora config below
32
+ "--enable-lora",
33
+ "--lora-modules",
34
+ f"zephyr-lora={zephyr_lora_files}",
35
+ f"zephyr-lora2={zephyr_lora_files}",
36
+ "--max-lora-rank",
37
+ "64",
38
+ "--max-cpu-loras",
39
+ "2",
40
+ "--max-num-seqs",
41
+ "128",
42
+ ]) as remote_server:
43
+ yield remote_server
44
45
46
@pytest.fixture(scope="module")
tests/entrypoints/openai/test_vision.py
+13
-20
@@ -3,7 +3,6 @@ from typing import Dict, List
3
import openai
4
import pytest
5
import pytest_asyncio
6
-import ray
6
7
from vllm.multimodal.utils import ImageFetchAiohttp, encode_image_base64
8
@@ -23,25 +22,19 @@ TEST_IMAGE_URLS = [
22
23
24
@pytest.fixture(scope="module")
26
-def ray_ctx():
27
- ray.init(runtime_env={"working_dir": VLLM_PATH})
28
- yield
29
- ray.shutdown()
30
-
31
-
32
-@pytest.fixture(scope="module")
33
-def server(ray_ctx):
34
- return RemoteOpenAIServer([
35
- "--model",
36
- MODEL_NAME,
37
- "--dtype",
38
- "bfloat16",
39
- "--max-model-len",
40
- "4096",
41
- "--enforce-eager",
42
- "--chat-template",
43
- str(LLAVA_CHAT_TEMPLATE),
44
- ])
25
+def server():
26
+ with RemoteOpenAIServer([
27
+ "--model",
28
+ MODEL_NAME,
29
+ "--dtype",
30
+ "bfloat16",
31
+ "--max-model-len",
32
+ "4096",
33
+ "--enforce-eager",
34
+ "--chat-template",
35
+ str(LLAVA_CHAT_TEMPLATE),
36
+ ]) as remote_server:
37
+ yield remote_server
38
39
40
@pytest.fixture(scope="module")
tests/tensorizer_loader/test_tensorizer.py
+14
-19
@@ -6,7 +6,6 @@ from unittest.mock import MagicMock, patch
6
7
import openai
8
import pytest
9
-import ray
9
import torch
10
from tensorizer import EncryptionParams
11
@@ -22,7 +21,7 @@ from vllm.model_executor.model_loader.tensorizer import (TensorizerConfig,
21
tensorize_vllm_model)
22
23
from ..conftest import VllmRunner, cleanup
25
-from ..utils import VLLM_PATH, RemoteOpenAIServer
24
+from ..utils import RemoteOpenAIServer
25
26
# yapf conflicts with isort for this docstring
27
@@ -220,23 +219,21 @@ def test_openai_apiserver_with_tensorizer(vllm_runner, tmp_path):
219
json.dumps(model_loader_extra_config),
220
]
221
223
- ray.init(runtime_env={"working_dir": VLLM_PATH})
222
+ with RemoteOpenAIServer(openai_args) as server:
223
+ print("Server ready.")
224
225
- server = RemoteOpenAIServer(openai_args)
226
- print("Server ready.")
225
+ client = server.get_client()
226
+ completion = client.completions.create(model=model_ref,
227
+ prompt="Hello, my name is",
228
+ max_tokens=5,
229
+ temperature=0.0)
230
228
- client = server.get_client()
229
- completion = client.completions.create(model=model_ref,
230
- prompt="Hello, my name is",
231
- max_tokens=5,
232
- temperature=0.0)
233
-
234
- assert completion.id is not None
235
- assert len(completion.choices) == 1
236
- assert len(completion.choices[0].text) >= 5
237
- assert completion.choices[0].finish_reason == "length"
238
- assert completion.usage == openai.types.CompletionUsage(
239
- completion_tokens=5, prompt_tokens=6, total_tokens=11)
231
+ assert completion.id is not None
232
+ assert len(completion.choices) == 1
233
+ assert len(completion.choices[0].text) >= 5
234
+ assert completion.choices[0].finish_reason == "length"
235
+ assert completion.usage == openai.types.CompletionUsage(
236
+ completion_tokens=5, prompt_tokens=6, total_tokens=11)
237
238
239
def test_raise_value_error_on_invalid_load_format(vllm_runner):
@@ -282,7 +279,6 @@ def test_deserialized_encrypted_vllm_model_with_tp_has_same_outputs(vllm_runner,
279
base_model.model.llm_engine.model_executor.shutdown()
280
del base_model
281
cleanup()
285
- ray.shutdown()
282
283
# load model with two shards and serialize with encryption
284
model_path = str(tmp_path / (model_ref + "-%02d.tensors"))
@@ -305,7 +301,6 @@ def test_deserialized_encrypted_vllm_model_with_tp_has_same_outputs(vllm_runner,
301
assert os.path.isfile(model_path % 0), "Serialization subprocess failed"
302
assert os.path.isfile(model_path % 1), "Serialization subprocess failed"
303
cleanup()
308
- ray.shutdown()
304
305
loaded_vllm_model = vllm_runner(
306
model_ref,
tests/utils.py
+36
-57
@@ -49,53 +49,7 @@ class RemoteOpenAIServer:
49
DUMMY_API_KEY = "token-abc123" # vLLM's OpenAI server does not need API key
50
MAX_SERVER_START_WAIT_S = 600 # wait for server to start for 60 seconds
51
52
- class _RemoteRunner:
53
-
54
- def __init__(self, cli_args: List[str], *, wait_url: str,
55
- wait_timeout: float) -> None:
56
- env = os.environ.copy()
57
- env["PYTHONUNBUFFERED"] = "1"
58
- self.proc = subprocess.Popen(
59
- [
60
- sys.executable, "-m", "vllm.entrypoints.openai.api_server",
61
- *cli_args
62
- ],
63
- env=env,
64
- stdout=sys.stdout,
65
- stderr=sys.stderr,
66
- )
67
-
68
- self._wait_for_server(url=wait_url, timeout=wait_timeout)
69
-
70
- def ready(self):
71
- return True
72
-
73
- def _wait_for_server(self, *, url: str, timeout: float):
74
- # run health check
75
- start = time.time()
76
- while True:
77
- try:
78
- if requests.get(url).status_code == 200:
79
- break
80
- except Exception as err:
81
- if self.proc.poll() is not None:
82
- raise RuntimeError(
83
- "Server exited unexpectedly.") from err
84
-
85
- time.sleep(0.5)
86
- if time.time() - start > timeout:
87
- raise RuntimeError(
88
- "Server failed to start in time.") from err
89
-
90
- def __del__(self):
91
- if hasattr(self, "proc"):
92
- self.proc.terminate()
93
-
94
- def __init__(self,
95
- cli_args: List[str],
96
- *,
97
- auto_port: bool = True,
98
- num_gpus: int = 1) -> None:
52
+ def __init__(self, cli_args: List[str], *, auto_port: bool = True) -> None:
53
if auto_port:
54
if "-p" in cli_args or "--port" in cli_args:
55
raise ValueError("You have manually specified the port"
@@ -108,13 +62,41 @@ class RemoteOpenAIServer:
62
self.host = str(args.host or 'localhost')
63
self.port = int(args.port)
64
111
- self._runner = ray.remote(num_gpus=num_gpus)(
112
- self._RemoteRunner).remote(
113
- cli_args,
114
- wait_url=self.url_for("health"),
115
- wait_timeout=self.MAX_SERVER_START_WAIT_S)
116
-
117
- self._wait_until_ready()
65
+ env = os.environ.copy()
66
+ # the current process might initialize cuda,
67
+ # to be safe, we should use spawn method
68
+ env['VLLM_WORKER_MULTIPROC_METHOD'] = 'spawn'
69
+ self.proc = subprocess.Popen(
70
+ [sys.executable, "-m", "vllm.entrypoints.openai.api_server"] +
71
+ cli_args,
72
+ env=env,
73
+ stdout=sys.stdout,
74
+ stderr=sys.stderr)
75
+ self._wait_for_server(url=self.url_for("health"),
76
+ timeout=self.MAX_SERVER_START_WAIT_S)
77
+
78
+ def __enter__(self):
79
+ return self
80
+
81
+ def __exit__(self, exc_type, exc_value, traceback):
82
+ self.proc.terminate()
83
+
84
+ def _wait_for_server(self, *, url: str, timeout: float):
85
+ # run health check
86
+ start = time.time()
87
+ while True:
88
+ try:
89
+ if requests.get(url).status_code == 200:
90
+ break
91
+ except Exception as err:
92
+ result = self.proc.poll()
93
+ if result is not None and result != 0:
94
+ raise RuntimeError("Server exited unexpectedly.") from err
95
+
96
+ time.sleep(0.5)
97
+ if time.time() - start > timeout:
98
+ raise RuntimeError(
99
+ "Server failed to start in time.") from err
100
101
@property
102
def url_root(self) -> str:
@@ -123,9 +105,6 @@ class RemoteOpenAIServer:
105
def url_for(self, *parts: str) -> str:
106
return self.url_root + "/" + "/".join(parts)
107
126
- def _wait_until_ready(self) -> None:
127
- ray.get(self._runner.ready.remote())
128
-
108
def get_client(self):
109
return openai.OpenAI(
110
base_url=self.url_for("v1"),
vllm/executor/ray_gpu_executor.py
+7
-10
@@ -224,16 +224,13 @@ class RayGPUExecutor(DistributedGPUExecutor):
224
# broadcasted to.
225
self.non_driver_workers: List[RayWorkerWrapper] = []
226
227
- for pp_rank in range(self.parallel_config.pipeline_parallel_size):
228
- for tp_rank in range(self.parallel_config.tensor_parallel_size):
229
- rank = (pp_rank *
230
- self.parallel_config.tensor_parallel_size) + tp_rank
231
- if rank == 0:
232
- pass
233
- elif rank % self.parallel_config.tensor_parallel_size == 0:
234
- self.tp_driver_workers.append(self.workers[rank - 1])
235
- else:
236
- self.non_driver_workers.append(self.workers[rank - 1])
227
+ for idx, rank in enumerate(worker_ranks[1:]):
228
+ # We need to skip the driver worker, which we
229
+ # do by skipping worker_ranks[0] which is always 0.
230
+ if rank % self.parallel_config.tensor_parallel_size == 0:
231
+ self.tp_driver_workers.append(self.workers[idx])
232
+ else:
233
+ self.non_driver_workers.append(self.workers[idx])
234
235
def _driver_execute_model(
236
self, execute_model_req: Optional[ExecuteModelRequest]