A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 20:45 UTC
[ci][distributed] add pipeline parallel correctness test (#6410)
youkaichao committed
Jul 16, 2024 at 15:44 UTC
09c2eb85ddd3b2585979f4cd9cc97168d86718b6
3 files changed
+129
-128
.buildkite/test-pipeline.yaml
+2
-7
@@ -72,7 +72,7 @@ steps:
72
commands:
73
- # the following commands are for the first node, with ip 192.168.10.10 (ray environment already set up)
74
- VLLM_TEST_SAME_HOST=0 torchrun --nnodes 2 --nproc-per-node=2 --rdzv_backend=c10d --rdzv_endpoint=192.168.10.10 distributed/test_same_node.py
75
- - TP_SIZE=2 PP_SIZE=2 EAGER_MODE=1 CHUNKED_PREFILL=0 pytest -v -s distributed/test_pipeline_parallel.py
75
+ - pytest -v -s distributed/test_pipeline_parallel.py
76
- # the following commands are for the second node, with ip 192.168.10.11 (ray environment already set up)
77
- VLLM_TEST_SAME_HOST=0 torchrun --nnodes 2 --nproc-per-node=2 --rdzv_backend=c10d --rdzv_endpoint=192.168.10.10 distributed/test_same_node.py
78
@@ -115,12 +115,7 @@ steps:
115
working_dir: "/vllm-workspace/tests"
116
num_gpus: 4
117
commands:
118
- - TP_SIZE=2 PP_SIZE=2 EAGER_MODE=1 CHUNKED_PREFILL=1 pytest -v -s distributed/test_pipeline_parallel.py
119
- - TP_SIZE=2 PP_SIZE=2 EAGER_MODE=1 CHUNKED_PREFILL=0 pytest -v -s distributed/test_pipeline_parallel.py
120
- - TP_SIZE=1 PP_SIZE=3 EAGER_MODE=1 CHUNKED_PREFILL=0 pytest -v -s distributed/test_pipeline_parallel.py
121
- - PP_SIZE=4 EAGER_MODE=1 CHUNKED_PREFILL=1 pytest -v -s distributed/test_pipeline_parallel.py
122
- - PP_SIZE=4 EAGER_MODE=1 CHUNKED_PREFILL=0 pytest -v -s distributed/test_pipeline_parallel.py
123
-
118
+ - pytest -v -s distributed/test_pipeline_parallel.py
119
120
- label: Engine Test
121
mirror_hardwares: [amd]
tests/distributed/test_pipeline_parallel.py
+112
-121
@@ -1,25 +1,18 @@
1
-import os
2
-
3
-import openai # use the official client for correctness check
1
import pytest
2
3
from ..utils import RemoteOpenAIServer
4
8
-# downloading lora to test lora requests
9
-
10
-# any model with a chat template should work here
11
-MODEL_NAME = "meta-llama/Meta-Llama-3-8B"
12
-EAGER_MODE = bool(int(os.getenv("EAGER_MODE", 0)))
13
-CHUNKED_PREFILL = bool(int(os.getenv("CHUNKED_PREFILL", 0)))
14
-TP_SIZE = int(os.getenv("TP_SIZE", 1))
15
-PP_SIZE = int(os.getenv("PP_SIZE", 1))
16
-
17
-pytestmark = pytest.mark.asyncio
18
-
5
20
-@pytest.fixture(scope="module")
21
-def server():
22
- args = [
6
+@pytest.mark.parametrize(
7
+ "TP_SIZE, PP_SIZE, EAGER_MODE, CHUNKED_PREFILL, MODEL_NAME", [
8
+ (2, 2, 0, 1, "meta-llama/Meta-Llama-3-8B"),
9
+ (2, 2, 1, 0, "meta-llama/Meta-Llama-3-8B"),
10
+ (1, 3, 0, 0, "meta-llama/Meta-Llama-3-8B"),
11
+ (1, 4, 0, 1, "meta-llama/Meta-Llama-3-8B"),
12
+ (1, 4, 1, 0, "meta-llama/Meta-Llama-3-8B"),
13
+ ])
14
+def test_compare_tp(TP_SIZE, PP_SIZE, EAGER_MODE, CHUNKED_PREFILL, MODEL_NAME):
15
+ pp_args = [
16
"--model",
17
MODEL_NAME,
18
# use half precision for speed and memory savings in CI environment
@@ -32,109 +25,107 @@ def server():
25
"--distributed-executor-backend",
26
"ray",
27
]
28
+
29
+ # compare without pipeline parallelism
30
+ # NOTE: use mp backend for TP
31
+ # PP tests might involve multiple nodes, and ray might
32
+ # schedule all workers in a node other than the head node,
33
+ # which can cause the test to fail.
34
+ tp_args = [
35
+ "--model",
36
+ MODEL_NAME,
37
+ # use half precision for speed and memory savings in CI environment
38
+ "--dtype",
39
+ "bfloat16",
40
+ "--tensor-parallel-size",
41
+ str(max(TP_SIZE, 2)), # use at least TP_SIZE=2 to hold the model
42
+ "--distributed-executor-backend",
43
+ "mp",
44
+ ]
45
if CHUNKED_PREFILL:
36
- args += [
37
- "--enable-chunked-prefill",
38
- ]
46
+ pp_args.append("--enable-chunked-prefill")
47
+ tp_args.append("--enable-chunked-prefill")
48
if EAGER_MODE:
40
- args += [
41
- "--enforce-eager",
42
- ]
43
- with RemoteOpenAIServer(args) as remote_server:
44
- yield remote_server
45
-
46
-
47
-@pytest.fixture(scope="module")
48
-def client(server):
49
- return server.get_async_client()
50
-
51
-
52
-async def test_check_models(server, client: openai.AsyncOpenAI):
53
- models = await client.models.list()
54
- models = models.data
55
- served_model = models[0]
56
- assert served_model.id == MODEL_NAME
57
- assert all(model.root == MODEL_NAME for model in models)
58
-
59
-
60
-@pytest.mark.parametrize(
61
- "model_name",
62
- [MODEL_NAME],
63
-)
64
-async def test_single_completion(server, client: openai.AsyncOpenAI,
65
- model_name: str):
66
- completion = await client.completions.create(model=model_name,
67
- prompt="Hello, my name is",
68
- max_tokens=5,
69
- temperature=0.0)
70
-
71
- assert completion.id is not None
72
- assert completion.choices is not None and len(completion.choices) == 1
73
- assert completion.choices[0].text is not None and len(
74
- completion.choices[0].text) >= 5
75
- assert completion.choices[0].finish_reason == "length"
76
- assert completion.usage == openai.types.CompletionUsage(
77
- completion_tokens=5, prompt_tokens=6, total_tokens=11)
78
-
79
- # test using token IDs
80
- completion = await client.completions.create(
81
- model=MODEL_NAME,
82
- prompt=[0, 0, 0, 0, 0],
83
- max_tokens=5,
84
- temperature=0.0,
85
- )
86
- assert completion.choices[0].text is not None and len(
87
- completion.choices[0].text) >= 5
88
-
89
-
90
-@pytest.mark.parametrize(
91
- # just test 1 lora hereafter
92
- "model_name",
93
- [MODEL_NAME],
94
-)
95
-async def test_batch_completions(server, client: openai.AsyncOpenAI,
96
- model_name: str):
97
- # test simple list
98
- batch = await client.completions.create(
99
- model=model_name,
100
- prompt=["Hello, my name is", "Hello, my name is"],
101
- max_tokens=5,
102
- temperature=0.0,
103
- )
104
- assert len(batch.choices) == 2
105
- assert batch.choices[0].text == batch.choices[1].text
106
-
107
- # test n = 2
108
- batch = await client.completions.create(
109
- model=model_name,
110
- prompt=["Hello, my name is", "Hello, my name is"],
111
- n=2,
112
- max_tokens=5,
113
- temperature=0.0,
114
- extra_body=dict(
115
- # NOTE: this has to be true for n > 1 in vLLM, but not necessary
116
- # for official client.
117
- use_beam_search=True),
118
- )
119
- assert len(batch.choices) == 4
120
- assert batch.choices[0].text != batch.choices[
121
- 1].text, "beam search should be different"
122
- assert batch.choices[0].text == batch.choices[
123
- 2].text, "two copies of the same prompt should be the same"
124
- assert batch.choices[1].text == batch.choices[
125
- 3].text, "two copies of the same prompt should be the same"
126
-
127
- # test streaming
128
- batch = await client.completions.create(
129
- model=model_name,
130
- prompt=["Hello, my name is", "Hello, my name is"],
131
- max_tokens=5,
132
- temperature=0.0,
133
- stream=True,
134
- )
135
- texts = [""] * 2
136
- async for chunk in batch:
137
- assert len(chunk.choices) == 1
138
- choice = chunk.choices[0]
139
- texts[choice.index] += choice.text
140
- assert texts[0] == texts[1]
49
+ pp_args.append("--enforce-eager")
50
+ tp_args.append("--enforce-eager")
51
+
52
+ results = []
53
+ for args in [pp_args, tp_args]:
54
+ with RemoteOpenAIServer(args) as server:
55
+ client = server.get_client()
56
+
57
+ # test models list
58
+ models = client.models.list()
59
+ models = models.data
60
+ served_model = models[0]
61
+ results.append({
62
+ "test": "models_list",
63
+ "id": served_model.id,
64
+ "root": served_model.root,
65
+ })
66
+
67
+ # test with text prompt
68
+ completion = client.completions.create(model=MODEL_NAME,
69
+ prompt="Hello, my name is",
70
+ max_tokens=5,
71
+ temperature=0.0)
72
+
73
+ results.append({
74
+ "test": "single_completion",
75
+ "text": completion.choices[0].text,
76
+ "finish_reason": completion.choices[0].finish_reason,
77
+ "usage": completion.usage,
78
+ })
79
+
80
+ # test using token IDs
81
+ completion = client.completions.create(
82
+ model=MODEL_NAME,
83
+ prompt=[0, 0, 0, 0, 0],
84
+ max_tokens=5,
85
+ temperature=0.0,
86
+ )
87
+
88
+ results.append({
89
+ "test": "token_ids",
90
+ "text": completion.choices[0].text,
91
+ "finish_reason": completion.choices[0].finish_reason,
92
+ "usage": completion.usage,
93
+ })
94
+
95
+ # test simple list
96
+ batch = client.completions.create(
97
+ model=MODEL_NAME,
98
+ prompt=["Hello, my name is", "Hello, my name is"],
99
+ max_tokens=5,
100
+ temperature=0.0,
101
+ )
102
+
103
+ results.append({
104
+ "test": "simple_list",
105
+ "text0": batch.choices[0].text,
106
+ "text1": batch.choices[1].text,
107
+ })
108
+
109
+ # test streaming
110
+ batch = client.completions.create(
111
+ model=MODEL_NAME,
112
+ prompt=["Hello, my name is", "Hello, my name is"],
113
+ max_tokens=5,
114
+ temperature=0.0,
115
+ stream=True,
116
+ )
117
+ texts = [""] * 2
118
+ for chunk in batch:
119
+ assert len(chunk.choices) == 1
120
+ choice = chunk.choices[0]
121
+ texts[choice.index] += choice.text
122
+ results.append({
123
+ "test": "streaming",
124
+ "texts": texts,
125
+ })
126
+
127
+ n = len(results) // 2
128
+ pp_results = results[:n]
129
+ tp_results = results[n:]
130
+ for pp, tp in zip(pp_results, tp_results):
131
+ assert pp == tp
vllm/executor/multiproc_gpu_executor.py
+15
@@ -1,5 +1,7 @@
1
import asyncio
2
import os
3
+import signal
4
+import weakref
5
from functools import partial
6
from typing import Any, List, Optional
7
@@ -78,6 +80,19 @@ class MultiprocessingGPUExecutor(DistributedGPUExecutor):
80
result_handler.start()
81
self.worker_monitor.start()
82
83
+ # Set up signal handlers to shutdown the executor cleanly
84
+ # sometimes gc does not work well
85
+
86
+ # Use weakref to avoid holding a reference to self
87
+ ref = weakref.ref(self)
88
+
89
+ def shutdown(signum, frame):
90
+ if executor := ref():
91
+ executor.shutdown()
92
+
93
+ signal.signal(signal.SIGINT, shutdown)
94
+ signal.signal(signal.SIGTERM, shutdown)
95
+
96
self.driver_worker = self._create_worker(
97
distributed_init_method=distributed_init_method)
98
self._run_workers("init_device")