@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 09c2eb85dd
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 20:45 UTC

[ci][distributed] add pipeline parallel correctness test (#6410)

youkaichao committed Jul 16, 2024 at 15:44 UTC 09c2eb85ddd3b2585979f4cd9cc97168d86718b6
3 files changed +129 -128
.buildkite/test-pipeline.yaml
+2 -7
@@ -72,7 +72,7 @@ steps:
72 commands:
73 - # the following commands are for the first node, with ip 192.168.10.10 (ray environment already set up)
74 - VLLM_TEST_SAME_HOST=0 torchrun --nnodes 2 --nproc-per-node=2 --rdzv_backend=c10d --rdzv_endpoint=192.168.10.10 distributed/test_same_node.py
75 - - TP_SIZE=2 PP_SIZE=2 EAGER_MODE=1 CHUNKED_PREFILL=0 pytest -v -s distributed/test_pipeline_parallel.py
75 + - pytest -v -s distributed/test_pipeline_parallel.py
76 - # the following commands are for the second node, with ip 192.168.10.11 (ray environment already set up)
77 - VLLM_TEST_SAME_HOST=0 torchrun --nnodes 2 --nproc-per-node=2 --rdzv_backend=c10d --rdzv_endpoint=192.168.10.10 distributed/test_same_node.py
78
@@ -115,12 +115,7 @@ steps:
115 working_dir: "/vllm-workspace/tests"
116 num_gpus: 4
117 commands:
118 - - TP_SIZE=2 PP_SIZE=2 EAGER_MODE=1 CHUNKED_PREFILL=1 pytest -v -s distributed/test_pipeline_parallel.py
119 - - TP_SIZE=2 PP_SIZE=2 EAGER_MODE=1 CHUNKED_PREFILL=0 pytest -v -s distributed/test_pipeline_parallel.py
120 - - TP_SIZE=1 PP_SIZE=3 EAGER_MODE=1 CHUNKED_PREFILL=0 pytest -v -s distributed/test_pipeline_parallel.py
121 - - PP_SIZE=4 EAGER_MODE=1 CHUNKED_PREFILL=1 pytest -v -s distributed/test_pipeline_parallel.py
122 - - PP_SIZE=4 EAGER_MODE=1 CHUNKED_PREFILL=0 pytest -v -s distributed/test_pipeline_parallel.py
123 -
118 + - pytest -v -s distributed/test_pipeline_parallel.py
119
120 - label: Engine Test
121 mirror_hardwares: [amd]
tests/distributed/test_pipeline_parallel.py
+112 -121
@@ -1,25 +1,18 @@
1 -import os
2 -
3 -import openai # use the official client for correctness check
1 import pytest
2
3 from ..utils import RemoteOpenAIServer
4
8 -# downloading lora to test lora requests
9 -
10 -# any model with a chat template should work here
11 -MODEL_NAME = "meta-llama/Meta-Llama-3-8B"
12 -EAGER_MODE = bool(int(os.getenv("EAGER_MODE", 0)))
13 -CHUNKED_PREFILL = bool(int(os.getenv("CHUNKED_PREFILL", 0)))
14 -TP_SIZE = int(os.getenv("TP_SIZE", 1))
15 -PP_SIZE = int(os.getenv("PP_SIZE", 1))
16 -
17 -pytestmark = pytest.mark.asyncio
18 -
5
20 -@pytest.fixture(scope="module")
21 -def server():
22 - args = [
6 +@pytest.mark.parametrize(
7 + "TP_SIZE, PP_SIZE, EAGER_MODE, CHUNKED_PREFILL, MODEL_NAME", [
8 + (2, 2, 0, 1, "meta-llama/Meta-Llama-3-8B"),
9 + (2, 2, 1, 0, "meta-llama/Meta-Llama-3-8B"),
10 + (1, 3, 0, 0, "meta-llama/Meta-Llama-3-8B"),
11 + (1, 4, 0, 1, "meta-llama/Meta-Llama-3-8B"),
12 + (1, 4, 1, 0, "meta-llama/Meta-Llama-3-8B"),
13 + ])
14 +def test_compare_tp(TP_SIZE, PP_SIZE, EAGER_MODE, CHUNKED_PREFILL, MODEL_NAME):
15 + pp_args = [
16 "--model",
17 MODEL_NAME,
18 # use half precision for speed and memory savings in CI environment
@@ -32,109 +25,107 @@ def server():
25 "--distributed-executor-backend",
26 "ray",
27 ]
28 +
29 + # compare without pipeline parallelism
30 + # NOTE: use mp backend for TP
31 + # PP tests might involve multiple nodes, and ray might
32 + # schedule all workers in a node other than the head node,
33 + # which can cause the test to fail.
34 + tp_args = [
35 + "--model",
36 + MODEL_NAME,
37 + # use half precision for speed and memory savings in CI environment
38 + "--dtype",
39 + "bfloat16",
40 + "--tensor-parallel-size",
41 + str(max(TP_SIZE, 2)), # use at least TP_SIZE=2 to hold the model
42 + "--distributed-executor-backend",
43 + "mp",
44 + ]
45 if CHUNKED_PREFILL:
36 - args += [
37 - "--enable-chunked-prefill",
38 - ]
46 + pp_args.append("--enable-chunked-prefill")
47 + tp_args.append("--enable-chunked-prefill")
48 if EAGER_MODE:
40 - args += [
41 - "--enforce-eager",
42 - ]
43 - with RemoteOpenAIServer(args) as remote_server:
44 - yield remote_server
45 -
46 -
47 -@pytest.fixture(scope="module")
48 -def client(server):
49 - return server.get_async_client()
50 -
51 -
52 -async def test_check_models(server, client: openai.AsyncOpenAI):
53 - models = await client.models.list()
54 - models = models.data
55 - served_model = models[0]
56 - assert served_model.id == MODEL_NAME
57 - assert all(model.root == MODEL_NAME for model in models)
58 -
59 -
60 -@pytest.mark.parametrize(
61 - "model_name",
62 - [MODEL_NAME],
63 -)
64 -async def test_single_completion(server, client: openai.AsyncOpenAI,
65 - model_name: str):
66 - completion = await client.completions.create(model=model_name,
67 - prompt="Hello, my name is",
68 - max_tokens=5,
69 - temperature=0.0)
70 -
71 - assert completion.id is not None
72 - assert completion.choices is not None and len(completion.choices) == 1
73 - assert completion.choices[0].text is not None and len(
74 - completion.choices[0].text) >= 5
75 - assert completion.choices[0].finish_reason == "length"
76 - assert completion.usage == openai.types.CompletionUsage(
77 - completion_tokens=5, prompt_tokens=6, total_tokens=11)
78 -
79 - # test using token IDs
80 - completion = await client.completions.create(
81 - model=MODEL_NAME,
82 - prompt=[0, 0, 0, 0, 0],
83 - max_tokens=5,
84 - temperature=0.0,
85 - )
86 - assert completion.choices[0].text is not None and len(
87 - completion.choices[0].text) >= 5
88 -
89 -
90 -@pytest.mark.parametrize(
91 - # just test 1 lora hereafter
92 - "model_name",
93 - [MODEL_NAME],
94 -)
95 -async def test_batch_completions(server, client: openai.AsyncOpenAI,
96 - model_name: str):
97 - # test simple list
98 - batch = await client.completions.create(
99 - model=model_name,
100 - prompt=["Hello, my name is", "Hello, my name is"],
101 - max_tokens=5,
102 - temperature=0.0,
103 - )
104 - assert len(batch.choices) == 2
105 - assert batch.choices[0].text == batch.choices[1].text
106 -
107 - # test n = 2
108 - batch = await client.completions.create(
109 - model=model_name,
110 - prompt=["Hello, my name is", "Hello, my name is"],
111 - n=2,
112 - max_tokens=5,
113 - temperature=0.0,
114 - extra_body=dict(
115 - # NOTE: this has to be true for n > 1 in vLLM, but not necessary
116 - # for official client.
117 - use_beam_search=True),
118 - )
119 - assert len(batch.choices) == 4
120 - assert batch.choices[0].text != batch.choices[
121 - 1].text, "beam search should be different"
122 - assert batch.choices[0].text == batch.choices[
123 - 2].text, "two copies of the same prompt should be the same"
124 - assert batch.choices[1].text == batch.choices[
125 - 3].text, "two copies of the same prompt should be the same"
126 -
127 - # test streaming
128 - batch = await client.completions.create(
129 - model=model_name,
130 - prompt=["Hello, my name is", "Hello, my name is"],
131 - max_tokens=5,
132 - temperature=0.0,
133 - stream=True,
134 - )
135 - texts = [""] * 2
136 - async for chunk in batch:
137 - assert len(chunk.choices) == 1
138 - choice = chunk.choices[0]
139 - texts[choice.index] += choice.text
140 - assert texts[0] == texts[1]
49 + pp_args.append("--enforce-eager")
50 + tp_args.append("--enforce-eager")
51 +
52 + results = []
53 + for args in [pp_args, tp_args]:
54 + with RemoteOpenAIServer(args) as server:
55 + client = server.get_client()
56 +
57 + # test models list
58 + models = client.models.list()
59 + models = models.data
60 + served_model = models[0]
61 + results.append({
62 + "test": "models_list",
63 + "id": served_model.id,
64 + "root": served_model.root,
65 + })
66 +
67 + # test with text prompt
68 + completion = client.completions.create(model=MODEL_NAME,
69 + prompt="Hello, my name is",
70 + max_tokens=5,
71 + temperature=0.0)
72 +
73 + results.append({
74 + "test": "single_completion",
75 + "text": completion.choices[0].text,
76 + "finish_reason": completion.choices[0].finish_reason,
77 + "usage": completion.usage,
78 + })
79 +
80 + # test using token IDs
81 + completion = client.completions.create(
82 + model=MODEL_NAME,
83 + prompt=[0, 0, 0, 0, 0],
84 + max_tokens=5,
85 + temperature=0.0,
86 + )
87 +
88 + results.append({
89 + "test": "token_ids",
90 + "text": completion.choices[0].text,
91 + "finish_reason": completion.choices[0].finish_reason,
92 + "usage": completion.usage,
93 + })
94 +
95 + # test simple list
96 + batch = client.completions.create(
97 + model=MODEL_NAME,
98 + prompt=["Hello, my name is", "Hello, my name is"],
99 + max_tokens=5,
100 + temperature=0.0,
101 + )
102 +
103 + results.append({
104 + "test": "simple_list",
105 + "text0": batch.choices[0].text,
106 + "text1": batch.choices[1].text,
107 + })
108 +
109 + # test streaming
110 + batch = client.completions.create(
111 + model=MODEL_NAME,
112 + prompt=["Hello, my name is", "Hello, my name is"],
113 + max_tokens=5,
114 + temperature=0.0,
115 + stream=True,
116 + )
117 + texts = [""] * 2
118 + for chunk in batch:
119 + assert len(chunk.choices) == 1
120 + choice = chunk.choices[0]
121 + texts[choice.index] += choice.text
122 + results.append({
123 + "test": "streaming",
124 + "texts": texts,
125 + })
126 +
127 + n = len(results) // 2
128 + pp_results = results[:n]
129 + tp_results = results[n:]
130 + for pp, tp in zip(pp_results, tp_results):
131 + assert pp == tp
vllm/executor/multiproc_gpu_executor.py
+15
@@ -1,5 +1,7 @@
1 import asyncio
2 import os
3 +import signal
4 +import weakref
5 from functools import partial
6 from typing import Any, List, Optional
7
@@ -78,6 +80,19 @@ class MultiprocessingGPUExecutor(DistributedGPUExecutor):
80 result_handler.start()
81 self.worker_monitor.start()
82
83 + # Set up signal handlers to shutdown the executor cleanly
84 + # sometimes gc does not work well
85 +
86 + # Use weakref to avoid holding a reference to self
87 + ref = weakref.ref(self)
88 +
89 + def shutdown(signum, frame):
90 + if executor := ref():
91 + executor.shutdown()
92 +
93 + signal.signal(signal.SIGINT, shutdown)
94 + signal.signal(signal.SIGTERM, shutdown)
95 +
96 self.driver_worker = self._create_worker(
97 distributed_init_method=distributed_init_method)
98 self._run_workers("init_device")