A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
TP/quantization/weight loading refactor part 1 - Simplify parallel linear logic (#1181)
Zhuohan Li committed
Oct 2, 2023 at 15:36 UTC
ba0bfd40e21cacfd5da6a1e43028a37258a29cb4
42 files changed
+823
-1551
.github/workflows/pylint.yml
+1
-1
@@ -28,4 +28,4 @@ jobs:
28
pip install pylint==2.8.2
29
- name: Analysing the code with pylint
30
run: |
31
- pylint vllm
31
+ pylint vllm tests
.github/workflows/yapf.yml
+1
-1
@@ -28,4 +28,4 @@ jobs:
28
pip install toml==0.10.2
29
- name: Running yapf
30
run: |
31
- yapf --diff --recursive vllm --exclude 'vllm/model_executor/parallel_utils/**'
31
+ yapf --diff --recursive vllm tests
.pylintrc
+1
-1
@@ -8,7 +8,7 @@
8
[MASTER]
9
10
# Files or directories to be skipped. They should be base names, not paths.
11
-ignore=docs,parallel_utils
11
+ignore=docs
12
13
# Files or directories matching the regex patterns are skipped. The regex
14
# matches against base names, not paths.
format.sh
+2
-3
@@ -44,7 +44,6 @@ YAPF_FLAGS=(
44
45
YAPF_EXCLUDES=(
46
'--exclude' 'build/**'
47
- '--exclude' 'vllm/model_executor/parallel_utils/**'
47
)
48
49
# Format specified files
@@ -72,7 +71,7 @@ format_changed() {
71
72
# Format all files
73
format_all() {
75
- yapf --in-place "${YAPF_FLAGS[@]}" "${YAPF_EXCLUDES[@]}" vllm
74
+ yapf --in-place "${YAPF_FLAGS[@]}" "${YAPF_EXCLUDES[@]}" vllm tests
75
}
76
77
## This flag formats individual files. --files *must* be the first command line
@@ -96,7 +95,7 @@ echo 'vLLM yapf: Done'
95
96
# Run Pylint
97
echo 'vLLM Pylint:'
99
-pylint vllm
98
+pylint vllm tests
99
100
if ! git diff --quiet &>/dev/null; then
101
echo 'Reformatted files. Please review and stage the changes.'
tests/async_engine/api_server_async_engine.py
+1
@@ -14,6 +14,7 @@ app = vllm.entrypoints.api_server.app
14
15
class AsyncLLMEngineWithStats(AsyncLLMEngine):
16
17
+ # pylint: disable=redefined-outer-name
18
def __init__(self, *args, **kwargs):
19
super().__init__(*args, **kwargs)
20
self._num_aborts = 0
tests/async_engine/test_api_server.py
+3
@@ -24,6 +24,7 @@ def _query_server(prompt: str) -> dict:
24
def api_server():
25
script_path = Path(__file__).parent.joinpath(
26
"api_server_async_engine.py").absolute()
27
+ # pylint: disable=consider-using-with
28
uvicorn_process = subprocess.Popen([
29
sys.executable, "-u",
30
str(script_path), "--model", "facebook/opt-125m"
@@ -32,6 +33,7 @@ def api_server():
33
uvicorn_process.terminate()
34
35
36
+# pylint: disable=redefined-outer-name, unused-argument
37
def test_api_server(api_server):
38
"""
39
Run the API server and test it.
@@ -47,6 +49,7 @@ def test_api_server(api_server):
49
prompts = ["Hello world"] * 1
50
result = None
51
while not result:
52
+ # pylint: disable=bare-except
53
try:
54
for result in pool.map(_query_server, prompts):
55
break
tests/async_engine/test_async_llm_engine.py
+2
-2
@@ -32,12 +32,12 @@ class MockEngine:
32
self.request_id = None
33
34
def add_request(self, **kwargs):
35
+ del kwargs # Unused
36
self.add_request_calls += 1
36
- return
37
38
def abort_request(self, request_id):
39
+ del request_id # Unused
40
self.abort_request_calls += 1
40
- return
41
42
43
class MockAsyncLLMEngine(AsyncLLMEngine):
tests/async_engine/test_request_tracker.py
+11
-11
@@ -7,22 +7,22 @@ from vllm.outputs import RequestOutput
7
class DummyEvent:
8
9
def __init__(self):
10
- self._flag = False
10
+ self.flag = False
11
12
def set(self):
13
- self._flag = True
13
+ self.flag = True
14
15
def clear(self):
16
- self._flag = False
16
+ self.flag = False
17
18
19
def test_request_tracker():
20
tracker = RequestTracker()
21
tracker.new_requests_event = DummyEvent()
22
stream_1 = tracker.add_request("1")
23
- assert tracker.new_requests_event._flag
23
+ assert tracker.new_requests_event.flag
24
new, finished = tracker.get_new_and_finished_requests()
25
- assert not tracker.new_requests_event._flag
25
+ assert not tracker.new_requests_event.flag
26
assert len(new) == 1
27
assert new[0]["request_id"] == "1"
28
assert not finished
@@ -30,9 +30,9 @@ def test_request_tracker():
30
31
stream_2 = tracker.add_request("2")
32
stream_3 = tracker.add_request("3")
33
- assert tracker.new_requests_event._flag
33
+ assert tracker.new_requests_event.flag
34
new, finished = tracker.get_new_and_finished_requests()
35
- assert not tracker.new_requests_event._flag
35
+ assert not tracker.new_requests_event.flag
36
assert len(new) == 2
37
assert new[0]["request_id"] == "2"
38
assert new[1]["request_id"] == "3"
@@ -43,7 +43,7 @@ def test_request_tracker():
43
# request_ids must be unique
44
with pytest.raises(KeyError):
45
tracker.add_request("1")
46
- assert not tracker.new_requests_event._flag
46
+ assert not tracker.new_requests_event.flag
47
48
tracker.abort_request("1")
49
new, finished = tracker.get_new_and_finished_requests()
@@ -54,7 +54,7 @@ def test_request_tracker():
54
55
stream_4 = tracker.add_request("4")
56
tracker.abort_request("4")
57
- assert tracker.new_requests_event._flag
57
+ assert tracker.new_requests_event.flag
58
new, finished = tracker.get_new_and_finished_requests()
59
assert len(finished) == 1
60
assert "4" in finished
@@ -62,11 +62,11 @@ def test_request_tracker():
62
assert stream_4.finished
63
64
stream_5 = tracker.add_request("5")
65
- assert tracker.new_requests_event._flag
65
+ assert tracker.new_requests_event.flag
66
tracker.process_request_output(
67
RequestOutput("2", "output", [], [], finished=True))
68
new, finished = tracker.get_new_and_finished_requests()
69
- assert not tracker.new_requests_event._flag
69
+ assert not tracker.new_requests_event.flag
70
assert len(finished) == 1
71
assert "2" in finished
72
assert len(new) == 1
tests/conftest.py
+1
@@ -8,6 +8,7 @@ from vllm import LLM, SamplingParams
8
from vllm.transformers_utils.tokenizer import get_tokenizer
9
10
_TEST_PROMPTS = [
11
+ # pylint: disable=line-too-long
12
"vLLM is a high-throughput and memory-efficient inference and serving engine for LLMs.",
13
"Briefly describe the major milestones in the development of artificial intelligence from 1950 to 2020.",
14
"Compare and contrast artificial intelligence with human intelligence in terms of processing information.",
tests/distributed/test_comm_ops.py
new
+82
@@ -0,0 +1,82 @@
1
+"""Test the communication operators.
2
+
3
+Run `pytest tests/distributed/test_comm_ops.py --forked`.
4
+"""
5
+from multiprocessing import Process
6
+
7
+import pytest
8
+import torch
9
+
10
+from vllm.config import ParallelConfig
11
+from vllm.engine.ray_utils import get_open_port
12
+from vllm.model_executor.parallel_utils.communication_op import (
13
+ tensor_model_parallel_all_reduce,
14
+ tensor_model_parallel_all_gather,
15
+)
16
+from vllm.worker.worker import _init_distributed_environment
17
+
18
+
19
+def init_test_distributed_environment(pipeline_parallel_size: int,
20
+ tensor_parallel_size: int, rank: int,
21
+ distributed_init_port: str):
22
+ parallel_config = ParallelConfig(pipeline_parallel_size,
23
+ tensor_parallel_size,
24
+ worker_use_ray=True)
25
+ distributed_init_method = f"tcp://localhost:{distributed_init_port}"
26
+ torch.cuda.set_device(rank)
27
+ _init_distributed_environment(parallel_config, rank,
28
+ distributed_init_method)
29
+
30
+
31
+def all_reduce_test_worker(tensor_parallel_size: int, rank: int,
32
+ distributed_init_port: str):
33
+ init_test_distributed_environment(1, tensor_parallel_size, rank,
34
+ distributed_init_port)
35
+ num_elements = 8
36
+ all_tensors = [
37
+ torch.arange(num_elements, dtype=torch.float32, device="cuda") *
38
+ (r + 1) for r in range(tensor_parallel_size)
39
+ ]
40
+ expected = torch.sum(torch.stack(all_tensors, dim=0), dim=0)
41
+ t = all_tensors[rank]
42
+ t = tensor_model_parallel_all_reduce(t)
43
+ assert torch.allclose(t, expected)
44
+
45
+
46
+def all_gather_test_worker(tensor_parallel_size: int, rank: int,
47
+ distributed_init_port: str):
48
+ init_test_distributed_environment(1, tensor_parallel_size, rank,
49
+ distributed_init_port)
50
+ num_dimensions = 3
51
+ tensor_size = list(range(2, num_dimensions + 2))
52
+ total_size = 1
53
+ for s in tensor_size:
54
+ total_size *= s
55
+ for all_gather_dimension in range(num_dimensions):
56
+ all_tensors = [
57
+ torch.arange(total_size, dtype=torch.float32,
58
+ device="cuda").reshape(tensor_size) * (r + 1)
59
+ for r in range(tensor_parallel_size)
60
+ ]
61
+ expected = torch.cat(all_tensors, dim=all_gather_dimension)
62
+ t = all_tensors[rank]
63
+ t = tensor_model_parallel_all_gather(t, all_gather_dimension)
64
+ assert torch.allclose(t, expected)
65
+
66
+
67
+@pytest.mark.skipif(torch.cuda.device_count() < 2,
68
+ reason="Need at least 2 GPUs to run the test.")
69
+@pytest.mark.parametrize("tensor_parallel_size", [2])
70
+@pytest.mark.parametrize("test_target",
71
+ [all_reduce_test_worker, all_gather_test_worker])
72
+def test_multi_process_tensor_parallel(tensor_parallel_size, test_target):
73
+ distributed_init_port = get_open_port()
74
+ processes = []
75
+ for rank in range(tensor_parallel_size):
76
+ p = Process(target=test_target,
77
+ args=(tensor_parallel_size, rank, distributed_init_port))
78
+ p.start()
79
+ processes.append(p)
80
+ for p in processes:
81
+ p.join()
82
+ assert all(p.exitcode == 0 for p in processes)
tests/engine/test_detokenize.py
+1
@@ -5,6 +5,7 @@ from transformers import AutoTokenizer
5
from vllm.transformers_utils.tokenizer import detokenize_incrementally
6
7
TRUTH = [
8
+ # pylint: disable=line-too-long
9
"Hello here, this is a simple test",
10
"vLLM is a high-throughput and memory-efficient inference and serving engine for LLMs. It is designed to be used in production environments, where inference and serving",
11
"我很感谢你的热情"
tests/kernels/test_activation.py
+6
-6
@@ -29,8 +29,8 @@ def test_silu_and_mul(
29
) -> None:
30
torch.random.manual_seed(seed)
31
torch.cuda.manual_seed(seed)
32
- x = torch.randn(num_tokens, 2 * d, dtype=dtype, device='cuda')
33
- out = torch.empty(num_tokens, d, dtype=dtype, device='cuda')
32
+ x = torch.randn(num_tokens, 2 * d, dtype=dtype, device="cuda")
33
+ out = torch.empty(num_tokens, d, dtype=dtype, device="cuda")
34
activation_ops.silu_and_mul(out, x)
35
ref_out = ref_silu_and_mul(x)
36
assert torch.allclose(out, ref_out, atol=1e-5, rtol=1e-5)
@@ -49,8 +49,8 @@ def test_gelu_new(
49
) -> None:
50
torch.random.manual_seed(seed)
51
torch.cuda.manual_seed(seed)
52
- x = torch.randn(num_tokens, d, dtype=dtype, device='cuda')
53
- out = torch.empty(num_tokens, d, dtype=dtype, device='cuda')
52
+ x = torch.randn(num_tokens, d, dtype=dtype, device="cuda")
53
+ out = torch.empty(num_tokens, d, dtype=dtype, device="cuda")
54
activation_ops.gelu_new(out, x)
55
ref_out = get_activation("gelu_new")(x)
56
assert torch.allclose(out, ref_out, atol=1e-5, rtol=1e-5)
@@ -68,8 +68,8 @@ def test_gelu_fast(
68
) -> None:
69
torch.random.manual_seed(seed)
70
torch.cuda.manual_seed(seed)
71
- x = torch.randn(num_tokens, d, dtype=dtype, device='cuda')
72
- out = torch.empty(num_tokens, d, dtype=dtype, device='cuda')
71
+ x = torch.randn(num_tokens, d, dtype=dtype, device="cuda")
72
+ out = torch.empty(num_tokens, d, dtype=dtype, device="cuda")
73
activation_ops.gelu_fast(out, x)
74
ref_out = get_activation("gelu_fast")(x)
75
assert torch.allclose(out, ref_out, atol=1e-5, rtol=1e-5)
tests/kernels/test_cache.py
+3
-3
@@ -106,14 +106,14 @@ def test_reshape_and_cache(
106
# Create a random slot mapping.
107
num_slots = block_size * num_blocks
108
slot_mapping = random.sample(range(num_slots), num_tokens)
109
- slot_mapping = torch.tensor(slot_mapping, dtype=torch.int, device='cuda')
109
+ slot_mapping = torch.tensor(slot_mapping, dtype=torch.int, device="cuda")
110
111
qkv = torch.randn(num_tokens,
112
3,
113
num_heads,
114
head_size,
115
dtype=dtype,
116
- device='cuda')
116
+ device="cuda")
117
_, key, value = qkv.unbind(dim=1)
118
119
# Create the KV caches.
@@ -132,7 +132,7 @@ def test_reshape_and_cache(
132
133
# Run the reference implementation.
134
reshaped_key = key.reshape(num_tokens, *key_cache[0, :, :, 0, :].shape)
135
- block_indicies = torch.div(slot_mapping, block_size, rounding_mode='floor')
135
+ block_indicies = torch.div(slot_mapping, block_size, rounding_mode="floor")
136
block_indicies = block_indicies.cpu().tolist()
137
block_offsets = slot_mapping % block_size
138
block_offsets = block_offsets.cpu().tolist()
tests/kernels/test_pos_encoding.py
+1
-1
@@ -140,7 +140,7 @@ def test_rotary_embedding(
140
cos = freqs.cos()
141
sin = freqs.sin()
142
cos_sin_cache = torch.cat((cos, sin), dim=-1)
143
- cos_sin_cache = cos_sin_cache.to(dtype=dtype, device='cuda')
143
+ cos_sin_cache = cos_sin_cache.to(dtype=dtype, device="cuda")
144
145
# Run the kernel. The kernel is in-place, so we need to clone the inputs.
146
out_query = query.clone()
tests/samplers/test_sampler.py
+2
-1
@@ -1,3 +1,4 @@
1
+# pylint: disable=protected-access
2
import pytest
3
import random
4
from typing import Tuple
@@ -108,7 +109,7 @@ def test_sampler_all_random(seed: int):
109
def test_sampler_all_beam(seed: int):
110
set_random_seed(seed)
111
batch_size = random.randint(1, 256)
111
- input_tensor, fake_logits, sampler, worker = _prepare_test(batch_size)
112
+ input_tensor, _, sampler, worker = _prepare_test(batch_size)
113
114
seq_group_metadata_list = []
115
for i in range(batch_size):
vllm/model_executor/layers/quantized_linear/__init__.py
+2
-2
@@ -1,7 +1,7 @@
1
from vllm.model_executor.layers.quantized_linear.awq import (
2
AWQColumnParallelLinear, AWQRowParallelLinear)
3
-from vllm.model_executor.parallel_utils.tensor_parallel import (
4
- ColumnParallelLinear, RowParallelLinear)
3
+from vllm.model_executor.parallel_utils.layers import (ColumnParallelLinear,
4
+ RowParallelLinear)
5
6
_QUANTIZED_LINEAR_REGISTRY = {
7
"awq": (AWQColumnParallelLinear, AWQRowParallelLinear),
vllm/model_executor/layers/quantized_linear/awq.py
+2
-2
@@ -4,8 +4,8 @@ import torch
4
from torch.nn.parameter import Parameter
5
6
from vllm import quantization_ops
7
-from vllm.model_executor.parallel_utils.tensor_parallel.layers import (
8
- ColumnParallelLinear, RowParallelLinear)
7
+from vllm.model_executor.parallel_utils.layers import (ColumnParallelLinear,
8
+ RowParallelLinear)
9
10
11
class AWQColumnParallelLinear(ColumnParallelLinear):
vllm/model_executor/layers/sampler.py
+3
-3
@@ -5,8 +5,8 @@ import torch
5
import torch.nn as nn
6
7
from vllm.model_executor.input_metadata import InputMetadata
8
-from vllm.model_executor.parallel_utils.tensor_parallel import (
9
- gather_from_tensor_model_parallel_region)
8
+from vllm.model_executor.parallel_utils.communication_op import (
9
+ tensor_model_parallel_all_gather)
10
from vllm.sampling_params import SamplingParams, SamplingType
11
from vllm.sequence import SamplerOutput, SequenceData, SequenceOutputs
12
@@ -92,7 +92,7 @@ def _get_logits(hidden_states: torch.Tensor, embedding: torch.Tensor,
92
logits = torch.matmul(hidden_states, embedding.t())
93
if embedding_bias is not None:
94
logits += embedding_bias
95
- logits = gather_from_tensor_model_parallel_region(logits)
95
+ logits = tensor_model_parallel_all_gather(logits)
96
# Remove paddings in vocab (if any).
97
logits = logits[:, :vocab_size]
98
return logits
vllm/model_executor/models/aquila.py
+22
-20
@@ -39,8 +39,9 @@ from vllm.model_executor.weight_utils import (
39
load_tensor_parallel_weights)
40
from vllm.model_executor.parallel_utils.parallel_state import (
41
get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
42
-from vllm.model_executor.parallel_utils.tensor_parallel import (
43
- VocabParallelEmbedding, ColumnParallelLinear, RowParallelLinear)
42
+from vllm.model_executor.parallel_utils.layers import (VocabParallelEmbedding,
43
+ ColumnParallelLinear,
44
+ RowParallelLinear)
45
from vllm.sequence import SamplerOutput
46
from vllm.transformers_utils.configs.aquila import AquilaConfig
47
@@ -56,16 +57,18 @@ class AquilaMLP(nn.Module):
57
hidden_act: str,
58
):
59
super().__init__()
59
- self.gate_up_proj = ColumnParallelLinear(hidden_size,
60
- 2 * intermediate_size,
61
- bias=False,
62
- gather_output=False,
63
- perform_initialization=False)
64
- self.down_proj = RowParallelLinear(intermediate_size,
65
- hidden_size,
66
- bias=False,
67
- input_is_parallel=True,
68
- perform_initialization=False)
60
+ self.gate_up_proj = ColumnParallelLinear(
61
+ hidden_size,
62
+ 2 * intermediate_size,
63
+ bias=False,
64
+ gather_output=False,
65
+ )
66
+ self.down_proj = RowParallelLinear(
67
+ intermediate_size,
68
+ hidden_size,
69
+ bias=False,
70
+ input_is_parallel=True,
71
+ )
72
if hidden_act != "silu":
73
raise ValueError(f"Unsupported activation: {hidden_act}. "
74
"Only silu is supported for now.")
@@ -130,14 +133,12 @@ class AquilaAttention(nn.Module):
133
self.head_dim,
134
bias=False,
135
gather_output=False,
133
- perform_initialization=False,
136
)
137
self.o_proj = RowParallelLinear(
138
self.total_num_heads * self.head_dim,
139
hidden_size,
140
bias=False,
141
input_is_parallel=True,
140
- perform_initialization=False,
142
)
143
self.attn = PagedAttentionWithRoPE(
144
self.num_heads,
@@ -230,7 +231,7 @@ class AquilaModel(nn.Module):
231
self.embed_tokens = VocabParallelEmbedding(
232
config.vocab_size,
233
config.hidden_size,
233
- perform_initialization=False)
234
+ )
235
self.layers = nn.ModuleList([
236
AquilaDecoderLayer(config) for _ in range(config.num_hidden_layers)
237
])
@@ -270,11 +271,12 @@ class AquilaForCausalLM(nn.Module):
271
self.config = config
272
self.model = AquilaModel(config)
273
vocab_size = ((config.vocab_size + 63) // 64) * 64
273
- self.lm_head = ColumnParallelLinear(config.hidden_size,
274
- vocab_size,
275
- bias=False,
276
- gather_output=False,
277
- perform_initialization=False)
274
+ self.lm_head = ColumnParallelLinear(
275
+ config.hidden_size,
276
+ vocab_size,
277
+ bias=False,
278
+ gather_output=False,
279
+ )
280
self.sampler = Sampler(config.vocab_size)
281
282
def forward(
vllm/model_executor/models/baichuan.py
+22
-20
@@ -39,8 +39,9 @@ from vllm.model_executor.weight_utils import (
39
load_padded_tensor_parallel_vocab, load_tensor_parallel_weights)
40
from vllm.model_executor.parallel_utils.parallel_state import (
41
get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
42
-from vllm.model_executor.parallel_utils.tensor_parallel import (
43
- VocabParallelEmbedding, ColumnParallelLinear, RowParallelLinear)
42
+from vllm.model_executor.parallel_utils.layers import (VocabParallelEmbedding,
43
+ ColumnParallelLinear,
44
+ RowParallelLinear)
45
from vllm.sequence import SamplerOutput
46
from vllm.transformers_utils.configs.baichuan import BaiChuanConfig
47
@@ -81,16 +82,18 @@ class BaiChuanMLP(nn.Module):
82
hidden_act: str,
83
):
84
super().__init__()
84
- self.gate_up_proj = ColumnParallelLinear(hidden_size,
85
- 2 * intermediate_size,
86
- bias=False,
87
- gather_output=False,
88
- perform_initialization=False)
89
- self.down_proj = RowParallelLinear(intermediate_size,
90
- hidden_size,
91
- bias=False,
92
- input_is_parallel=True,
93
- perform_initialization=False)
85
+ self.gate_up_proj = ColumnParallelLinear(
86
+ hidden_size,
87
+ 2 * intermediate_size,
88
+ bias=False,
89
+ gather_output=False,
90
+ )
91
+ self.down_proj = RowParallelLinear(
92
+ intermediate_size,
93
+ hidden_size,
94
+ bias=False,
95
+ input_is_parallel=True,
96
+ )
97
if hidden_act != "silu":
98
raise ValueError(f"Unsupported activation: {hidden_act}. "
99
"Only silu is supported for now.")
@@ -133,14 +136,12 @@ class BaiChuanAttention(nn.Module):
136
3 * hidden_size,
137
bias=False,
138
gather_output=False,
136
- perform_initialization=False,
139
)
140
self.o_proj = RowParallelLinear(
141
self.total_num_heads * self.head_dim,
142
hidden_size,
143
bias=False,
144
input_is_parallel=True,
143
- perform_initialization=False,
145
)
146
# Create the alibi slopes and slice them.
147
if self.postion_embedding == "ALIBI":
@@ -249,7 +250,7 @@ class BaiChuanModel(nn.Module):
250
self.embed_tokens = VocabParallelEmbedding(
251
config.vocab_size,
252
config.hidden_size,
252
- perform_initialization=False)
253
+ )
254
self.layers = nn.ModuleList([
255
BaiChuanDecoderLayer(config, position_embedding)
256
for _ in range(config.num_hidden_layers)
@@ -288,11 +289,12 @@ class BaiChuanBaseForCausalLM(nn.Module):
289
super().__init__()
290
self.config = config
291
self.model = BaiChuanModel(config, position_embedding)
291
- self.lm_head = ColumnParallelLinear(config.hidden_size,
292
- config.vocab_size,
293
- bias=False,
294
- gather_output=False,
295
- perform_initialization=False)
292
+ self.lm_head = ColumnParallelLinear(
293
+ config.hidden_size,
294
+ config.vocab_size,
295
+ bias=False,
296
+ gather_output=False,
297
+ )
298
self.sampler = Sampler(config.vocab_size)
299
300
def forward(
vllm/model_executor/models/bloom.py
+16
-13
@@ -35,8 +35,9 @@ from vllm.model_executor.weight_utils import (hf_model_weights_iterator,
35
load_tensor_parallel_weights)
36
from vllm.model_executor.parallel_utils.parallel_state import (
37
get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
38
-from vllm.model_executor.parallel_utils.tensor_parallel import (
39
- VocabParallelEmbedding, ColumnParallelLinear, RowParallelLinear)
38
+from vllm.model_executor.parallel_utils.layers import (VocabParallelEmbedding,
39
+ ColumnParallelLinear,
40
+ RowParallelLinear)
41
from vllm.sequence import SamplerOutput
42
43
KVCache = Tuple[torch.Tensor, torch.Tensor]
@@ -85,14 +86,12 @@ class BloomAttention(nn.Module):
86
3 * self.hidden_size,
87
bias=True,
88
gather_output=False,
88
- perform_initialization=False,
89
)
90
self.dense = RowParallelLinear(
91
self.hidden_size,
92
self.hidden_size,
93
bias=True,
94
input_is_parallel=True,
95
- perform_initialization=False,
95
)
96
97
# Create the alibi slopes and slice them.
@@ -129,15 +128,17 @@ class BloomMLP(nn.Module):
128
def __init__(self, config: BloomConfig):
129
super().__init__()
130
hidden_size = config.hidden_size
132
- self.dense_h_to_4h = ColumnParallelLinear(hidden_size,
133
- 4 * hidden_size,
134
- gather_output=False,
135
- perform_initialization=False)
131
+ self.dense_h_to_4h = ColumnParallelLinear(
132
+ hidden_size,
133
+ 4 * hidden_size,
134
+ gather_output=False,
135
+ )
136
self.act = get_act_fn("gelu")
137
- self.dense_4h_to_h = RowParallelLinear(4 * hidden_size,
138
- hidden_size,
139
- input_is_parallel=True,
140
- perform_initialization=False)
137
+ self.dense_4h_to_h = RowParallelLinear(
138
+ 4 * hidden_size,
139
+ hidden_size,
140
+ input_is_parallel=True,
141
+ )
142
143
def forward(self, x: torch.Tensor) -> torch.Tensor:
144
x, _ = self.dense_h_to_4h(x)
@@ -208,7 +209,9 @@ class BloomModel(nn.Module):
209
210
# Embedding + LN Embedding
211
self.word_embeddings = VocabParallelEmbedding(
211
- config.vocab_size, self.embed_dim, perform_initialization=False)
212
+ config.vocab_size,
213
+ self.embed_dim,
214
+ )
215
self.word_embeddings_layernorm = nn.LayerNorm(
216
self.embed_dim, eps=config.layer_norm_epsilon)
217
vllm/model_executor/models/falcon.py
+15
-16
@@ -36,9 +36,11 @@ from vllm.model_executor.weight_utils import (convert_pyslice_to_tensor,
36
load_tensor_parallel_weights)
37
from vllm.model_executor.parallel_utils.parallel_state import (
38
get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
39
-from vllm.model_executor.parallel_utils.tensor_parallel import (
40
- VocabParallelEmbedding, ColumnParallelLinear, RowParallelLinear,
41
- reduce_from_tensor_model_parallel_region)
39
+from vllm.model_executor.parallel_utils.layers import (VocabParallelEmbedding,
40
+ ColumnParallelLinear,
41
+ RowParallelLinear)
42
+from vllm.model_executor.parallel_utils.communication_op import (
43
+ tensor_model_parallel_all_reduce)
44
from vllm.sequence import SamplerOutput
45
from vllm.transformers_utils.configs import RWConfig
46
@@ -109,7 +111,6 @@ class FalconAttention(nn.Module):
111
self.head_dim,
112
bias=config.bias,
113
gather_output=False,
112
- perform_initialization=False,
114
skip_bias_add=True,
115
)
116
elif self.multi_query:
@@ -120,7 +121,6 @@ class FalconAttention(nn.Module):
121
self.total_num_heads * self.head_dim,
122
bias=config.bias,
123
gather_output=False,
123
- perform_initialization=False,
124
skip_bias_add=True,
125
)
126
self.key_value = FalconLinear(self.hidden_size,
@@ -135,7 +135,6 @@ class FalconAttention(nn.Module):
135
self.head_dim,
136
bias=config.bias,
137
gather_output=False,
138
- perform_initialization=False,
138
skip_bias_add=True,
139
)
140
@@ -151,7 +150,6 @@ class FalconAttention(nn.Module):
150
self.hidden_size,
151
bias=config.bias,
152
input_is_parallel=True,
154
- perform_initialization=False,
153
skip_bias_add=True,
154
reduce_results=self.reduce_row_parallel_results)
155
@@ -231,7 +229,6 @@ class FalconMLP(nn.Module):
229
4 * hidden_size,
230
bias=config.bias,
231
gather_output=False,
234
- perform_initialization=False,
232
skip_bias_add=True)
233
self.act = nn.GELU()
234
self.reduce_row_parallel_results = not (config.new_decoder_architecture
@@ -241,7 +238,6 @@ class FalconMLP(nn.Module):
238
hidden_size,
239
bias=config.bias,
240
input_is_parallel=True,
244
- perform_initialization=False,
241
skip_bias_add=True,
242
reduce_results=self.reduce_row_parallel_results)
243
@@ -325,7 +321,7 @@ class FalconDecoderLayer(nn.Module):
321
# only one all-reduce operator to reduce the results from
322
# both MLP and Attention layers.
323
mlp_output += attention_output
328
- mlp_output = reduce_from_tensor_model_parallel_region(mlp_output)
324
+ mlp_output = tensor_model_parallel_all_reduce(mlp_output)
325
if attention_bias is not None:
326
mlp_output += attention_bias
327
if mlp_bias is not None:
@@ -347,7 +343,9 @@ class FalconModel(nn.Module):
343
344
# Embedding + LN Embedding
345
self.word_embeddings = VocabParallelEmbedding(
350
- config.vocab_size, self.embed_dim, perform_initialization=False)
346
+ config.vocab_size,
347
+ self.embed_dim,
348
+ )
349
350
# Transformer blocks
351
self.h = nn.ModuleList([
@@ -389,11 +387,12 @@ class FalconForCausalLM(nn.Module):
387
super().__init__()
388
self.config = config
389
self.transformer = FalconModel(config)
392
- self.lm_head = ColumnParallelLinear(config.hidden_size,
393
- config.vocab_size,
394
- bias=False,
395
- gather_output=False,
396
- perform_initialization=False)
390
+ self.lm_head = ColumnParallelLinear(
391
+ config.hidden_size,
392
+ config.vocab_size,
393
+ bias=False,
394
+ gather_output=False,
395
+ )
396
self.sampler = Sampler(config.vocab_size)
397
398
def forward(
vllm/model_executor/models/gpt2.py
+27
-22
@@ -36,8 +36,9 @@ from vllm.model_executor.weight_utils import (
36
load_padded_tensor_parallel_vocab, load_tensor_parallel_weights)
37
from vllm.model_executor.parallel_utils.parallel_state import (
38
get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
39
-from vllm.model_executor.parallel_utils.tensor_parallel import (
40
- VocabParallelEmbedding, ColumnParallelLinear, RowParallelLinear)
39
+from vllm.model_executor.parallel_utils.layers import (VocabParallelEmbedding,
40
+ ColumnParallelLinear,
41
+ RowParallelLinear)
42
from vllm.sequence import SamplerOutput
43
44
KVCache = Tuple[torch.Tensor, torch.Tensor]
@@ -56,16 +57,18 @@ class GPT2Attention(nn.Module):
57
self.head_dim = self.hidden_size // total_num_heads
58
self.scale = self.head_dim**-0.5
59
59
- self.c_attn = ColumnParallelLinear(self.hidden_size,
60
- 3 * self.hidden_size,
61
- bias=True,
62
- gather_output=False,
63
- perform_initialization=False)
64
- self.c_proj = RowParallelLinear(self.hidden_size,
65
- self.hidden_size,
66
- bias=True,
67
- input_is_parallel=True,
68
- perform_initialization=False)
60
+ self.c_attn = ColumnParallelLinear(
61
+ self.hidden_size,
62
+ 3 * self.hidden_size,
63
+ bias=True,
64
+ gather_output=False,
65
+ )
66
+ self.c_proj = RowParallelLinear(
67
+ self.hidden_size,
68
+ self.hidden_size,
69
+ bias=True,
70
+ input_is_parallel=True,
71
+ )
72
self.attn = PagedAttention(self.num_heads,
73
self.head_dim,
74
scale=self.scale)
@@ -95,16 +98,18 @@ class GPT2MLP(nn.Module):
98
):
99
super().__init__()
100
hidden_size = config.hidden_size
98
- self.c_fc = ColumnParallelLinear(hidden_size,
99
- intermediate_size,
100
- bias=True,
101
- gather_output=False,
102
- perform_initialization=False)
103
- self.c_proj = RowParallelLinear(intermediate_size,
104
- hidden_size,
105
- bias=True,
106
- input_is_parallel=True,
107
- perform_initialization=False)
101
+ self.c_fc = ColumnParallelLinear(
102
+ hidden_size,
103
+ intermediate_size,
104
+ bias=True,
105
+ gather_output=False,
106
+ )
107
+ self.c_proj = RowParallelLinear(
108
+ intermediate_size,
109
+ hidden_size,
110
+ bias=True,
111
+ input_is_parallel=True,
112
+ )
113
self.act = get_act_fn(config.activation_function)
114
115
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
vllm/model_executor/models/gpt_bigcode.py
+34
-29
@@ -37,8 +37,9 @@ from vllm.model_executor.weight_utils import (
37
load_padded_tensor_parallel_vocab, load_tensor_parallel_weights)
38
from vllm.model_executor.parallel_utils.parallel_state import (
39
get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
40
-from vllm.model_executor.parallel_utils.tensor_parallel import (
41
- VocabParallelEmbedding, ColumnParallelLinear, RowParallelLinear)
40
+from vllm.model_executor.parallel_utils.layers import (VocabParallelEmbedding,
41
+ ColumnParallelLinear,
42
+ RowParallelLinear)
43
from vllm.sequence import SamplerOutput
44
45
KVCache = Tuple[torch.Tensor, torch.Tensor]
@@ -62,29 +63,31 @@ class GPTBigCodeAttention(nn.Module):
63
if self.multi_query:
64
self.num_kv_heads = 1
65
self.kv_dim = self.head_dim
65
- self.c_attn_q = ColumnParallelLinear(self.hidden_size,
66
- self.hidden_size,
67
- bias=True,
68
- gather_output=False,
69
- perform_initialization=False)
66
+ self.c_attn_q = ColumnParallelLinear(
67
+ self.hidden_size,
68
+ self.hidden_size,
69
+ bias=True,
70
+ gather_output=False,
71
+ )
72
self.c_attn_kv = nn.Linear(self.hidden_size,
73
2 * self.kv_dim,
74
bias=True)
75
else:
76
self.num_kv_heads = self.num_heads
77
self.kv_dim = self.num_kv_heads * self.head_dim
76
- self.c_attn = ColumnParallelLinear(self.hidden_size,
77
- self.hidden_size +
78
- 2 * self.kv_dim,
79
- bias=True,
80
- gather_output=False,
81
- perform_initialization=False)
82
-
83
- self.c_proj = RowParallelLinear(self.hidden_size,
84
- self.hidden_size,
85
- bias=True,
86
- input_is_parallel=True,
87
- perform_initialization=False)
78
+ self.c_attn = ColumnParallelLinear(
79
+ self.hidden_size,
80
+ self.hidden_size + 2 * self.kv_dim,
81
+ bias=True,
82
+ gather_output=False,
83
+ )
84
+
85
+ self.c_proj = RowParallelLinear(
86
+ self.hidden_size,
87
+ self.hidden_size,
88
+ bias=True,
89
+ input_is_parallel=True,
90
+ )
91
self.attn = PagedAttention(self.num_heads,
92
self.head_dim,
93
scale=self.scale,
@@ -124,16 +127,18 @@ class GPTBigMLP(nn.Module):
127
):
128
super().__init__()
129
hidden_size = config.hidden_size
127
- self.c_fc = ColumnParallelLinear(hidden_size,
128
- intermediate_size,
129
- bias=True,
130
- gather_output=False,
131
- perform_initialization=False)
132
- self.c_proj = RowParallelLinear(intermediate_size,
133
- hidden_size,
134
- bias=True,
135
- input_is_parallel=True,
136
- perform_initialization=False)
130
+ self.c_fc = ColumnParallelLinear(
131
+ hidden_size,
132
+ intermediate_size,
133
+ bias=True,
134
+ gather_output=False,
135
+ )
136
+ self.c_proj = RowParallelLinear(
137
+ intermediate_size,
138
+ hidden_size,
139
+ bias=True,
140
+ input_is_parallel=True,
141
+ )
142
self.act = get_act_fn(config.activation_function)
143
144
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
vllm/model_executor/models/gpt_j.py
+34
-27
@@ -34,8 +34,9 @@ from vllm.model_executor.weight_utils import (hf_model_weights_iterator,
34
load_tensor_parallel_weights)
35
from vllm.model_executor.parallel_utils.parallel_state import (
36
get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
37
-from vllm.model_executor.parallel_utils.tensor_parallel import (
38
- VocabParallelEmbedding, ColumnParallelLinear, RowParallelLinear)
37
+from vllm.model_executor.parallel_utils.layers import (VocabParallelEmbedding,
38
+ ColumnParallelLinear,
39
+ RowParallelLinear)
40
from vllm.sequence import SamplerOutput
41
42
KVCache = Tuple[torch.Tensor, torch.Tensor]
@@ -49,16 +50,18 @@ class GPTJAttention(nn.Module):
50
self.hidden_size = config.hidden_size
51
self.head_size = self.hidden_size // self.total_num_heads
52
52
- self.qkv_proj = ColumnParallelLinear(config.hidden_size,
53
- 3 * config.hidden_size,
54
- bias=False,
55
- gather_output=False,
56
- perform_initialization=False)
57
- self.out_proj = RowParallelLinear(config.hidden_size,
58
- config.hidden_size,
59
- bias=False,
60
- input_is_parallel=True,
61
- perform_initialization=False)
53
+ self.qkv_proj = ColumnParallelLinear(
54
+ config.hidden_size,
55
+ 3 * config.hidden_size,
56
+ bias=False,
57
+ gather_output=False,
58
+ )
59
+ self.out_proj = RowParallelLinear(
60
+ config.hidden_size,
61
+ config.hidden_size,
62
+ bias=False,
63
+ input_is_parallel=True,
64
+ )
65
66
tp_world_size = get_tensor_model_parallel_world_size()
67
assert self.total_num_heads % tp_world_size == 0
@@ -102,14 +105,16 @@ class GPTJMLP(nn.Module):
105
def __init__(self, intermediate_size: int, config: GPTJConfig):
106
super().__init__()
107
hidden_size = config.n_embd
105
- self.fc_in = ColumnParallelLinear(hidden_size,
106
- intermediate_size,
107
- gather_output=False,
108
- perform_initialization=False)
109
- self.fc_out = RowParallelLinear(intermediate_size,
110
- hidden_size,
111
- input_is_parallel=True,
112
- perform_initialization=False)
108
+ self.fc_in = ColumnParallelLinear(
109
+ hidden_size,
110
+ intermediate_size,
111
+ gather_output=False,
112
+ )
113
+ self.fc_out = RowParallelLinear(
114
+ intermediate_size,
115
+ hidden_size,
116
+ input_is_parallel=True,
117
+ )
118
self.act = get_act_fn(config.activation_function)
119
120
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
@@ -159,9 +164,10 @@ class GPTJModel(nn.Module):
164
super().__init__()
165
self.config = config
166
self.embed_dim = config.n_embd
162
- self.wte = VocabParallelEmbedding(config.vocab_size,
163
- self.embed_dim,
164
- perform_initialization=False)
167
+ self.wte = VocabParallelEmbedding(
168
+ config.vocab_size,
169
+ self.embed_dim,
170
+ )
171
self.h = nn.ModuleList(
172
[GPTJBlock(config) for _ in range(config.n_layer)])
173
self.ln_f = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_epsilon)
@@ -199,10 +205,11 @@ class GPTJForCausalLM(nn.Module):
205
self.config = config
206
assert not config.tie_word_embeddings
207
self.transformer = GPTJModel(config)
202
- self.lm_head = ColumnParallelLinear(config.n_embd,
203
- config.vocab_size,
204
- gather_output=False,
205
- perform_initialization=False)
208
+ self.lm_head = ColumnParallelLinear(
209
+ config.n_embd,
210
+ config.vocab_size,
211
+ gather_output=False,
212
+ )
213
self.sampler = Sampler(config.vocab_size)
214
215
def forward(
vllm/model_executor/models/gpt_neox.py
+29
-23
@@ -34,8 +34,9 @@ from vllm.model_executor.weight_utils import (hf_model_weights_iterator,
34
load_tensor_parallel_weights)
35
from vllm.model_executor.parallel_utils.parallel_state import (
36
get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
37
-from vllm.model_executor.parallel_utils.tensor_parallel import (
38
- VocabParallelEmbedding, ColumnParallelLinear, RowParallelLinear)
37
+from vllm.model_executor.parallel_utils.layers import (VocabParallelEmbedding,
38
+ ColumnParallelLinear,
39
+ RowParallelLinear)
40
from vllm.sequence import SamplerOutput
41
42
KVCache = Tuple[torch.Tensor, torch.Tensor]
@@ -59,11 +60,12 @@ class GPTNeoXAttention(nn.Module):
60
config.hidden_size,
61
3 * config.hidden_size,
62
gather_output=False,
62
- perform_initialization=False)
63
- self.dense = RowParallelLinear(config.hidden_size,
64
- config.hidden_size,
65
- input_is_parallel=True,
66
- perform_initialization=False)
63
+ )
64
+ self.dense = RowParallelLinear(
65
+ config.hidden_size,
66
+ config.hidden_size,
67
+ input_is_parallel=True,
68
+ )
69
70
scaling = self.head_size**-0.5
71
rotary_dim = int(self.head_size * config.rotary_pct)
@@ -100,14 +102,16 @@ class GPTNeoXMLP(nn.Module):
102
103
def __init__(self, config: GPTNeoXConfig):
104
super().__init__()
103
- self.dense_h_to_4h = ColumnParallelLinear(config.hidden_size,
104
- config.intermediate_size,
105
- gather_output=False,
106
- perform_initialization=False)
107
- self.dense_4h_to_h = RowParallelLinear(config.intermediate_size,
108
- config.hidden_size,
109
- input_is_parallel=True,
110
- perform_initialization=False)
105
+ self.dense_h_to_4h = ColumnParallelLinear(
106
+ config.hidden_size,
107
+ config.intermediate_size,
108
+ gather_output=False,
109
+ )
110
+ self.dense_4h_to_h = RowParallelLinear(
111
+ config.intermediate_size,
112
+ config.hidden_size,
113
+ input_is_parallel=True,
114
+ )
115
self.act = get_act_fn(config.hidden_act)
116
117
def forward(self, hidden_states):
@@ -169,9 +173,10 @@ class GPTNeoXModel(nn.Module):
173
super().__init__()
174
self.config = config
175
172
- self.embed_in = VocabParallelEmbedding(config.vocab_size,
173
- config.hidden_size,
174
- perform_initialization=False)
176
+ self.embed_in = VocabParallelEmbedding(
177
+ config.vocab_size,
178
+ config.hidden_size,
179
+ )
180
self.layers = nn.ModuleList(
181
[GPTNeoXLayer(config) for _ in range(config.num_hidden_layers)])
182
self.final_layer_norm = nn.LayerNorm(config.hidden_size,
@@ -209,11 +214,12 @@ class GPTNeoXForCausalLM(nn.Module):
214
super().__init__()
215
self.config = config
216
self.gpt_neox = GPTNeoXModel(config)
212
- self.embed_out = ColumnParallelLinear(config.hidden_size,
213
- config.vocab_size,
214
- bias=False,
215
- gather_output=False,
216
- perform_initialization=False)
217
+ self.embed_out = ColumnParallelLinear(
218
+ config.hidden_size,
219
+ config.vocab_size,
220
+ bias=False,
221
+ gather_output=False,
222
+ )
223
self.sampler = Sampler(config.vocab_size)
224
225
def forward(
vllm/model_executor/models/internlm.py
+24
-20
@@ -12,8 +12,9 @@ from vllm.model_executor.layers.layernorm import RMSNorm
12
from vllm.model_executor.layers.sampler import Sampler
13
from vllm.model_executor.parallel_utils.parallel_state import (
14
get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
15
-from vllm.model_executor.parallel_utils.tensor_parallel import (
16
- ColumnParallelLinear, RowParallelLinear, VocabParallelEmbedding)
15
+from vllm.model_executor.parallel_utils.layers import (ColumnParallelLinear,
16
+ RowParallelLinear,
17
+ VocabParallelEmbedding)
18
from vllm.model_executor.weight_utils import (
19
hf_model_weights_iterator, load_padded_tensor_parallel_vocab,
20
load_tensor_parallel_weights)
@@ -31,16 +32,18 @@ class InternLMMLP(nn.Module):
32
hidden_act: str,
33
):
34
super().__init__()
34
- self.gate_up_proj = ColumnParallelLinear(hidden_size,
35
- 2 * intermediate_size,
36
- bias=False,
37
- gather_output=False,
38
- perform_initialization=False)
39
- self.down_proj = RowParallelLinear(intermediate_size,
40
- hidden_size,
41
- bias=False,
42
- input_is_parallel=True,
43
- perform_initialization=False)
35
+ self.gate_up_proj = ColumnParallelLinear(
36
+ hidden_size,
37
+ 2 * intermediate_size,
38
+ bias=False,
39
+ gather_output=False,
40
+ )
41
+ self.down_proj = RowParallelLinear(
42
+ intermediate_size,
43
+ hidden_size,
44
+ bias=False,
45
+ input_is_parallel=True,
46
+ )
47
if hidden_act != "silu":
48
raise ValueError(f"Unsupported activation: {hidden_act}. "
49
"Only silu is supported for now.")
@@ -80,14 +83,12 @@ class InternLMAttention(nn.Module):
83
3 * self.total_num_heads * self.head_dim,
84
bias=True,
85
gather_output=False,
83
- perform_initialization=False,
86
)
87
self.o_proj = RowParallelLinear(
88
self.total_num_heads * self.head_dim,
89
hidden_size,
90
bias=True,
91
input_is_parallel=True,
90
- perform_initialization=False,
92
)
93
self.attn = PagedAttentionWithRoPE(
94
self.num_heads,
@@ -176,7 +177,9 @@ class InternLMModel(nn.Module):
177
178
vocab_size = ((config.vocab_size + 63) // 64) * 64
179
self.embed_tokens = VocabParallelEmbedding(
179
- vocab_size, config.hidden_size, perform_initialization=False)
180
+ vocab_size,
181
+ config.hidden_size,
182
+ )
183
self.layers = nn.ModuleList([
184
InternLMDecoderLayer(config)
185
for _ in range(config.num_hidden_layers)
@@ -216,11 +219,12 @@ class InternLMForCausalLM(nn.Module):
219
self.config = config
220
self.model = InternLMModel(config)
221
vocab_size = ((config.vocab_size + 63) // 64) * 64
219
- self.lm_head = ColumnParallelLinear(config.hidden_size,
220
- vocab_size,
221
- bias=False,
222
- gather_output=False,
223
- perform_initialization=False)
222
+ self.lm_head = ColumnParallelLinear(
223
+ config.hidden_size,
224
+ vocab_size,
225
+ bias=False,
226
+ gather_output=False,
227
+ )
228
self.sampler = Sampler(config.vocab_size)
229
230
def forward(
vllm/model_executor/models/llama.py
+4
-8
@@ -39,8 +39,7 @@ from vllm.model_executor.layers.sampler import Sampler
39
from vllm.model_executor.layers.quantized_linear import ParallelLinear
40
from vllm.model_executor.parallel_utils.parallel_state import (
41
get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
42
-from vllm.model_executor.parallel_utils.tensor_parallel import (
43
- VocabParallelEmbedding)
42
+from vllm.model_executor.parallel_utils.layers import VocabParallelEmbedding
43
from vllm.model_executor.quantization_utils import QuantizationConfig
44
from vllm.model_executor.weight_utils import (
45
convert_pyslice_to_tensor, hf_model_weights_iterator,
@@ -64,13 +63,11 @@ class LlamaMLP(nn.Module):
63
2 * intermediate_size,
64
bias=False,
65
gather_output=False,
67
- perform_initialization=False,
66
quant_config=quant_config)
67
self.down_proj = ParallelLinear.row(intermediate_size,
68
hidden_size,
69
bias=False,
70
input_is_parallel=True,
73
- perform_initialization=False,
71
quant_config=quant_config)
72
if hidden_act != "silu":
73
raise ValueError(f"Unsupported activation: {hidden_act}. "
@@ -127,7 +124,6 @@ class LlamaAttention(nn.Module):
124
self.head_dim,
125
bias=False,
126
gather_output=False,
130
- perform_initialization=False,
127
quant_config=quant_config,
128
)
129
self.o_proj = ParallelLinear.row(
@@ -135,7 +131,6 @@ class LlamaAttention(nn.Module):
131
hidden_size,
132
bias=False,
133
input_is_parallel=True,
138
- perform_initialization=False,
134
quant_config=quant_config,
135
)
136
self.attn = PagedAttentionWithRoPE(
@@ -241,7 +236,9 @@ class LlamaModel(nn.Module):
236
237
vocab_size = ((config.vocab_size + 63) // 64) * 64
238
self.embed_tokens = VocabParallelEmbedding(
244
- vocab_size, config.hidden_size, perform_initialization=False)
239
+ vocab_size,
240
+ config.hidden_size,
241
+ )
242
self.layers = nn.ModuleList([
243
LlamaDecoderLayer(config, quant_config)
244
for _ in range(config.num_hidden_layers)
@@ -291,7 +288,6 @@ class LlamaForCausalLM(nn.Module):
288
vocab_size,
289
bias=False,
290
gather_output=False,
294
- perform_initialization=False,
291
quant_config=None)
292
self.sampler = Sampler(config.vocab_size)
293
vllm/model_executor/models/mistral.py
+4
-8
@@ -38,8 +38,7 @@ from vllm.model_executor.layers.sampler import Sampler
38
from vllm.model_executor.layers.quantized_linear import ParallelLinear
39
from vllm.model_executor.parallel_utils.parallel_state import (
40
get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
41
-from vllm.model_executor.parallel_utils.tensor_parallel import (
42
- VocabParallelEmbedding)
41
+from vllm.model_executor.parallel_utils.layers import VocabParallelEmbedding
42
from vllm.model_executor.quantization_utils import QuantizationConfig
43
from vllm.model_executor.weight_utils import (
44
convert_pyslice_to_tensor, hf_model_weights_iterator,
@@ -64,13 +63,11 @@ class MistralMLP(nn.Module):
63
2 * intermediate_size,
64
bias=False,
65
gather_output=False,
67
- perform_initialization=False,
66
quant_config=quant_config)
67
self.down_proj = ParallelLinear.row(intermediate_size,
68
hidden_size,
69
bias=False,
70
input_is_parallel=True,
73
- perform_initialization=False,
71
quant_config=quant_config)
72
if hidden_act != "silu":
73
raise ValueError(f"Unsupported activation: {hidden_act}. "
@@ -116,7 +113,6 @@ class MistralAttention(nn.Module):
113
self.head_dim,
114
bias=False,
115
gather_output=False,
119
- perform_initialization=False,
116
quant_config=quant_config,
117
)
118
self.o_proj = ParallelLinear.row(
@@ -124,7 +120,6 @@ class MistralAttention(nn.Module):
120
hidden_size,
121
bias=False,
122
input_is_parallel=True,
127
- perform_initialization=False,
123
quant_config=quant_config,
124
)
125
self.attn = PagedAttentionWithRoPE(self.num_heads,
@@ -225,7 +220,9 @@ class MistralModel(nn.Module):
220
221
vocab_size = ((config.vocab_size + 63) // 64) * 64
222
self.embed_tokens = VocabParallelEmbedding(
228
- vocab_size, config.hidden_size, perform_initialization=False)
223
+ vocab_size,
224
+ config.hidden_size,
225
+ )
226
self.layers = nn.ModuleList([
227
MistralDecoderLayer(config, quant_config)
228
for _ in range(config.num_hidden_layers)
@@ -275,7 +272,6 @@ class MistralForCausalLM(nn.Module):
272
vocab_size,
273
bias=False,
274
gather_output=False,
278
- perform_initialization=False,
275
quant_config=None)
276
self.sampler = Sampler(config.vocab_size)
277
vllm/model_executor/models/mpt.py
+19
-17
@@ -15,8 +15,9 @@ from vllm.model_executor.weight_utils import (convert_pyslice_to_tensor,
15
load_tensor_parallel_weights)
16
from vllm.model_executor.parallel_utils.parallel_state import (
17
get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
18
-from vllm.model_executor.parallel_utils.tensor_parallel import (
19
- VocabParallelEmbedding, ColumnParallelLinear, RowParallelLinear)
18
+from vllm.model_executor.parallel_utils.layers import (VocabParallelEmbedding,
19
+ ColumnParallelLinear,
20
+ RowParallelLinear)
21
from vllm.sequence import SamplerOutput
22
from vllm.transformers_utils.configs.mpt import MPTConfig
23
@@ -53,7 +54,6 @@ class MPTAttention(nn.Module):
54
3 * self.d_model,
55
bias=not config.no_bias,
56
gather_output=False,
56
- perform_initialization=False,
57
)
58
if self.qk_ln:
59
self.q_ln = nn.LayerNorm(self.d_model)
@@ -63,7 +63,6 @@ class MPTAttention(nn.Module):
63
self.d_model,
64
bias=not config.no_bias,
65
input_is_parallel=True,
66
- perform_initialization=False,
66
)
67
68
tp_world_size = get_tensor_model_parallel_world_size()
@@ -113,17 +112,19 @@ class MPTMLP(nn.Module):
112
hidden_size = config.d_model
113
expansion_ratio = config.expansion_ratio
114
intermediate_size = expansion_ratio * hidden_size
116
- self.up_proj = ColumnParallelLinear(hidden_size,
117
- intermediate_size,
118
- bias=not config.no_bias,
119
- gather_output=False,
120
- perform_initialization=False)
115
+ self.up_proj = ColumnParallelLinear(
116
+ hidden_size,
117
+ intermediate_size,
118
+ bias=not config.no_bias,
119
+ gather_output=False,
120
+ )
121
self.act = get_act_fn("gelu")
122
- self.down_proj = RowParallelLinear(intermediate_size,
123
- hidden_size,
124
- bias=not config.no_bias,
125
- input_is_parallel=True,
126
- perform_initialization=False)
122
+ self.down_proj = RowParallelLinear(
123
+ intermediate_size,
124
+ hidden_size,
125
+ bias=not config.no_bias,
126
+ input_is_parallel=True,
127
+ )
128
129
def forward(self, x: torch.Tensor) -> torch.Tensor:
130
x, _ = self.up_proj(x)
@@ -172,9 +173,10 @@ class MPTModel(nn.Module):
173
assert config.embedding_fraction == 1.0
174
assert config.norm_type == "low_precision_layernorm"
175
175
- self.wte = VocabParallelEmbedding(config.vocab_size,
176
- config.d_model,
177
- perform_initialization=False)
176
+ self.wte = VocabParallelEmbedding(
177
+ config.vocab_size,
178
+ config.d_model,
179
+ )
180
self.blocks = nn.ModuleList(
181
[MPTBlock(config) for _ in range(config.n_layers)])
182
self.norm_f = nn.LayerNorm(config.d_model)
vllm/model_executor/models/opt.py
+28
-23
@@ -35,8 +35,9 @@ from vllm.model_executor.weight_utils import (hf_model_weights_iterator,
35
load_tensor_parallel_weights)
36
from vllm.model_executor.parallel_utils.parallel_state import (
37
get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
38
-from vllm.model_executor.parallel_utils.tensor_parallel import (
39
- VocabParallelEmbedding, ColumnParallelLinear, RowParallelLinear)
38
+from vllm.model_executor.parallel_utils.layers import (VocabParallelEmbedding,
39
+ ColumnParallelLinear,
40
+ RowParallelLinear)
41
from vllm.sequence import SamplerOutput
42
43
KVCache = Tuple[torch.Tensor, torch.Tensor]
@@ -73,16 +74,18 @@ class OPTAttention(nn.Module):
74
self.head_dim = embed_dim // total_num_heads
75
self.scaling = self.head_dim**-0.5
76
76
- self.qkv_proj = ColumnParallelLinear(embed_dim,
77
- 3 * embed_dim,
78
- bias=bias,
79
- gather_output=False,
80
- perform_initialization=False)
81
- self.out_proj = RowParallelLinear(embed_dim,
82
- embed_dim,
83
- bias=bias,
84
- input_is_parallel=True,
85
- perform_initialization=False)
77
+ self.qkv_proj = ColumnParallelLinear(
78
+ embed_dim,
79
+ 3 * embed_dim,
80
+ bias=bias,
81
+ gather_output=False,
82
+ )
83
+ self.out_proj = RowParallelLinear(
84
+ embed_dim,
85
+ embed_dim,
86
+ bias=bias,
87
+ input_is_parallel=True,
88
+ )
89
self.attn = PagedAttention(self.num_heads,
90
self.head_dim,
91
scale=self.scaling)
@@ -120,16 +123,18 @@ class OPTDecoderLayer(nn.Module):
123
self.self_attn_layer_norm = nn.LayerNorm(
124
self.embed_dim,
125
elementwise_affine=config.layer_norm_elementwise_affine)
123
- self.fc1 = ColumnParallelLinear(self.embed_dim,
124
- config.ffn_dim,
125
- bias=config.enable_bias,
126
- gather_output=False,
127
- perform_initialization=False)
128
- self.fc2 = RowParallelLinear(config.ffn_dim,
129
- self.embed_dim,
130
- bias=config.enable_bias,
131
- input_is_parallel=True,
132
- perform_initialization=False)
126
+ self.fc1 = ColumnParallelLinear(
127
+ self.embed_dim,
128
+ config.ffn_dim,
129
+ bias=config.enable_bias,
130
+ gather_output=False,
131
+ )
132
+ self.fc2 = RowParallelLinear(
133
+ config.ffn_dim,
134
+ self.embed_dim,
135
+ bias=config.enable_bias,
136
+ input_is_parallel=True,
137
+ )
138
self.final_layer_norm = nn.LayerNorm(
139
self.embed_dim,
140
elementwise_affine=config.layer_norm_elementwise_affine)
@@ -182,7 +187,7 @@ class OPTDecoder(nn.Module):
187
self.embed_tokens = VocabParallelEmbedding(
188
config.vocab_size,
189
config.word_embed_proj_dim,
185
- perform_initialization=False)
190
+ )
191
# Positional embeddings are replicated (not sharded).
192
self.embed_positions = OPTLearnedPositionalEmbedding(
193
config.max_position_embeddings, config.hidden_size)
vllm/model_executor/models/qwen.py
+5
-9
@@ -28,7 +28,7 @@ from vllm.model_executor.parallel_utils.parallel_state import (
28
get_tensor_model_parallel_rank,
29
get_tensor_model_parallel_world_size,
30
)
31
-from vllm.model_executor.parallel_utils.tensor_parallel import (
31
+from vllm.model_executor.parallel_utils.layers import (
32
VocabParallelEmbedding,
33
ColumnParallelLinear,
34
RowParallelLinear,
@@ -53,14 +53,12 @@ class QWenMLP(nn.Module):
53
2 * intermediate_size,
54
bias=False,
55
gather_output=False,
56
- perform_initialization=False,
56
)
57
self.c_proj = RowParallelLinear(
58
intermediate_size,
59
hidden_size,
60
bias=False,
61
input_is_parallel=True,
63
- perform_initialization=False,
62
)
63
if hidden_act != "silu":
64
raise ValueError(f"Unsupported activation: {hidden_act}. "
@@ -98,14 +96,12 @@ class QWenAttention(nn.Module):
96
3 * hidden_size,
97
bias=True,
98
gather_output=False,
101
- perform_initialization=False,
99
)
100
self.c_proj = RowParallelLinear(
101
self.total_num_heads * self.head_dim,
102
hidden_size,
103
bias=False,
104
input_is_parallel=True,
108
- perform_initialization=False,
105
)
106
self.scaling = self.head_dim**-0.5
107
self.attn = PagedAttentionWithRoPE(
@@ -190,9 +186,10 @@ class QWenModel(nn.Module):
186
self.vocab_size = config.vocab_size
187
188
vocab_size = ((config.vocab_size + 63) // 64) * 64
193
- self.wte = VocabParallelEmbedding(vocab_size,
194
- config.hidden_size,
195
- perform_initialization=False)
189
+ self.wte = VocabParallelEmbedding(
190
+ vocab_size,
191
+ config.hidden_size,
192
+ )
193
self.h = nn.ModuleList(
194
[QWenBlock(config) for _ in range(config.num_hidden_layers)])
195
self.ln_f = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon)
@@ -235,7 +232,6 @@ class QWenLMHeadModel(nn.Module):
232
vocab_size,
233
bias=False,
234
gather_output=False,
238
- perform_initialization=False,
235
)
236
self.sampler = Sampler(config.vocab_size)
237
vllm/model_executor/parallel_utils/__init__.py
-7
@@ -1,7 +0,0 @@
1
-import vllm.model_executor.parallel_utils.parallel_state
2
-import vllm.model_executor.parallel_utils.tensor_parallel
3
-
4
-__all__ = [
5
- "parallel_state",
6
- "tensor_parallel",
7
-]
vllm/model_executor/parallel_utils/communication_op.py
new
+47
@@ -0,0 +1,47 @@
1
+import torch
2
+
3
+from vllm.model_executor.parallel_utils.parallel_state import (
4
+ get_tensor_model_parallel_world_size,
5
+ get_tensor_model_parallel_group,
6
+)
7
+
8
+
9
+def tensor_model_parallel_all_reduce(input_):
10
+ """All-reduce the input tensor across model parallel group.
11
+
12
+ Note: This operation is applied in-place on the input tensor.
13
+ """
14
+ # Bypass the function if we are using only 1 GPU.
15
+ if get_tensor_model_parallel_world_size() == 1:
16
+ return input_
17
+ # All-reduce.
18
+ torch.distributed.all_reduce(input_,
19
+ group=get_tensor_model_parallel_group())
20
+ return input_
21
+
22
+
23
+def tensor_model_parallel_all_gather(input_, dim=-1):
24
+ """All-gather the input tensor across model parallel group."""
25
+ world_size = get_tensor_model_parallel_world_size()
26
+ # Bypass the function if we are using only 1 GPU.
27
+ if world_size == 1:
28
+ return input_
29
+ assert -input_.dim() <= dim < input_.dim(), (
30
+ f"Invalid dim ({dim}) for input tensor with shape {input_.size()}")
31
+ if dim < 0:
32
+ # Convert negative dim to positive.
33
+ dim += input_.dim()
34
+ input_size = input_.size()
35
+ # Allocate output tensor.
36
+ output_tensor = torch.empty((world_size, ) + input_size,
37
+ dtype=input_.dtype,
38
+ device=input_.device)
39
+ # All-gather.
40
+ torch.distributed.all_gather_into_tensor(
41
+ output_tensor, input_, group=get_tensor_model_parallel_group())
42
+ # Reshape
43
+ output_tensor = output_tensor.movedim(0, dim)
44
+ output_tensor = output_tensor.reshape(input_size[:dim] +
45
+ (world_size * input_size[dim], ) +
46
+ input_size[dim + 1:])
47
+ return output_tensor
vllm/model_executor/parallel_utils/layers.py
new
+303
@@ -0,0 +1,303 @@
1
+# Copyright 2023 The vLLM team.
2
+# Adapted from
3
+# https://github.com/NVIDIA/Megatron-LM/blob/main/megatron/core/tensor_parallel/layers.py
4
+# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
5
+
6
+# Parts of the code here are adapted from PyTorch
7
+# repo: https://github.com/pytorch/pytorch
8
+from typing import Optional
9
+
10
+import torch
11
+import torch.nn.functional as F
12
+from torch.nn.parameter import Parameter
13
+
14
+from vllm.model_executor.parallel_utils.parallel_state import (
15
+ get_tensor_model_parallel_rank,
16
+ get_tensor_model_parallel_world_size,
17
+)
18
+from vllm.model_executor.quantization_utils import QuantizationConfig
19
+from vllm.model_executor.parallel_utils.communication_op import (
20
+ tensor_model_parallel_all_reduce, tensor_model_parallel_all_gather)
21
+
22
+from vllm.model_executor.parallel_utils.utils import (
23
+ divide,
24
+ VocabUtility,
25
+ split_tensor_along_last_dim,
26
+)
27
+
28
+
29
+class VocabParallelEmbedding(torch.nn.Module):
30
+ """Embedding parallelized in the vocabulary dimension.
31
+
32
+ This is mainly adapted from torch.nn.Embedding and all the default
33
+ values are kept.
34
+ Arguments:
35
+ num_embeddings: vocabulary size.
36
+ embedding_dim: size of hidden state.
37
+ params_dtype: type of the parameters.
38
+ """
39
+
40
+ def __init__(self,
41
+ num_embeddings: int,
42
+ embedding_dim: int,
43
+ params_dtype: Optional[torch.dtype] = None):
44
+ super().__init__()
45
+
46
+ # Keep the input dimensions.
47
+ self.num_embeddings = num_embeddings
48
+ self.embedding_dim = embedding_dim
49
+ if params_dtype is None:
50
+ params_dtype = torch.get_default_dtype()
51
+
52
+ self.tp_size = get_tensor_model_parallel_world_size()
53
+ # TODO: Handle vocab padding here.
54
+ # Divide the weight matrix along the vocaburaly dimension.
55
+ self.vocab_start_index, self.vocab_end_index = (
56
+ VocabUtility.vocab_range_from_global_vocab_size(
57
+ self.num_embeddings, get_tensor_model_parallel_rank(),
58
+ self.tp_size))
59
+ self.num_embeddings_per_partition = (self.vocab_end_index -
60
+ self.vocab_start_index)
61
+
62
+ self.weight = Parameter(
63
+ torch.empty(self.num_embeddings_per_partition,
64
+ self.embedding_dim,
65
+ device=torch.cuda.current_device(),
66
+ dtype=params_dtype))
67
+
68
+ def forward(self, input_):
69
+ if self.tp_size > 1:
70
+ # Build the mask.
71
+ input_mask = ((input_ < self.vocab_start_index) |
72
+ (input_ >= self.vocab_end_index))
73
+ # Mask the input.
74
+ masked_input = input_.clone() - self.vocab_start_index
75
+ masked_input[input_mask] = 0
76
+ else:
77
+ masked_input = input_
78
+ # Get the embeddings.
79
+ output_parallel = F.embedding(masked_input, self.weight)
80
+ # Mask the output embedding.
81
+ if self.tp_size > 1:
82
+ output_parallel[input_mask, :] = 0.0
83
+ # Reduce across all the model parallel GPUs.
84
+ output = tensor_model_parallel_all_reduce(output_parallel)
85
+ return output
86
+
87
+
88
+class ColumnParallelLinear(torch.nn.Module):
89
+ """Linear layer with column parallelism.
90
+
91
+ The linear layer is defined as Y = XA + b. A is parallelized along
92
+ its second dimension as A = [A_1, ..., A_p].
93
+
94
+ Arguments:
95
+ input_size: first dimension of matrix A.
96
+ output_size: second dimension of matrix A.
97
+
98
+ Keyword Arguments
99
+ bias: If true, add bias
100
+ gather_output: If true, call all-gather on output and make Y available
101
+ to all GPUs, otherwise, every GPU will have its output
102
+ which is Y_i = XA_i
103
+ skip_bias_add: This was added to enable performance optimizations where
104
+ bias can be fused with other element-wise operations. we
105
+ skip adding bias but instead return it.
106
+ params_dtype: Data type for the parameters.
107
+ quant_config: Quantization configuration.
108
+ """
109
+
110
+ def __init__(
111
+ self,
112
+ input_size: int,
113
+ output_size: int,
114
+ bias: bool = True,
115
+ gather_output: bool = True,
116
+ skip_bias_add: bool = False,
117
+ params_dtype: Optional[torch.dtype] = None,
118
+ quant_config: Optional[QuantizationConfig] = None,
119
+ ):
120
+ super().__init__()
121
+
122
+ # Keep input parameters
123
+ self.input_size = input_size
124
+ self.output_size = output_size
125
+ self.gather_output = gather_output
126
+ # Divide the weight matrix along the last dimension.
127
+ self.tp_size = get_tensor_model_parallel_world_size()
128
+ self.output_size_per_partition = divide(output_size, self.tp_size)
129
+ self.skip_bias_add = skip_bias_add
130
+ self.quant_config = quant_config
131
+
132
+ if params_dtype is None:
133
+ params_dtype = torch.get_default_dtype()
134
+
135
+ # Parameters.
136
+ # Note: torch.nn.functional.linear performs XA^T + b and as a result
137
+ # we allocate the transpose.
138
+ self.create_weights(params_dtype)
139
+
140
+ if bias:
141
+ self.bias = Parameter(
142
+ torch.empty(self.output_size_per_partition,
143
+ device=torch.cuda.current_device(),
144
+ dtype=params_dtype))
145
+ else:
146
+ self.register_parameter('bias', None)
147
+
148
+ def create_weights(self, dtype: torch.dtype) -> None:
149
+ self.weight = Parameter(
150
+ torch.empty(self.output_size_per_partition,
151
+ self.input_size,
152
+ device=torch.cuda.current_device(),
153
+ dtype=dtype))
154
+
155
+ def apply_weights(
156
+ self,
157
+ x: torch.Tensor,
158
+ bias: Optional[torch.Tensor],
159
+ ) -> torch.Tensor:
160
+ return F.linear(x, self.weight, bias)
161
+
162
+ def forward(self, input_):
163
+ """Forward of ColumnParallelLinear
164
+
165
+ Args:
166
+ input_: Tensor whose last dimension is `input_size`.
167
+
168
+ Returns:
169
+ - output
170
+ - bias
171
+ """
172
+ bias = self.bias if not self.skip_bias_add else None
173
+
174
+ input_parallel = input_
175
+ # Matrix multiply.
176
+ output_parallel = self.apply_weights(input_parallel, bias)
177
+ if self.gather_output:
178
+ # All-gather across the partitions.
179
+ output = tensor_model_parallel_all_gather(output_parallel)
180
+ else:
181
+ output = output_parallel
182
+ output_bias = self.bias if self.skip_bias_add else None
183
+ return output, output_bias
184
+
185
+
186
+class RowParallelLinear(torch.nn.Module):
187
+ """Linear layer with row parallelism.
188
+
189
+ The linear layer is defined as Y = XA + b. A is parallelized along
190
+ its first dimension and X along its second dimension as:
191
+ - -
192
+ | A_1 |
193
+ | . |
194
+ A = | . | X = [X_1, ..., X_p]
195
+ | . |
196
+ | A_p |
197
+ - -
198
+ Arguments:
199
+ input_size: first dimension of matrix A.
200
+ output_size: second dimension of matrix A.
201
+
202
+ Keyword Arguments:
203
+ bias: If true, add bias. Note that bias is not parallelized.
204
+ input_is_parallel: If true, we assume that the input is already
205
+ split across the GPUs and we do not split
206
+ again.
207
+ skip_bias_add: This was added to enable performance optimization where
208
+ bias can be fused with other element-wise operations.
209
+ We skip adding bias but instead return it.
210
+ params_dtype: Data type for the parameters.
211
+ quant_config: Quantization configuration.
212
+ """
213
+
214
+ def __init__(
215
+ self,
216
+ input_size: int,
217
+ output_size: int,
218
+ bias: bool = True,
219
+ input_is_parallel: bool = False,
220
+ skip_bias_add: bool = False,
221
+ params_dtype: Optional[torch.dtype] = None,
222
+ reduce_results: bool = True,
223
+ quant_config: Optional[QuantizationConfig] = None,
224
+ ):
225
+ super().__init__()
226
+ # Keep input parameters
227
+ self.input_size = input_size
228
+ self.output_size = output_size
229
+ self.input_is_parallel = input_is_parallel
230
+ self.reduce_results = reduce_results
231
+ if params_dtype is None:
232
+ params_dtype = torch.get_default_dtype()
233
+
234
+ # Divide the weight matrix along the last dimension.
235
+ self.tp_size = get_tensor_model_parallel_world_size()
236
+ self.input_size_per_partition = divide(input_size, self.tp_size)
237
+ self.skip_bias_add = skip_bias_add
238
+ self.quant_config = quant_config
239
+
240
+ self.create_weights(params_dtype)
241
+
242
+ if not reduce_results and (bias and not skip_bias_add):
243
+ raise ValueError('When not reduce the results, adding bias to the '
244
+ 'results can lead to incorrect results')
245
+
246
+ if bias:
247
+ self.bias = Parameter(
248
+ torch.empty(self.output_size,
249
+ device=torch.cuda.current_device(),
250
+ dtype=params_dtype))
251
+
252
+ # Always initialize bias to zero.
253
+ with torch.no_grad():
254
+ self.bias.zero_()
255
+ else:
256
+ self.register_parameter('bias', None)
257
+
258
+ def create_weights(self, dtype: torch.dtype) -> None:
259
+ self.weight = Parameter(
260
+ torch.empty(self.output_size,
261
+ self.input_size_per_partition,
262
+ device=torch.cuda.current_device(),
263
+ dtype=dtype))
264
+
265
+ def apply_weights(self, x: torch.Tensor) -> torch.Tensor:
266
+ return F.linear(x, self.weight)
267
+
268
+ def forward(self, input_):
269
+ """Forward of RowParallelLinear
270
+
271
+ Args:
272
+ input_: tensor whose last dimension is `input_size`. If
273
+ `input_is_parallel` is set, then the last dimension
274
+ is `input_size // tp_size`.
275
+
276
+ Returns:
277
+ - output
278
+ - bias
279
+ """
280
+ # Set up backprop all-reduce.
281
+ if self.input_is_parallel:
282
+ input_parallel = input_
283
+ else:
284
+ # TODO: simplify code below
285
+ tp_rank = get_tensor_model_parallel_rank()
286
+ splitted_input = split_tensor_along_last_dim(
287
+ input_, num_partitions=self.tp_size)
288
+ input_parallel = splitted_input[tp_rank].contiguous()
289
+
290
+ # Matrix multiply.
291
+ output_parallel = self.apply_weights(input_parallel)
292
+ if self.reduce_results and self.tp_size > 1:
293
+ output_ = tensor_model_parallel_all_reduce(output_parallel)
294
+ else:
295
+ output_ = output_parallel
296
+
297
+ if not self.skip_bias_add:
298
+ output = output_ + self.bias if self.bias is not None else output_
299
+ output_bias = None
300
+ else:
301
+ output = output_
302
+ output_bias = self.bias
303
+ return output, output_bias
vllm/model_executor/parallel_utils/parallel_state.py
+56
-376
@@ -1,78 +1,42 @@
1
# Copyright 2023 The vLLM team.
2
-# Adapted from https://github.com/NVIDIA/Megatron-LM/blob/main/megatron/core/parallel_state.py
2
+# Adapted from
3
+# https://github.com/NVIDIA/Megatron-LM/blob/main/megatron/core/parallel_state.py
4
# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
4
-
5
"""Model and data parallel groups."""
6
7
import torch
8
-from typing import Optional
8
10
-# Intra-layer model parallel group that the current rank belongs to.
9
+# Tensor model parallel group that the current rank belongs to.
10
_TENSOR_MODEL_PARALLEL_GROUP = None
12
-# Inter-layer model parallel group that the current rank belongs to.
11
+# Pipeline model parallel group that the current rank belongs to.
12
_PIPELINE_MODEL_PARALLEL_GROUP = None
14
-# Model parallel group (both intra- and pipeline) that the current rank belongs to.
15
-_MODEL_PARALLEL_GROUP = None
16
-# Embedding group.
17
-_EMBEDDING_GROUP = None
18
-# Position embedding group.
19
-_POSITION_EMBEDDING_GROUP = None
20
-# Data parallel group that the current rank belongs to.
21
-_DATA_PARALLEL_GROUP = None
22
-
23
-_VIRTUAL_PIPELINE_MODEL_PARALLEL_RANK = None
24
-_VIRTUAL_PIPELINE_MODEL_PARALLEL_WORLD_SIZE = None
25
-_PIPELINE_MODEL_PARALLEL_SPLIT_RANK = None
26
-
27
-# These values enable us to change the mpu sizes on the fly.
28
-_MPU_TENSOR_MODEL_PARALLEL_WORLD_SIZE = None
29
-_MPU_PIPELINE_MODEL_PARALLEL_WORLD_SIZE = None
30
-_MPU_TENSOR_MODEL_PARALLEL_RANK = None
31
-_MPU_PIPELINE_MODEL_PARALLEL_RANK = None
32
-
33
-# A list of ranks that have a copy of the embedding.
34
-_EMBEDDING_GLOBAL_RANKS = None
35
-
36
-# A list of ranks that have a copy of the position embedding.
37
-_POSITION_EMBEDDING_GLOBAL_RANKS = None
38
-
39
-# A list of global ranks for each pipeline group to ease calculation of the source
40
-# rank when broadcasting from the first or last pipeline stage.
41
-_PIPELINE_GLOBAL_RANKS = None
13
43
-# A list of global ranks for each data parallel group to ease calculation of the source
44
-# rank when broadcasting weights from src to all other data parallel ranks
45
-_DATA_PARALLEL_GLOBAL_RANKS = None
14
+# A list of global ranks for each pipeline group to ease calculation of the
15
+# source rank when broadcasting from the first or last pipeline stage.
16
+_PIPELINE_GLOBAL_RANKS = None
17
18
19
def initialize_model_parallel(
20
tensor_model_parallel_size: int = 1,
21
pipeline_model_parallel_size: int = 1,
51
- virtual_pipeline_model_parallel_size: Optional[int] = None,
52
- pipeline_model_parallel_split_rank: Optional[int] = None,
22
) -> None:
23
"""
55
- Initialize model data parallel groups.
24
+ Initialize model parallel groups.
25
26
Arguments:
58
- tensor_model_parallel_size: number of GPUs used for tensor model parallelism.
59
- pipeline_model_parallel_size: number of GPUs used for pipeline model parallelism.
60
- virtual_pipeline_model_parallel_size: number of virtual stages (interleaved
61
- pipeline).
62
- pipeline_model_parallel_split_rank: for models with both encoder and decoder,
63
- rank in pipeline with split point.
64
-
65
- Let's say we have a total of 16 GPUs denoted by g0 ... g15 and we
27
+ tensor_model_parallel_size: number of GPUs used for tensor model
28
+ parallelism.
29
+ pipeline_model_parallel_size: number of GPUs used for pipeline model
30
+ parallelism.
31
+
32
+ Let's say we have a total of 8 GPUs denoted by g0 ... g7 and we
33
use 2 GPUs to parallelize the model tensor, and 4 GPUs to parallelize
34
the model pipeline. The present function will
68
- create 8 tensor model-parallel groups, 4 pipeline model-parallel groups
69
- and 8 data-parallel groups as:
70
- 8 data_parallel groups:
71
- [g0, g2], [g1, g3], [g4, g6], [g5, g7], [g8, g10], [g9, g11], [g12, g14], [g13, g15]
72
- 8 tensor model-parallel groups:
73
- [g0, g1], [g2, g3], [g4, g5], [g6, g7], [g8, g9], [g10, g11], [g12, g13], [g14, g15]
74
- 4 pipeline model-parallel groups:
75
- [g0, g4, g8, g12], [g1, g5, g9, g13], [g2, g6, g10, g14], [g3, g7, g11, g15]
35
+ create 4 tensor model-parallel groups and 2 pipeline model-parallel groups:
36
+ 4 tensor model-parallel groups:
37
+ [g0, g1], [g2, g3], [g4, g5], [g6, g7]
38
+ 2 pipeline model-parallel groups:
39
+ [g0, g2, g4, g6], [g1, g3, g5, g7]
40
Note that for efficiency, the caller should make sure adjacent ranks
41
are on the same DGX box. For example if we are using 2 DGX-1 boxes
42
with a total of 16 GPUs, rank 0 to 7 belong to the first box and
@@ -82,64 +46,23 @@ def initialize_model_parallel(
46
assert torch.distributed.is_initialized()
47
world_size: int = torch.distributed.get_world_size()
48
85
- if world_size % (tensor_model_parallel_size * pipeline_model_parallel_size) != 0:
49
+ if (world_size !=
50
+ tensor_model_parallel_size * pipeline_model_parallel_size):
51
raise RuntimeError(
87
- f"world_size ({world_size}) is not divisible by tensor_model_parallel_size "
88
- f"({tensor_model_parallel_size}) x pipeline_model_parallel_size ({pipeline_model_parallel_size})"
89
- )
90
-
91
- data_parallel_size: int = world_size // (tensor_model_parallel_size *
92
- pipeline_model_parallel_size)
93
-
94
- num_tensor_model_parallel_groups: int = world_size // tensor_model_parallel_size
95
- num_pipeline_model_parallel_groups: int = world_size // pipeline_model_parallel_size
96
- num_data_parallel_groups: int = world_size // data_parallel_size
97
-
98
- if virtual_pipeline_model_parallel_size is not None:
99
- if not pipeline_model_parallel_size > 2:
100
- raise RuntimeError("pipeline-model-parallel size should be greater than 2 with "
101
- "interleaved schedule")
102
- global _VIRTUAL_PIPELINE_MODEL_PARALLEL_RANK
103
- global _VIRTUAL_PIPELINE_MODEL_PARALLEL_WORLD_SIZE
104
- _VIRTUAL_PIPELINE_MODEL_PARALLEL_RANK = 0
105
- _VIRTUAL_PIPELINE_MODEL_PARALLEL_WORLD_SIZE = virtual_pipeline_model_parallel_size
106
-
107
- if pipeline_model_parallel_split_rank is not None:
108
- global _PIPELINE_MODEL_PARALLEL_SPLIT_RANK
109
- _PIPELINE_MODEL_PARALLEL_SPLIT_RANK = pipeline_model_parallel_split_rank
110
-
52
+ f"world_size ({world_size}) is not equal to "
53
+ f"tensor_model_parallel_size ({tensor_model_parallel_size}) x "
54
+ f"pipeline_model_parallel_size ({pipeline_model_parallel_size})")
55
+
56
+ num_tensor_model_parallel_groups: int = (world_size //
57
+ tensor_model_parallel_size)
58
+ num_pipeline_model_parallel_groups: int = (world_size //
59
+ pipeline_model_parallel_size)
60
rank = torch.distributed.get_rank()
61
113
- # Build the data-parallel groups.
114
- global _DATA_PARALLEL_GROUP
115
- global _DATA_PARALLEL_GLOBAL_RANKS
116
- assert _DATA_PARALLEL_GROUP is None, 'data parallel group is already initialized'
117
- all_data_parallel_group_ranks = []
118
- for i in range(pipeline_model_parallel_size):
119
- start_rank = i * num_pipeline_model_parallel_groups
120
- end_rank = (i + 1) * num_pipeline_model_parallel_groups
121
- for j in range(tensor_model_parallel_size):
122
- ranks = range(start_rank + j, end_rank, tensor_model_parallel_size)
123
- all_data_parallel_group_ranks.append(list(ranks))
124
- group = torch.distributed.new_group(ranks)
125
- if rank in ranks:
126
- _DATA_PARALLEL_GROUP = group
127
- _DATA_PARALLEL_GLOBAL_RANKS = ranks
128
-
129
- # Build the model-parallel groups.
130
- global _MODEL_PARALLEL_GROUP
131
- assert _MODEL_PARALLEL_GROUP is None, 'model parallel group is already initialized'
132
- for i in range(data_parallel_size):
133
- ranks = [data_parallel_group_ranks[i]
134
- for data_parallel_group_ranks in all_data_parallel_group_ranks]
135
- group = torch.distributed.new_group(ranks)
136
- if rank in ranks:
137
- _MODEL_PARALLEL_GROUP = group
138
-
62
# Build the tensor model-parallel groups.
63
global _TENSOR_MODEL_PARALLEL_GROUP
141
- assert _TENSOR_MODEL_PARALLEL_GROUP is None, \
142
- 'tensor model parallel group is already initialized'
64
+ assert _TENSOR_MODEL_PARALLEL_GROUP is None, (
65
+ "tensor model parallel group is already initialized")
66
for i in range(num_tensor_model_parallel_groups):
67
ranks = range(i * tensor_model_parallel_size,
68
(i + 1) * tensor_model_parallel_size)
@@ -147,268 +70,60 @@ def initialize_model_parallel(
70
if rank in ranks:
71
_TENSOR_MODEL_PARALLEL_GROUP = group
72
150
- # Build the pipeline model-parallel groups and embedding groups
151
- # (first and last rank in each pipeline model-parallel group).
73
+ # Build the pipeline model-parallel groups.
74
global _PIPELINE_MODEL_PARALLEL_GROUP
75
global _PIPELINE_GLOBAL_RANKS
154
- assert _PIPELINE_MODEL_PARALLEL_GROUP is None, \
155
- 'pipeline model parallel group is already initialized'
156
- global _EMBEDDING_GROUP
157
- global _EMBEDDING_GLOBAL_RANKS
158
- assert _EMBEDDING_GROUP is None, 'embedding group is already initialized'
159
- global _POSITION_EMBEDDING_GROUP
160
- global _POSITION_EMBEDDING_GLOBAL_RANKS
161
- assert _POSITION_EMBEDDING_GROUP is None, \
162
- 'position embedding group is already initialized'
76
+ assert _PIPELINE_MODEL_PARALLEL_GROUP is None, (
77
+ "pipeline model parallel group is already initialized")
78
for i in range(num_pipeline_model_parallel_groups):
79
ranks = range(i, world_size, num_pipeline_model_parallel_groups)
80
group = torch.distributed.new_group(ranks)
81
if rank in ranks:
82
_PIPELINE_MODEL_PARALLEL_GROUP = group
83
_PIPELINE_GLOBAL_RANKS = ranks
169
- # Setup embedding group (to exchange gradients between
170
- # first and last stages).
171
- if len(ranks) > 1:
172
- embedding_ranks = [ranks[0], ranks[-1]]
173
- position_embedding_ranks = [ranks[0]]
174
- if pipeline_model_parallel_split_rank is not None:
175
- if ranks[pipeline_model_parallel_split_rank] not in embedding_ranks:
176
- embedding_ranks = [ranks[0],
177
- ranks[pipeline_model_parallel_split_rank],
178
- ranks[-1]]
179
- if ranks[pipeline_model_parallel_split_rank] not in position_embedding_ranks:
180
- position_embedding_ranks = [ranks[0],
181
- ranks[pipeline_model_parallel_split_rank]]
182
- else:
183
- embedding_ranks = ranks
184
- position_embedding_ranks = ranks
185
-
186
- group = torch.distributed.new_group(embedding_ranks)
187
- if rank in embedding_ranks:
188
- _EMBEDDING_GROUP = group
189
- if rank in ranks:
190
- _EMBEDDING_GLOBAL_RANKS = embedding_ranks
84
192
- group = torch.distributed.new_group(position_embedding_ranks)
193
- if rank in position_embedding_ranks:
194
- _POSITION_EMBEDDING_GROUP = group
195
- if rank in ranks:
196
- _POSITION_EMBEDDING_GLOBAL_RANKS = position_embedding_ranks
85
86
def model_parallel_is_initialized():
87
"""Check if model and data parallel groups are initialized."""
200
- if _TENSOR_MODEL_PARALLEL_GROUP is None or \
201
- _PIPELINE_MODEL_PARALLEL_GROUP is None or \
202
- _DATA_PARALLEL_GROUP is None:
203
- return False
204
- return True
205
-
206
-
207
-def get_model_parallel_group():
208
- """Get the model parallel group the caller rank belongs to."""
209
- assert _MODEL_PARALLEL_GROUP is not None, \
210
- 'model parallel group is not initialized'
211
- return _MODEL_PARALLEL_GROUP
88
+ return (_TENSOR_MODEL_PARALLEL_GROUP is not None
89
+ and _PIPELINE_MODEL_PARALLEL_GROUP is not None)
90
91
92
def get_tensor_model_parallel_group():
93
"""Get the tensor model parallel group the caller rank belongs to."""
216
- assert _TENSOR_MODEL_PARALLEL_GROUP is not None, \
217
- 'intra_layer_model parallel group is not initialized'
94
+ assert _TENSOR_MODEL_PARALLEL_GROUP is not None, (
95
+ "tenosr model parallel group is not initialized")
96
return _TENSOR_MODEL_PARALLEL_GROUP
97
98
99
def get_pipeline_model_parallel_group():
100
"""Get the pipeline model parallel group the caller rank belongs to."""
223
- assert _PIPELINE_MODEL_PARALLEL_GROUP is not None, \
224
- 'pipeline_model parallel group is not initialized'
101
+ assert _PIPELINE_MODEL_PARALLEL_GROUP is not None, (
102
+ "pipeline model parallel group is not initialized")
103
return _PIPELINE_MODEL_PARALLEL_GROUP
104
105
228
-def get_data_parallel_group():
229
- """Get the data parallel group the caller rank belongs to."""
230
- assert _DATA_PARALLEL_GROUP is not None, \
231
- 'data parallel group is not initialized'
232
- return _DATA_PARALLEL_GROUP
233
-
234
-
235
-def get_embedding_group():
236
- """Get the embedding group the caller rank belongs to."""
237
- assert _EMBEDDING_GROUP is not None, \
238
- 'embedding group is not initialized'
239
- return _EMBEDDING_GROUP
240
-
241
-
242
-def get_position_embedding_group():
243
- """Get the position embedding group the caller rank belongs to."""
244
- assert _POSITION_EMBEDDING_GROUP is not None, \
245
- 'position embedding group is not initialized'
246
- return _POSITION_EMBEDDING_GROUP
247
-
248
-
249
-def set_tensor_model_parallel_world_size(world_size):
250
- """Set the tensor model parallel size"""
251
- global _MPU_TENSOR_MODEL_PARALLEL_WORLD_SIZE
252
- _MPU_TENSOR_MODEL_PARALLEL_WORLD_SIZE = world_size
253
-
254
-
255
-def set_pipeline_model_parallel_world_size(world_size):
256
- """Set the pipeline model parallel size"""
257
- global _MPU_PIPELINE_MODEL_PARALLEL_WORLD_SIZE
258
- _MPU_PIPELINE_MODEL_PARALLEL_WORLD_SIZE = world_size
259
-
260
-
106
def get_tensor_model_parallel_world_size():
107
"""Return world size for the tensor model parallel group."""
263
- global _MPU_TENSOR_MODEL_PARALLEL_WORLD_SIZE
264
- if _MPU_TENSOR_MODEL_PARALLEL_WORLD_SIZE is not None:
265
- return _MPU_TENSOR_MODEL_PARALLEL_WORLD_SIZE
266
- return torch.distributed.get_world_size(group=get_tensor_model_parallel_group())
108
+ return torch.distributed.get_world_size(
109
+ group=get_tensor_model_parallel_group())
110
111
112
def get_pipeline_model_parallel_world_size():
113
"""Return world size for the pipeline model parallel group."""
271
- global _MPU_PIPELINE_MODEL_PARALLEL_WORLD_SIZE
272
- if _MPU_PIPELINE_MODEL_PARALLEL_WORLD_SIZE is not None:
273
- return _MPU_PIPELINE_MODEL_PARALLEL_WORLD_SIZE
274
- return torch.distributed.get_world_size(group=get_pipeline_model_parallel_group())
275
-
276
-
277
-def set_tensor_model_parallel_rank(rank):
278
- """Set tensor model parallel rank."""
279
- global _MPU_TENSOR_MODEL_PARALLEL_RANK
280
- _MPU_TENSOR_MODEL_PARALLEL_RANK = rank
281
-
282
-
283
-def set_pipeline_model_parallel_rank(rank):
284
- """Set pipeline model parallel rank."""
285
- global _MPU_PIPELINE_MODEL_PARALLEL_RANK
286
- _MPU_PIPELINE_MODEL_PARALLEL_RANK = rank
287
-
288
-
289
-def set_pipeline_model_parallel_split_rank(rank):
290
- """Set pipeline model parallel split rank."""
291
- global _MPU_PIPELINE_MODEL_PARALLEL_SPLIT_RANK
292
- _MPU_PIPELINE_MODEL_PARALLEL_SPLIT_RANK = rank
114
+ return torch.distributed.get_world_size(
115
+ group=get_pipeline_model_parallel_group())
116
117
118
def get_tensor_model_parallel_rank():
119
"""Return my rank for the tensor model parallel group."""
297
- global _MPU_TENSOR_MODEL_PARALLEL_RANK
298
- if _MPU_TENSOR_MODEL_PARALLEL_RANK is not None:
299
- return _MPU_TENSOR_MODEL_PARALLEL_RANK
120
return torch.distributed.get_rank(group=get_tensor_model_parallel_group())
121
122
123
def get_pipeline_model_parallel_rank():
124
"""Return my rank for the pipeline model parallel group."""
305
- global _MPU_PIPELINE_MODEL_PARALLEL_RANK
306
- if _MPU_PIPELINE_MODEL_PARALLEL_RANK is not None:
307
- return _MPU_PIPELINE_MODEL_PARALLEL_RANK
308
- return torch.distributed.get_rank(group=get_pipeline_model_parallel_group())
309
-
310
-
311
-
312
-def is_pipeline_first_stage(ignore_virtual=False):
313
- """Return True if in the first pipeline model-parallel stage, False otherwise."""
314
- if not ignore_virtual:
315
- if get_virtual_pipeline_model_parallel_world_size() is not None and \
316
- get_virtual_pipeline_model_parallel_rank() != 0:
317
- return False
318
- return get_pipeline_model_parallel_rank() == 0
319
-
320
-
321
-def is_pipeline_last_stage(ignore_virtual=False):
322
- """Return True if in the last pipeline model-parallel stage, False otherwise."""
323
- if not ignore_virtual:
324
- virtual_pipeline_model_parallel_world_size = \
325
- get_virtual_pipeline_model_parallel_world_size()
326
- if virtual_pipeline_model_parallel_world_size is not None and \
327
- get_virtual_pipeline_model_parallel_rank() != (
328
- virtual_pipeline_model_parallel_world_size - 1):
329
- return False
330
- return get_pipeline_model_parallel_rank() == (
331
- get_pipeline_model_parallel_world_size() - 1)
332
-
333
-
334
-def is_rank_in_embedding_group(ignore_virtual=False):
335
- """Return true if current rank is in embedding group, False otherwise."""
336
- rank = torch.distributed.get_rank()
337
- global _EMBEDDING_GLOBAL_RANKS
338
- if ignore_virtual:
339
- return rank in _EMBEDDING_GLOBAL_RANKS
340
- if rank in _EMBEDDING_GLOBAL_RANKS:
341
- if rank == _EMBEDDING_GLOBAL_RANKS[0]:
342
- return is_pipeline_first_stage(ignore_virtual=False)
343
- elif rank == _EMBEDDING_GLOBAL_RANKS[-1]:
344
- return is_pipeline_last_stage(ignore_virtual=False)
345
- else:
346
- return True
347
- return False
348
-
349
-
350
-def is_rank_in_position_embedding_group():
351
- """Return true if current rank is in position embedding group, False otherwise."""
352
- rank = torch.distributed.get_rank()
353
- global _POSITION_EMBEDDING_GLOBAL_RANKS
354
- return rank in _POSITION_EMBEDDING_GLOBAL_RANKS
355
-
356
-
357
-def is_pipeline_stage_before_split(rank=None):
358
- """Return True if pipeline stage executes encoder block for a model
359
- with both encoder and decoder."""
360
- if get_pipeline_model_parallel_world_size() == 1:
361
- return True
362
- if rank is None:
363
- rank = get_pipeline_model_parallel_rank()
364
- global _PIPELINE_MODEL_PARALLEL_SPLIT_RANK
365
- if _PIPELINE_MODEL_PARALLEL_SPLIT_RANK is None:
366
- return True
367
- if rank < _PIPELINE_MODEL_PARALLEL_SPLIT_RANK:
368
- return True
369
- return False
370
-
371
-
372
-def is_pipeline_stage_after_split(rank=None):
373
- """Return True if pipeline stage executes decoder block for a model
374
- with both encoder and decoder."""
375
- if get_pipeline_model_parallel_world_size() == 1:
376
- return True
377
- if rank is None:
378
- rank = get_pipeline_model_parallel_rank()
379
- global _PIPELINE_MODEL_PARALLEL_SPLIT_RANK
380
- if _PIPELINE_MODEL_PARALLEL_SPLIT_RANK is None:
381
- return True
382
- if rank >= _PIPELINE_MODEL_PARALLEL_SPLIT_RANK:
383
- return True
384
- return False
385
-
386
-
387
-def is_pipeline_stage_at_split():
388
- """Return true if pipeline stage executes decoder block and next
389
- stage executes encoder block for a model with both encoder and
390
- decoder."""
391
- rank = get_pipeline_model_parallel_rank()
392
- return is_pipeline_stage_before_split(rank) and \
393
- is_pipeline_stage_after_split(rank+1)
394
-
395
-
396
-def get_virtual_pipeline_model_parallel_rank():
397
- """Return the virtual pipeline-parallel rank."""
398
- global _VIRTUAL_PIPELINE_MODEL_PARALLEL_RANK
399
- return _VIRTUAL_PIPELINE_MODEL_PARALLEL_RANK
400
-
401
-
402
-def set_virtual_pipeline_model_parallel_rank(rank):
403
- """Set the virtual pipeline-parallel rank."""
404
- global _VIRTUAL_PIPELINE_MODEL_PARALLEL_RANK
405
- _VIRTUAL_PIPELINE_MODEL_PARALLEL_RANK = rank
406
-
407
-
408
-def get_virtual_pipeline_model_parallel_world_size():
409
- """Return the virtual pipeline-parallel world size."""
410
- global _VIRTUAL_PIPELINE_MODEL_PARALLEL_WORLD_SIZE
411
- return _VIRTUAL_PIPELINE_MODEL_PARALLEL_WORLD_SIZE
125
+ return torch.distributed.get_rank(
126
+ group=get_pipeline_model_parallel_group())
127
128
129
def get_tensor_model_parallel_src_rank():
@@ -419,35 +134,27 @@ def get_tensor_model_parallel_src_rank():
134
return (global_rank // local_world_size) * local_world_size
135
136
422
-def get_data_parallel_src_rank():
423
- """Calculate the global rank corresponding to the first local rank
424
- in the data parallel group."""
425
- assert _DATA_PARALLEL_GLOBAL_RANKS is not None, \
426
- "Data parallel group is not initialized"
427
- return _DATA_PARALLEL_GLOBAL_RANKS[0]
428
-
429
-
137
def get_pipeline_model_parallel_first_rank():
138
"""Return the global rank of the first process in the pipeline for the
139
current tensor parallel group"""
433
- assert _PIPELINE_GLOBAL_RANKS is not None, \
434
- "Pipeline parallel group is not initialized"
140
+ assert _PIPELINE_GLOBAL_RANKS is not None, (
141
+ "Pipeline parallel group is not initialized")
142
return _PIPELINE_GLOBAL_RANKS[0]
143
144
145
def get_pipeline_model_parallel_last_rank():
146
"""Return the global rank of the last process in the pipeline for the
147
current tensor parallel group"""
441
- assert _PIPELINE_GLOBAL_RANKS is not None, \
442
- "Pipeline parallel group is not initialized"
148
+ assert _PIPELINE_GLOBAL_RANKS is not None, (
149
+ "Pipeline parallel group is not initialized")
150
last_rank_local = get_pipeline_model_parallel_world_size() - 1
151
return _PIPELINE_GLOBAL_RANKS[last_rank_local]
152
153
154
def get_pipeline_model_parallel_next_rank():
155
"""Return the global rank that follows the caller in the pipeline"""
449
- assert _PIPELINE_GLOBAL_RANKS is not None, \
450
- "Pipeline parallel group is not initialized"
156
+ assert _PIPELINE_GLOBAL_RANKS is not None, (
157
+ "Pipeline parallel group is not initialized")
158
rank_in_pipeline = get_pipeline_model_parallel_rank()
159
world_size = get_pipeline_model_parallel_world_size()
160
return _PIPELINE_GLOBAL_RANKS[(rank_in_pipeline + 1) % world_size]
@@ -455,45 +162,18 @@ def get_pipeline_model_parallel_next_rank():
162
163
def get_pipeline_model_parallel_prev_rank():
164
"""Return the global rank that preceeds the caller in the pipeline"""
458
- assert _PIPELINE_GLOBAL_RANKS is not None, \
459
- "Pipeline parallel group is not initialized"
165
+ assert _PIPELINE_GLOBAL_RANKS is not None, (
166
+ "Pipeline parallel group is not initialized")
167
rank_in_pipeline = get_pipeline_model_parallel_rank()
168
world_size = get_pipeline_model_parallel_world_size()
169
return _PIPELINE_GLOBAL_RANKS[(rank_in_pipeline - 1) % world_size]
170
171
465
-def get_data_parallel_world_size():
466
- """Return world size for the data parallel group."""
467
- return torch.distributed.get_world_size(group=get_data_parallel_group())
468
-
469
-
470
-def get_data_parallel_rank():
471
- """Return my rank for the data parallel group."""
472
- return torch.distributed.get_rank(group=get_data_parallel_group())
473
-
172
def destroy_model_parallel():
173
"""Set the groups to none."""
476
- global _MODEL_PARALLEL_GROUP
477
- _MODEL_PARALLEL_GROUP = None
174
global _TENSOR_MODEL_PARALLEL_GROUP
175
_TENSOR_MODEL_PARALLEL_GROUP = None
176
global _PIPELINE_MODEL_PARALLEL_GROUP
177
_PIPELINE_MODEL_PARALLEL_GROUP = None
482
- global _DATA_PARALLEL_GROUP
483
- _DATA_PARALLEL_GROUP = None
484
- global _EMBEDDING_GROUP
485
- _EMBEDDING_GROUP = None
486
- global _POSITION_EMBEDDING_GROUP
487
- _POSITION_EMBEDDING_GROUP = None
488
- global _VIRTUAL_PIPELINE_MODEL_PARALLEL_RANK
489
- _VIRTUAL_PIPELINE_MODEL_PARALLEL_RANK = None
490
- global _VIRTUAL_PIPELINE_MODEL_PARALLEL_WORLD_SIZE
491
- _VIRTUAL_PIPELINE_MODEL_PARALLEL_WORLD_SIZE = None
492
- global _MPU_TENSOR_MODEL_PARALLEL_WORLD_SIZE
493
- _MPU_TENSOR_MODEL_PARALLEL_WORLD_SIZE = None
494
- global _MPU_PIPELINE_MODEL_PARALLEL_WORLD_SIZE
495
- _MPU_PIPELINE_MODEL_PARALLEL_WORLD_SIZE = None
496
- global _MPU_TENSOR_MODEL_PARALLEL_RANK
497
- _MPU_TENSOR_MODEL_PARALLEL_RANK = None
498
- global _MPU_PIPELINE_MODEL_PARALLEL_RANK
499
- _MPU_PIPELINE_MODEL_PARALLEL_RANK = None
178
+ global _PIPELINE_GLOBAL_RANKS
179
+ _PIPELINE_GLOBAL_RANKS = None
vllm/model_executor/parallel_utils/tensor_parallel/__init__.py
deleted
-50
@@ -1,50 +0,0 @@
1
-from .layers import (
2
- ColumnParallelLinear,
3
- RowParallelLinear,
4
- VocabParallelEmbedding,
5
- set_tensor_model_parallel_attributes,
6
- set_defaults_if_not_set_tensor_model_parallel_attributes,
7
- copy_tensor_model_parallel_attributes,
8
- param_is_not_tensor_parallel_duplicate,
9
-)
10
-
11
-from .mappings import (
12
- copy_to_tensor_model_parallel_region,
13
- gather_from_tensor_model_parallel_region,
14
- gather_from_sequence_parallel_region,
15
- reduce_from_tensor_model_parallel_region,
16
- scatter_to_tensor_model_parallel_region,
17
- scatter_to_sequence_parallel_region,
18
-)
19
-
20
-from .random import (
21
- get_cuda_rng_tracker,
22
- model_parallel_cuda_manual_seed,
23
-)
24
-
25
-from .utils import (
26
- split_tensor_along_last_dim,
27
-)
28
-
29
-__all__ = [
30
- #layers.py
31
- "ColumnParallelLinear",
32
- "RowParallelLinear",
33
- "VocabParallelEmbedding",
34
- "set_tensor_model_parallel_attributes",
35
- "set_defaults_if_not_set_tensor_model_parallel_attributes",
36
- "copy_tensor_model_parallel_attributes",
37
- "param_is_not_tensor_parallel_duplicate",
38
- # mappings.py
39
- "copy_to_tensor_model_parallel_region",
40
- "gather_from_tensor_model_parallel_region",
41
- "gather_from_sequence_parallel_region",
42
- "reduce_from_tensor_model_parallel_region",
43
- "scatter_to_tensor_model_parallel_region",
44
- "scatter_to_sequence_parallel_region",
45
- # random.py
46
- "get_cuda_rng_tracker",
47
- "model_parallel_cuda_manual_seed",
48
- # utils.py
49
- "split_tensor_along_last_dim",
50
-]
vllm/model_executor/parallel_utils/tensor_parallel/layers.py
deleted
-366
@@ -1,366 +0,0 @@
1
-# Copyright 2023 The vLLM team.
2
-# Adapted from https://github.com/NVIDIA/Megatron-LM/blob/main/megatron/core/tensor_parallel/layers.py
3
-# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
4
-
5
-# Parts of the code here are adapted from PyTorch
6
-# repo: https://github.com/pytorch/pytorch
7
-from typing import Optional
8
-
9
-import torch
10
-import torch.nn.functional as F
11
-import torch.nn.init as init
12
-from torch.nn.parameter import Parameter
13
-
14
-from vllm.model_executor.parallel_utils.parallel_state import (
15
- get_tensor_model_parallel_rank,
16
- get_tensor_model_parallel_world_size,
17
-)
18
-from .mappings import (
19
- gather_from_tensor_model_parallel_region,
20
- reduce_from_tensor_model_parallel_region,
21
- scatter_to_tensor_model_parallel_region,
22
-)
23
-
24
-from .utils import (
25
- divide,
26
- VocabUtility,
27
-)
28
-
29
-_MODEL_PARALLEL_ATTRIBUTE_DEFAULTS = {'tensor_model_parallel': False,
30
- 'partition_dim': -1,
31
- 'partition_stride': 1}
32
-
33
-def param_is_not_tensor_parallel_duplicate(param):
34
- return (hasattr(param, 'tensor_model_parallel') and
35
- param.tensor_model_parallel) or (
36
- get_tensor_model_parallel_rank() == 0)
37
-
38
-
39
-def set_tensor_model_parallel_attributes(tensor, is_parallel, dim, stride):
40
- # Make sure the attributes are not set.
41
- for attribute in _MODEL_PARALLEL_ATTRIBUTE_DEFAULTS:
42
- assert not hasattr(tensor, attribute)
43
- # Set the attributes.
44
- setattr(tensor, 'tensor_model_parallel', is_parallel)
45
- setattr(tensor, 'partition_dim', dim)
46
- setattr(tensor, 'partition_stride', stride)
47
-
48
-
49
-def set_defaults_if_not_set_tensor_model_parallel_attributes(tensor):
50
- def maybe_set(attribute, value):
51
- if not hasattr(tensor, attribute):
52
- setattr(tensor, attribute, value)
53
- for attribute in _MODEL_PARALLEL_ATTRIBUTE_DEFAULTS:
54
- maybe_set(attribute, _MODEL_PARALLEL_ATTRIBUTE_DEFAULTS[attribute])
55
-
56
-
57
-def copy_tensor_model_parallel_attributes(destination_tensor, source_tensor):
58
- def maybe_copy(attribute):
59
- if hasattr(source_tensor, attribute):
60
- setattr(destination_tensor, attribute,
61
- getattr(source_tensor, attribute))
62
- for attribute in _MODEL_PARALLEL_ATTRIBUTE_DEFAULTS:
63
- maybe_copy(attribute)
64
-
65
-
66
-class VocabParallelEmbedding(torch.nn.Module):
67
- """Embedding parallelized in the vocabulary dimension.
68
-
69
- This is mainly adapted from torch.nn.Embedding and all the default
70
- values are kept.
71
- Arguments:
72
- num_embeddings: vocabulary size.
73
- embedding_dim: size of hidden state.
74
-
75
- Keyword Arguments:
76
- init_method: method to initialize weights.
77
- params_dtype
78
- use_cpu_initialization
79
- perform_initialization
80
- """
81
-
82
- def __init__(self, num_embeddings: int, embedding_dim: int, *,
83
- init_method=init.xavier_normal_,
84
- params_dtype: torch.dtype=None,
85
- use_cpu_initialization: bool=False,
86
- perform_initialization: bool=False):
87
- super(VocabParallelEmbedding, self).__init__()
88
- assert not perform_initialization
89
- assert not use_cpu_initialization
90
-
91
- # Keep the input dimensions.
92
- self.num_embeddings = num_embeddings
93
- self.embedding_dim = embedding_dim
94
- if params_dtype is None:
95
- params_dtype = torch.get_default_dtype()
96
-
97
- # Set the defaults for compatibility.
98
- self.padding_idx = None
99
- self.max_norm = None
100
- self.norm_type = 2.
101
- self.scale_grad_by_freq = False
102
- self.sparse = False
103
- self._weight = None
104
- self.tensor_model_parallel_size = get_tensor_model_parallel_world_size()
105
- # Divide the weight matrix along the vocaburaly dimension.
106
- self.vocab_start_index, self.vocab_end_index = \
107
- VocabUtility.vocab_range_from_global_vocab_size(
108
- self.num_embeddings, get_tensor_model_parallel_rank(),
109
- self.tensor_model_parallel_size)
110
- self.num_embeddings_per_partition = self.vocab_end_index - \
111
- self.vocab_start_index
112
-
113
- self.weight = Parameter(torch.empty(
114
- self.num_embeddings_per_partition, self.embedding_dim,
115
- device=torch.cuda.current_device(), dtype=params_dtype))
116
-
117
- def forward(self, input_):
118
- if self.tensor_model_parallel_size > 1:
119
- # Build the mask.
120
- input_mask = (input_ < self.vocab_start_index) | \
121
- (input_ >= self.vocab_end_index)
122
- # Mask the input.
123
- masked_input = input_.clone() - self.vocab_start_index
124
- masked_input[input_mask] = 0
125
- else:
126
- masked_input = input_
127
- # Get the embeddings.
128
- output_parallel = F.embedding(masked_input, self.weight,
129
- self.padding_idx, self.max_norm,
130
- self.norm_type, self.scale_grad_by_freq,
131
- self.sparse)
132
- # Mask the output embedding.
133
- if self.tensor_model_parallel_size > 1:
134
- output_parallel[input_mask, :] = 0.0
135
- # Reduce across all the model parallel GPUs.
136
- output = reduce_from_tensor_model_parallel_region(output_parallel)
137
- return output
138
-
139
-
140
-class ColumnParallelLinear(torch.nn.Module):
141
- """Linear layer with column parallelism.
142
-
143
- The linear layer is defined as Y = XA + b. A is parallelized along
144
- its second dimension as A = [A_1, ..., A_p].
145
-
146
- Arguments:
147
- input_size: first dimension of matrix A.
148
- output_size: second dimension of matrix A.
149
-
150
- Keyword Arguments
151
- bias: If true, add bias
152
- gather_output: If true, call all-gather on output and make Y available
153
- to all GPUs, otherwise, every GPU will have its output
154
- which is Y_i = XA_i
155
- init_method: method to initialize weights. Note that bias is always set
156
- to zero.
157
- stride: For the strided linear layers.
158
- keep_master_weight_for_test: This was added for testing and should be
159
- set to False. It returns the master weights
160
- used for initialization.
161
- skip_bias_add: This was added to enable performance optimations where bias
162
- can be fused with other elementwise operations. we skip
163
- adding bias but instead return it.
164
- params_dtype:
165
- use_cpu_initialization:
166
- """
167
-
168
- def __init__(self, input_size, output_size, *,
169
- bias=True, gather_output=True,
170
- init_method=init.xavier_normal_, stride=1,
171
- keep_master_weight_for_test=False,
172
- skip_bias_add=False,
173
- params_dtype=None,
174
- use_cpu_initialization=False,
175
- perform_initialization=False,
176
- quant_config=None,
177
- ):
178
- super(ColumnParallelLinear, self).__init__()
179
- assert not perform_initialization
180
- assert not use_cpu_initialization
181
-
182
- # Keep input parameters
183
- self.input_size = input_size
184
- self.output_size = output_size
185
- self.gather_output = gather_output
186
- # Divide the weight matrix along the last dimension.
187
- self.world_size = get_tensor_model_parallel_world_size()
188
- self.output_size_per_partition = divide(output_size, self.world_size)
189
- self.skip_bias_add = skip_bias_add
190
- self.quant_config = quant_config
191
-
192
- if params_dtype is None:
193
- params_dtype = torch.get_default_dtype()
194
-
195
- # Parameters.
196
- # Note: torch.nn.functional.linear performs XA^T + b and as a result
197
- # we allocate the transpose.
198
- self.create_weights(params_dtype)
199
-
200
- if bias:
201
- self.bias = Parameter(torch.empty(
202
- self.output_size_per_partition,
203
- device=torch.cuda.current_device(),
204
- dtype=params_dtype))
205
- set_tensor_model_parallel_attributes(self.bias, True, 0, stride)
206
- # Always initialize bias to zero.
207
- with torch.no_grad():
208
- self.bias.zero_()
209
- else:
210
- self.register_parameter('bias', None)
211
-
212
- def create_weights(self, dtype: torch.dtype) -> None:
213
- self.weight = Parameter(torch.empty(
214
- self.output_size_per_partition, self.input_size,
215
- device=torch.cuda.current_device(), dtype=dtype))
216
-
217
- def apply_weights(
218
- self,
219
- x: torch.Tensor,
220
- bias: Optional[torch.Tensor],
221
- ) -> torch.Tensor:
222
- return F.linear(x, self.weight, bias)
223
-
224
- def forward(self, input_):
225
- """Forward of ColumnParallelLinear
226
-
227
- Args:
228
- input_: 3D tensor whose order of dimension is [sequence, batch, hidden]
229
-
230
- Returns:
231
- - output
232
- - bias
233
- """
234
- bias = self.bias if not self.skip_bias_add else None
235
-
236
- input_parallel = input_
237
- # Matrix multiply.
238
- output_parallel = self.apply_weights(input_parallel, bias)
239
- if self.gather_output:
240
- # All-gather across the partitions.
241
- output = gather_from_tensor_model_parallel_region(output_parallel)
242
- else:
243
- output = output_parallel
244
- output_bias = self.bias if self.skip_bias_add else None
245
- return output, output_bias
246
-
247
-
248
-class RowParallelLinear(torch.nn.Module):
249
- """Linear layer with row parallelism.
250
-
251
- The linear layer is defined as Y = XA + b. A is parallelized along
252
- its first dimension and X along its second dimension as:
253
- - -
254
- | A_1 |
255
- | . |
256
- A = | . | X = [X_1, ..., X_p]
257
- | . |
258
- | A_p |
259
- - -
260
- Arguments:
261
- input_size: first dimension of matrix A.
262
- output_size: second dimension of matrix A.
263
-
264
- Keyword Arguments:
265
- bias: If true, add bias. Note that bias is not parallelized.
266
- input_is_parallel: If true, we assume that the input is already
267
- split across the GPUs and we do not split
268
- again.
269
- init_method: method to initialize weights. Note that bias is always set
270
- to zero.
271
- stride: For the strided linear layers.
272
- keep_master_weight_for_test: This was added for testing and should be
273
- set to False. It returns the master weights
274
- used for initialization.
275
- skip_bias_add: This was added to enable performance optimization where bias
276
- can be fused with other elementwise operations. We skip
277
- adding bias but instead return it.
278
- params_dtype:
279
- use_cpu_initialization:
280
- perform_initialization:
281
- reduce_results:
282
- """
283
-
284
- def __init__(self, input_size, output_size, *,
285
- bias=True, input_is_parallel=False,
286
- init_method=init.xavier_normal_, stride=1,
287
- keep_master_weight_for_test=False,
288
- skip_bias_add=False,
289
- params_dtype=None,
290
- use_cpu_initialization=False,
291
- perform_initialization=False,
292
- reduce_results=True,
293
- quant_config=None,
294
- ):
295
- super(RowParallelLinear, self).__init__()
296
- assert not perform_initialization
297
- assert not use_cpu_initialization
298
-
299
- # Keep input parameters
300
- self.input_size = input_size
301
- self.output_size = output_size
302
- self.input_is_parallel = input_is_parallel
303
- self.reduce_results = reduce_results
304
- if params_dtype is None:
305
- params_dtype = torch.get_default_dtype()
306
-
307
- # Divide the weight matrix along the last dimension.
308
- self.world_size = get_tensor_model_parallel_world_size()
309
- self.input_size_per_partition = divide(input_size, self.world_size)
310
- self.skip_bias_add = skip_bias_add
311
- self.quant_config = quant_config
312
-
313
- self.create_weights(params_dtype)
314
-
315
- if not reduce_results and (bias and not skip_bias_add):
316
- raise ValueError("When not reduce the results, adding bias to the "
317
- "results can lead to incorrect results")
318
-
319
- if bias:
320
- self.bias = Parameter(torch.empty(
321
- self.output_size, device=torch.cuda.current_device(),
322
- dtype=params_dtype))
323
-
324
- # Always initialize bias to zero.
325
- with torch.no_grad():
326
- self.bias.zero_()
327
- else:
328
- self.register_parameter('bias', None)
329
-
330
- def create_weights(self, dtype: torch.dtype) -> None:
331
- self.weight = Parameter(torch.empty(
332
- self.output_size, self.input_size_per_partition,
333
- device=torch.cuda.current_device(), dtype=dtype))
334
-
335
- def apply_weights(self, x: torch.Tensor) -> torch.Tensor:
336
- return F.linear(x, self.weight)
337
-
338
- def forward(self, input_):
339
- """Forward of RowParallelLinear
340
-
341
- Args:
342
- input_: 3D tensor whose order of dimension is [sequence, batch, hidden]
343
-
344
- Returns:
345
- - output
346
- - bias
347
- """
348
- # Set up backprop all-reduce.
349
- if self.input_is_parallel:
350
- input_parallel = input_
351
- else:
352
- input_parallel = scatter_to_tensor_model_parallel_region(input_)
353
- # Matrix multiply.
354
- output_parallel = self.apply_weights(input_parallel)
355
- if self.reduce_results and self.world_size > 1:
356
- output_ = reduce_from_tensor_model_parallel_region(output_parallel)
357
- else:
358
- output_ = output_parallel
359
-
360
- if not self.skip_bias_add:
361
- output = output_ + self.bias if self.bias is not None else output_
362
- output_bias = None
363
- else:
364
- output = output_
365
- output_bias = self.bias
366
- return output, output_bias
vllm/model_executor/parallel_utils/tensor_parallel/mappings.py
deleted
-281
@@ -1,281 +0,0 @@
1
-# Copyright 2023 The vLLM team.
2
-# Adapted from https://github.com/NVIDIA/Megatron-LM/blob/main/megatron/core/tensor_parallel/mappings.py
3
-# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
4
-
5
-import torch
6
-
7
-from vllm.model_executor.parallel_utils.parallel_state import (
8
- get_tensor_model_parallel_rank,
9
- get_tensor_model_parallel_world_size,
10
- get_tensor_model_parallel_group,
11
-)
12
-from .utils import split_tensor_along_last_dim
13
-
14
-
15
-def _reduce(input_):
16
- """All-reduce the input tensor across model parallel group."""
17
-
18
- # Bypass the function if we are using only 1 GPU.
19
- if get_tensor_model_parallel_world_size()==1:
20
- return input_
21
-
22
- # All-reduce.
23
- torch.distributed.all_reduce(input_, group=get_tensor_model_parallel_group())
24
-
25
- return input_
26
-
27
-
28
-def _split_along_last_dim(input_):
29
- """Split the tensor along its last dimension and keep the
30
- corresponding slice."""
31
-
32
- world_size = get_tensor_model_parallel_world_size()
33
- # Bypass the function if we are using only 1 GPU.
34
- if world_size == 1:
35
- return input_
36
-
37
- # Split along last dimension.
38
- input_list = split_tensor_along_last_dim(input_, world_size)
39
-
40
- # Note: torch.split does not create contiguous tensors by default.
41
- rank = get_tensor_model_parallel_rank()
42
- output = input_list[rank].contiguous()
43
-
44
- return output
45
-
46
-
47
-def _split_along_first_dim(input_):
48
- """Split the tensor along its first dimension and keep the
49
- corresponding slice."""
50
-
51
- world_size = get_tensor_model_parallel_world_size()
52
- # Bypass the function if we are using only 1 GPU.
53
- if world_size == 1:
54
- return input_
55
-
56
- # Split along first dimension.
57
- dim_size = input_.size()[0]
58
- assert dim_size % world_size == 0, \
59
- "First dimension of the tensor should be divisible by tensor parallel size"
60
- local_dim_size = dim_size // world_size
61
- rank = get_tensor_model_parallel_rank()
62
- dim_offset = rank * local_dim_size
63
-
64
- output = input_[dim_offset:dim_offset+local_dim_size].contiguous()
65
-
66
- return output
67
-
68
-
69
-def _gather_along_last_dim(input_):
70
- """Gather tensors and concatinate along the last dimension."""
71
-
72
- world_size = get_tensor_model_parallel_world_size()
73
- # Bypass the function if we are using only 1 GPU.
74
- if world_size == 1:
75
- return input_
76
-
77
- # Size and dimension.
78
- last_dim = input_.dim() - 1
79
- rank = get_tensor_model_parallel_rank()
80
-
81
- tensor_list = [torch.empty_like(input_) for _ in range(world_size)]
82
- tensor_list[rank] = input_
83
- torch.distributed.all_gather(tensor_list, input_, group=get_tensor_model_parallel_group())
84
-
85
- # Note: torch.cat already creates a contiguous tensor.
86
- output = torch.cat(tensor_list, dim=last_dim).contiguous()
87
-
88
- return output
89
-
90
-
91
-def _gather_along_first_dim(input_):
92
- """Gather tensors and concatinate along the first dimension."""
93
-
94
- world_size = get_tensor_model_parallel_world_size()
95
- # Bypass the function if we are using only 1 GPU.
96
- if world_size == 1:
97
- return input_
98
-
99
- dim_size = list(input_.size())
100
- dim_size[0] = dim_size[0] * world_size
101
-
102
- output = torch.empty(dim_size, dtype=input_.dtype,
103
- device=torch.cuda.current_device())
104
- torch.distributed._all_gather_base(output, input_.contiguous(),
105
- group=get_tensor_model_parallel_group())
106
-
107
- return output
108
-
109
-def _reduce_scatter_along_first_dim(input_):
110
- """Reduce-scatter the input tensor across model parallel group."""
111
- world_size = get_tensor_model_parallel_world_size()
112
- # Bypass the function if we are using only 1 GPU.
113
- if world_size == 1:
114
- return input_
115
-
116
- dim_size = list(input_.size())
117
- assert dim_size[0] % world_size == 0, \
118
- "First dimension of the tensor should be divisible by tensor parallel size"
119
-
120
- dim_size[0] = dim_size[0] // world_size
121
-
122
- output = torch.empty(dim_size, dtype=input_.dtype,
123
- device=torch.cuda.current_device())
124
- torch.distributed._reduce_scatter_base(output, input_.contiguous(),
125
- group=get_tensor_model_parallel_group())
126
- return output
127
-
128
-
129
-class _CopyToModelParallelRegion(torch.autograd.Function):
130
- """Pass the input to the model parallel region."""
131
-
132
- @staticmethod
133
- def symbolic(graph, input_):
134
- return input_
135
-
136
- @staticmethod
137
- def forward(ctx, input_):
138
- return input_
139
-
140
- @staticmethod
141
- def backward(ctx, grad_output):
142
- return _reduce(grad_output)
143
-
144
-
145
-class _ReduceFromModelParallelRegion(torch.autograd.Function):
146
- """All-reduce the input from the model parallel region."""
147
-
148
- @staticmethod
149
- def symbolic(graph, input_):
150
- return _reduce(input_)
151
-
152
- @staticmethod
153
- def forward(ctx, input_):
154
- return _reduce(input_)
155
-
156
- @staticmethod
157
- def backward(ctx, grad_output):
158
- return grad_output
159
-
160
-
161
-class _ScatterToModelParallelRegion(torch.autograd.Function):
162
- """Split the input and keep only the corresponding chuck to the rank."""
163
-
164
- @staticmethod
165
- def symbolic(graph, input_):
166
- return _split_along_last_dim(input_)
167
-
168
- @staticmethod
169
- def forward(ctx, input_):
170
- return _split_along_last_dim(input_)
171
-
172
- @staticmethod
173
- def backward(ctx, grad_output):
174
- return _gather_along_last_dim(grad_output)
175
-
176
-
177
-class _GatherFromModelParallelRegion(torch.autograd.Function):
178
- """Gather the input from model parallel region and concatinate."""
179
-
180
- @staticmethod
181
- def symbolic(graph, input_):
182
- return _gather_along_last_dim(input_)
183
-
184
- @staticmethod
185
- def forward(ctx, input_):
186
- return _gather_along_last_dim(input_)
187
-
188
- @staticmethod
189
- def backward(ctx, grad_output):
190
- return _split_along_last_dim(grad_output)
191
-
192
-
193
-class _ScatterToSequenceParallelRegion(torch.autograd.Function):
194
- """Split the input and keep only the corresponding chuck to the rank."""
195
-
196
- @staticmethod
197
- def symbolic(graph, input_):
198
- return _split_along_first_dim(input_)
199
-
200
- @staticmethod
201
- def forward(ctx, input_):
202
- return _split_along_first_dim(input_)
203
-
204
- @staticmethod
205
- def backward(ctx, grad_output):
206
- return _gather_along_first_dim(grad_output)
207
-
208
-
209
-class _GatherFromSequenceParallelRegion(torch.autograd.Function):
210
- """Gather the input from sequence parallel region and concatinate."""
211
-
212
- @staticmethod
213
- def symbolic(graph, input_, tensor_parallel_output_grad=True):
214
- return _gather_along_first_dim(input_)
215
-
216
- @staticmethod
217
- def forward(ctx, input_, tensor_parallel_output_grad=True):
218
- ctx.tensor_parallel_output_grad = tensor_parallel_output_grad
219
- return _gather_along_first_dim(input_)
220
-
221
- @staticmethod
222
- def backward(ctx, grad_output):
223
- tensor_parallel_output_grad = ctx.tensor_parallel_output_grad
224
-
225
- # If the computation graph after the gather operation is
226
- # in the tensor parallel mode, output gradients need to reduce
227
- # scattered and whereas if the computation is duplicated,
228
- # output gradients need to be scattered.
229
- if tensor_parallel_output_grad:
230
- return _reduce_scatter_along_first_dim(grad_output), None
231
- else:
232
- return _split_along_first_dim(grad_output), None
233
-
234
-
235
-class _ReduceScatterToSequenceParallelRegion(torch.autograd.Function):
236
- """Reduce scatter the input from the model parallel region."""
237
-
238
- @staticmethod
239
- def symbolic(graph, input_):
240
- return _reduce_scatter_along_first_dim(input_)
241
-
242
- @staticmethod
243
- def forward(ctx, input_):
244
- return _reduce_scatter_along_first_dim(input_)
245
-
246
- @staticmethod
247
- def backward(ctx, grad_output):
248
- return _gather_along_first_dim(grad_output)
249
-
250
-
251
-# -----------------
252
-# Helper functions.
253
-# -----------------
254
-
255
-def copy_to_tensor_model_parallel_region(input_):
256
- return _CopyToModelParallelRegion.apply(input_)
257
-
258
-
259
-def reduce_from_tensor_model_parallel_region(input_):
260
- return _ReduceFromModelParallelRegion.apply(input_)
261
-
262
-
263
-def scatter_to_tensor_model_parallel_region(input_):
264
- return _ScatterToModelParallelRegion.apply(input_)
265
-
266
-
267
-def gather_from_tensor_model_parallel_region(input_):
268
- return _GatherFromModelParallelRegion.apply(input_)
269
-
270
-
271
-def scatter_to_sequence_parallel_region(input_):
272
- return _ScatterToSequenceParallelRegion.apply(input_)
273
-
274
-
275
-def gather_from_sequence_parallel_region(input_, tensor_parallel_output_grad=True):
276
- return _GatherFromSequenceParallelRegion.apply(input_, tensor_parallel_output_grad)
277
-
278
-
279
-def reduce_scatter_to_sequence_parallel_region(input_):
280
- return _ReduceScatterToSequenceParallelRegion.apply(input_)
281
-
vllm/model_executor/parallel_utils/tensor_parallel/random.py
deleted
-164
@@ -1,164 +0,0 @@
1
-# Copyright 2023 The vLLM team.
2
-# Adapted from https://github.com/NVIDIA/Megatron-LM/blob/main/megatron/core/tensor_parallel/random.py
3
-# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
4
-
5
-# Parts of the code here are adapted from PyTorch
6
-# repo: https://github.com/pytorch/pytorch
7
-
8
-import contextlib
9
-
10
-import torch
11
-from torch import _C
12
-from torch.cuda import _lazy_call, device as device_ctx_manager
13
-
14
-from vllm.model_executor.parallel_utils.parallel_state import (
15
- get_tensor_model_parallel_rank,
16
-)
17
-
18
-# Default name for the model parallel rng tracker.
19
-_MODEL_PARALLEL_RNG_TRACKER_NAME = 'model-parallel-rng'
20
-
21
-
22
-def _set_cuda_rng_state(new_state, device=-1):
23
- """Sets the random number generator state of the current GPU.
24
-
25
- Argumentss:
26
- new_state (torch.ByteTensor): The desired state
27
- This function is adapted from PyTorch repo (torch.cuda.set_rng_state)
28
- with a single change: the input state is not cloned. Cloning caused
29
- major performance issues for +4 GPU cases.
30
- """
31
- if hasattr(_C, '_cuda_setRNGState') and callable(_C._cuda_setRNGState):
32
- # older PyTorch
33
- def cb():
34
- with device_ctx_manager(device):
35
- _C._cuda_setRNGState(new_state)
36
- else:
37
- # newer PyTorch
38
- if device == -1:
39
- device = torch.device('cuda')
40
- elif isinstance(device, str):
41
- device = torch.device(device)
42
- elif isinstance(device, int):
43
- device = torch.device('cuda', device)
44
-
45
- def cb():
46
- idx = device.index
47
- if idx is None:
48
- idx = torch.cuda.current_device()
49
- default_generator = torch.cuda.default_generators[idx]
50
- default_generator.set_state(new_state)
51
-
52
- _lazy_call(cb)
53
-
54
-
55
-
56
-class CudaRNGStatesTracker:
57
- """Tracker for the cuda RNG states.
58
-
59
- Using the `add` method, a cuda rng state is initialized based on
60
- the input `seed` and is assigned to `name`. Later, by forking the
61
- rng state, we can perform operations and return to our starting
62
- cuda state.
63
- """
64
-
65
- def __init__(self):
66
- # Map from a string name to the cuda rng state.
67
- self.states_ = {}
68
- # Seeds are just for book keeping and ensure no seed is set twice.
69
- self.seeds_ = set()
70
-
71
- def reset(self):
72
- """Set to the initial state (no tracker)."""
73
- self.states_ = {}
74
- self.seeds_ = set()
75
-
76
- def get_states(self):
77
- """Get rng states. Copy the dictionary so we have direct
78
- pointers to the states, not just a pointer to the dictionary."""
79
- states = {}
80
- for name in self.states_:
81
- states[name] = self.states_[name]
82
- return states
83
-
84
- def set_states(self, states):
85
- """Set the rng states. For efficiency purposes, we do not check
86
- the size of seed for compatibility."""
87
- self.states_ = states
88
-
89
- def add(self, name, seed):
90
- """Track the rng state."""
91
- # Check seed is not already used.
92
- if seed in self.seeds_:
93
- raise Exception('seed {} already exists'.format(seed))
94
- self.seeds_.add(seed)
95
- # Check that state is not already defined.
96
- if name in self.states_:
97
- raise Exception('cuda rng state {} already exists'.format(name))
98
- # Get the current rng state.
99
- orig_rng_state = torch.cuda.get_rng_state()
100
- # Set the new state and store it.
101
- torch.cuda.manual_seed(seed)
102
- self.states_[name] = torch.cuda.get_rng_state()
103
- # Reset rng state to what it was.
104
- _set_cuda_rng_state(orig_rng_state)
105
-
106
- @contextlib.contextmanager
107
- def fork(self, name=_MODEL_PARALLEL_RNG_TRACKER_NAME):
108
- """Fork the cuda rng state, perform operations, and exit with
109
- the original state."""
110
- # Check if we have added the state
111
- if name not in self.states_:
112
- raise Exception('cuda rng state {} is not added'.format(name))
113
- # Store current rng state.
114
- orig_cuda_rng_state = torch.cuda.get_rng_state()
115
- # Set rng state to the desired one
116
- _set_cuda_rng_state(self.states_[name])
117
- # Do the stuff we wanted to do.
118
- try:
119
- yield
120
- finally:
121
- # Update the current rng state for later use.
122
- self.states_[name] = torch.cuda.get_rng_state()
123
- # And set the state to the original state we started with.
124
- _set_cuda_rng_state(orig_cuda_rng_state)
125
-
126
-
127
-# RNG tracker object.
128
-_CUDA_RNG_STATE_TRACKER = CudaRNGStatesTracker()
129
-
130
-
131
-def get_cuda_rng_tracker():
132
- """Get cuda rng tracker."""
133
- return _CUDA_RNG_STATE_TRACKER
134
-
135
-
136
-def model_parallel_cuda_manual_seed(seed):
137
- """Initialize model parallel cuda seed.
138
-
139
- This function should be called after the model parallel is
140
- initialized. Also, no torch.cuda.manual_seed should be called
141
- after this function. Basically, this is replacement for that
142
- function.
143
- Two set of RNG states are tracked:
144
- default state: This is for data parallelism and is the same among a
145
- set of model parallel GPUs but different across
146
- different model paralle groups. This is used for
147
- example for dropout in the non-tensor-model-parallel regions.
148
- tensor-model-parallel state: This state is different among a set of model
149
- parallel GPUs, but the same across data parallel
150
- groups. This is used for example for dropout in
151
- model parallel regions.
152
- """
153
- # 2718 is just for fun and any POSITIVE value will work.
154
- offset = seed + 2718
155
- tensor_model_parallel_seed = offset + get_tensor_model_parallel_rank()
156
- # Data parallel gets the original seed.
157
- data_parallel_seed = seed
158
-
159
- _CUDA_RNG_STATE_TRACKER.reset()
160
- # Set the default state.
161
- torch.cuda.manual_seed(data_parallel_seed)
162
- # and model parallel state.
163
- _CUDA_RNG_STATE_TRACKER.add(_MODEL_PARALLEL_RNG_TRACKER_NAME,
164
- tensor_model_parallel_seed)
vllm/model_executor/parallel_utils/utils.py
renamed
+9
-9
@@ -1,15 +1,16 @@
1
# Copyright 2023 The vLLM team.
2
-# Adapted from https://github.com/NVIDIA/Megatron-LM/blob/main/megatron/core/tensor_parallel/utils.py
2
+# Adapted from
3
+# https://github.com/NVIDIA/Megatron-LM/blob/main/megatron/core/tensor_parallel/utils.py
4
# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
5
+from typing import List, Sequence
6
7
import torch
6
-from typing import List, Sequence
8
+
9
10
def ensure_divisibility(numerator, denominator):
11
"""Ensure that numerator is divisible by the denominator."""
12
assert numerator % denominator == 0, "{} is not divisible by {}".format(
11
- numerator, denominator
12
- )
13
+ numerator, denominator)
14
15
16
def divide(numerator, denominator):
@@ -56,15 +57,14 @@ class VocabUtility:
57
58
@staticmethod
59
def vocab_range_from_per_partition_vocab_size(
59
- per_partition_vocab_size: int, rank, world_size: int
60
- ) -> Sequence[int]:
60
+ per_partition_vocab_size: int, rank: int) -> Sequence[int]:
61
index_f = rank * per_partition_vocab_size
62
index_l = index_f + per_partition_vocab_size
63
return index_f, index_l
64
65
@staticmethod
66
- def vocab_range_from_global_vocab_size(global_vocab_size: int, rank: int, world_size: int) -> Sequence[int]:
66
+ def vocab_range_from_global_vocab_size(global_vocab_size: int, rank: int,
67
+ world_size: int) -> Sequence[int]:
68
per_partition_vocab_size = divide(global_vocab_size, world_size)
69
return VocabUtility.vocab_range_from_per_partition_vocab_size(
69
- per_partition_vocab_size, rank, world_size
70
- )
70
+ per_partition_vocab_size, rank)
vllm/model_executor/utils.py
-6
@@ -4,9 +4,6 @@ import random
4
import numpy as np
5
import torch
6
7
-from vllm.model_executor.parallel_utils.parallel_state import model_parallel_is_initialized
8
-from vllm.model_executor.parallel_utils.tensor_parallel import model_parallel_cuda_manual_seed
9
-
7
8
def set_random_seed(seed: int) -> None:
9
random.seed(seed)
@@ -14,6 +11,3 @@ def set_random_seed(seed: int) -> None:
11
torch.manual_seed(seed)
12
if torch.cuda.is_available():
13
torch.cuda.manual_seed_all(seed)
17
-
18
- if model_parallel_is_initialized():
19
- model_parallel_cuda_manual_seed(seed)