@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / ba0bfd40e2
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

TP/quantization/weight loading refactor part 1 - Simplify parallel linear logic (#1181)

Zhuohan Li committed Oct 2, 2023 at 15:36 UTC ba0bfd40e21cacfd5da6a1e43028a37258a29cb4
42 files changed +823 -1551
.github/workflows/pylint.yml
+1 -1
@@ -28,4 +28,4 @@ jobs:
28 pip install pylint==2.8.2
29 - name: Analysing the code with pylint
30 run: |
31 - pylint vllm
31 + pylint vllm tests
.github/workflows/yapf.yml
+1 -1
@@ -28,4 +28,4 @@ jobs:
28 pip install toml==0.10.2
29 - name: Running yapf
30 run: |
31 - yapf --diff --recursive vllm --exclude 'vllm/model_executor/parallel_utils/**'
31 + yapf --diff --recursive vllm tests
.pylintrc
+1 -1
@@ -8,7 +8,7 @@
8 [MASTER]
9
10 # Files or directories to be skipped. They should be base names, not paths.
11 -ignore=docs,parallel_utils
11 +ignore=docs
12
13 # Files or directories matching the regex patterns are skipped. The regex
14 # matches against base names, not paths.
format.sh
+2 -3
@@ -44,7 +44,6 @@ YAPF_FLAGS=(
44
45 YAPF_EXCLUDES=(
46 '--exclude' 'build/**'
47 - '--exclude' 'vllm/model_executor/parallel_utils/**'
47 )
48
49 # Format specified files
@@ -72,7 +71,7 @@ format_changed() {
71
72 # Format all files
73 format_all() {
75 - yapf --in-place "${YAPF_FLAGS[@]}" "${YAPF_EXCLUDES[@]}" vllm
74 + yapf --in-place "${YAPF_FLAGS[@]}" "${YAPF_EXCLUDES[@]}" vllm tests
75 }
76
77 ## This flag formats individual files. --files *must* be the first command line
@@ -96,7 +95,7 @@ echo 'vLLM yapf: Done'
95
96 # Run Pylint
97 echo 'vLLM Pylint:'
99 -pylint vllm
98 +pylint vllm tests
99
100 if ! git diff --quiet &>/dev/null; then
101 echo 'Reformatted files. Please review and stage the changes.'
tests/async_engine/api_server_async_engine.py
+1
@@ -14,6 +14,7 @@ app = vllm.entrypoints.api_server.app
14
15 class AsyncLLMEngineWithStats(AsyncLLMEngine):
16
17 + # pylint: disable=redefined-outer-name
18 def __init__(self, *args, **kwargs):
19 super().__init__(*args, **kwargs)
20 self._num_aborts = 0
tests/async_engine/test_api_server.py
+3
@@ -24,6 +24,7 @@ def _query_server(prompt: str) -> dict:
24 def api_server():
25 script_path = Path(__file__).parent.joinpath(
26 "api_server_async_engine.py").absolute()
27 + # pylint: disable=consider-using-with
28 uvicorn_process = subprocess.Popen([
29 sys.executable, "-u",
30 str(script_path), "--model", "facebook/opt-125m"
@@ -32,6 +33,7 @@ def api_server():
33 uvicorn_process.terminate()
34
35
36 +# pylint: disable=redefined-outer-name, unused-argument
37 def test_api_server(api_server):
38 """
39 Run the API server and test it.
@@ -47,6 +49,7 @@ def test_api_server(api_server):
49 prompts = ["Hello world"] * 1
50 result = None
51 while not result:
52 + # pylint: disable=bare-except
53 try:
54 for result in pool.map(_query_server, prompts):
55 break
tests/async_engine/test_async_llm_engine.py
+2 -2
@@ -32,12 +32,12 @@ class MockEngine:
32 self.request_id = None
33
34 def add_request(self, **kwargs):
35 + del kwargs # Unused
36 self.add_request_calls += 1
36 - return
37
38 def abort_request(self, request_id):
39 + del request_id # Unused
40 self.abort_request_calls += 1
40 - return
41
42
43 class MockAsyncLLMEngine(AsyncLLMEngine):
tests/async_engine/test_request_tracker.py
+11 -11
@@ -7,22 +7,22 @@ from vllm.outputs import RequestOutput
7 class DummyEvent:
8
9 def __init__(self):
10 - self._flag = False
10 + self.flag = False
11
12 def set(self):
13 - self._flag = True
13 + self.flag = True
14
15 def clear(self):
16 - self._flag = False
16 + self.flag = False
17
18
19 def test_request_tracker():
20 tracker = RequestTracker()
21 tracker.new_requests_event = DummyEvent()
22 stream_1 = tracker.add_request("1")
23 - assert tracker.new_requests_event._flag
23 + assert tracker.new_requests_event.flag
24 new, finished = tracker.get_new_and_finished_requests()
25 - assert not tracker.new_requests_event._flag
25 + assert not tracker.new_requests_event.flag
26 assert len(new) == 1
27 assert new[0]["request_id"] == "1"
28 assert not finished
@@ -30,9 +30,9 @@ def test_request_tracker():
30
31 stream_2 = tracker.add_request("2")
32 stream_3 = tracker.add_request("3")
33 - assert tracker.new_requests_event._flag
33 + assert tracker.new_requests_event.flag
34 new, finished = tracker.get_new_and_finished_requests()
35 - assert not tracker.new_requests_event._flag
35 + assert not tracker.new_requests_event.flag
36 assert len(new) == 2
37 assert new[0]["request_id"] == "2"
38 assert new[1]["request_id"] == "3"
@@ -43,7 +43,7 @@ def test_request_tracker():
43 # request_ids must be unique
44 with pytest.raises(KeyError):
45 tracker.add_request("1")
46 - assert not tracker.new_requests_event._flag
46 + assert not tracker.new_requests_event.flag
47
48 tracker.abort_request("1")
49 new, finished = tracker.get_new_and_finished_requests()
@@ -54,7 +54,7 @@ def test_request_tracker():
54
55 stream_4 = tracker.add_request("4")
56 tracker.abort_request("4")
57 - assert tracker.new_requests_event._flag
57 + assert tracker.new_requests_event.flag
58 new, finished = tracker.get_new_and_finished_requests()
59 assert len(finished) == 1
60 assert "4" in finished
@@ -62,11 +62,11 @@ def test_request_tracker():
62 assert stream_4.finished
63
64 stream_5 = tracker.add_request("5")
65 - assert tracker.new_requests_event._flag
65 + assert tracker.new_requests_event.flag
66 tracker.process_request_output(
67 RequestOutput("2", "output", [], [], finished=True))
68 new, finished = tracker.get_new_and_finished_requests()
69 - assert not tracker.new_requests_event._flag
69 + assert not tracker.new_requests_event.flag
70 assert len(finished) == 1
71 assert "2" in finished
72 assert len(new) == 1
tests/conftest.py
+1
@@ -8,6 +8,7 @@ from vllm import LLM, SamplingParams
8 from vllm.transformers_utils.tokenizer import get_tokenizer
9
10 _TEST_PROMPTS = [
11 + # pylint: disable=line-too-long
12 "vLLM is a high-throughput and memory-efficient inference and serving engine for LLMs.",
13 "Briefly describe the major milestones in the development of artificial intelligence from 1950 to 2020.",
14 "Compare and contrast artificial intelligence with human intelligence in terms of processing information.",
tests/distributed/test_comm_ops.py new
+82
@@ -0,0 +1,82 @@
1 +"""Test the communication operators.
2 +
3 +Run `pytest tests/distributed/test_comm_ops.py --forked`.
4 +"""
5 +from multiprocessing import Process
6 +
7 +import pytest
8 +import torch
9 +
10 +from vllm.config import ParallelConfig
11 +from vllm.engine.ray_utils import get_open_port
12 +from vllm.model_executor.parallel_utils.communication_op import (
13 + tensor_model_parallel_all_reduce,
14 + tensor_model_parallel_all_gather,
15 +)
16 +from vllm.worker.worker import _init_distributed_environment
17 +
18 +
19 +def init_test_distributed_environment(pipeline_parallel_size: int,
20 + tensor_parallel_size: int, rank: int,
21 + distributed_init_port: str):
22 + parallel_config = ParallelConfig(pipeline_parallel_size,
23 + tensor_parallel_size,
24 + worker_use_ray=True)
25 + distributed_init_method = f"tcp://localhost:{distributed_init_port}"
26 + torch.cuda.set_device(rank)
27 + _init_distributed_environment(parallel_config, rank,
28 + distributed_init_method)
29 +
30 +
31 +def all_reduce_test_worker(tensor_parallel_size: int, rank: int,
32 + distributed_init_port: str):
33 + init_test_distributed_environment(1, tensor_parallel_size, rank,
34 + distributed_init_port)
35 + num_elements = 8
36 + all_tensors = [
37 + torch.arange(num_elements, dtype=torch.float32, device="cuda") *
38 + (r + 1) for r in range(tensor_parallel_size)
39 + ]
40 + expected = torch.sum(torch.stack(all_tensors, dim=0), dim=0)
41 + t = all_tensors[rank]
42 + t = tensor_model_parallel_all_reduce(t)
43 + assert torch.allclose(t, expected)
44 +
45 +
46 +def all_gather_test_worker(tensor_parallel_size: int, rank: int,
47 + distributed_init_port: str):
48 + init_test_distributed_environment(1, tensor_parallel_size, rank,
49 + distributed_init_port)
50 + num_dimensions = 3
51 + tensor_size = list(range(2, num_dimensions + 2))
52 + total_size = 1
53 + for s in tensor_size:
54 + total_size *= s
55 + for all_gather_dimension in range(num_dimensions):
56 + all_tensors = [
57 + torch.arange(total_size, dtype=torch.float32,
58 + device="cuda").reshape(tensor_size) * (r + 1)
59 + for r in range(tensor_parallel_size)
60 + ]
61 + expected = torch.cat(all_tensors, dim=all_gather_dimension)
62 + t = all_tensors[rank]
63 + t = tensor_model_parallel_all_gather(t, all_gather_dimension)
64 + assert torch.allclose(t, expected)
65 +
66 +
67 +@pytest.mark.skipif(torch.cuda.device_count() < 2,
68 + reason="Need at least 2 GPUs to run the test.")
69 +@pytest.mark.parametrize("tensor_parallel_size", [2])
70 +@pytest.mark.parametrize("test_target",
71 + [all_reduce_test_worker, all_gather_test_worker])
72 +def test_multi_process_tensor_parallel(tensor_parallel_size, test_target):
73 + distributed_init_port = get_open_port()
74 + processes = []
75 + for rank in range(tensor_parallel_size):
76 + p = Process(target=test_target,
77 + args=(tensor_parallel_size, rank, distributed_init_port))
78 + p.start()
79 + processes.append(p)
80 + for p in processes:
81 + p.join()
82 + assert all(p.exitcode == 0 for p in processes)
tests/engine/test_detokenize.py
+1
@@ -5,6 +5,7 @@ from transformers import AutoTokenizer
5 from vllm.transformers_utils.tokenizer import detokenize_incrementally
6
7 TRUTH = [
8 + # pylint: disable=line-too-long
9 "Hello here, this is a simple test",
10 "vLLM is a high-throughput and memory-efficient inference and serving engine for LLMs. It is designed to be used in production environments, where inference and serving",
11 "我很感谢你的热情"
tests/kernels/test_activation.py
+6 -6
@@ -29,8 +29,8 @@ def test_silu_and_mul(
29 ) -> None:
30 torch.random.manual_seed(seed)
31 torch.cuda.manual_seed(seed)
32 - x = torch.randn(num_tokens, 2 * d, dtype=dtype, device='cuda')
33 - out = torch.empty(num_tokens, d, dtype=dtype, device='cuda')
32 + x = torch.randn(num_tokens, 2 * d, dtype=dtype, device="cuda")
33 + out = torch.empty(num_tokens, d, dtype=dtype, device="cuda")
34 activation_ops.silu_and_mul(out, x)
35 ref_out = ref_silu_and_mul(x)
36 assert torch.allclose(out, ref_out, atol=1e-5, rtol=1e-5)
@@ -49,8 +49,8 @@ def test_gelu_new(
49 ) -> None:
50 torch.random.manual_seed(seed)
51 torch.cuda.manual_seed(seed)
52 - x = torch.randn(num_tokens, d, dtype=dtype, device='cuda')
53 - out = torch.empty(num_tokens, d, dtype=dtype, device='cuda')
52 + x = torch.randn(num_tokens, d, dtype=dtype, device="cuda")
53 + out = torch.empty(num_tokens, d, dtype=dtype, device="cuda")
54 activation_ops.gelu_new(out, x)
55 ref_out = get_activation("gelu_new")(x)
56 assert torch.allclose(out, ref_out, atol=1e-5, rtol=1e-5)
@@ -68,8 +68,8 @@ def test_gelu_fast(
68 ) -> None:
69 torch.random.manual_seed(seed)
70 torch.cuda.manual_seed(seed)
71 - x = torch.randn(num_tokens, d, dtype=dtype, device='cuda')
72 - out = torch.empty(num_tokens, d, dtype=dtype, device='cuda')
71 + x = torch.randn(num_tokens, d, dtype=dtype, device="cuda")
72 + out = torch.empty(num_tokens, d, dtype=dtype, device="cuda")
73 activation_ops.gelu_fast(out, x)
74 ref_out = get_activation("gelu_fast")(x)
75 assert torch.allclose(out, ref_out, atol=1e-5, rtol=1e-5)
tests/kernels/test_cache.py
+3 -3
@@ -106,14 +106,14 @@ def test_reshape_and_cache(
106 # Create a random slot mapping.
107 num_slots = block_size * num_blocks
108 slot_mapping = random.sample(range(num_slots), num_tokens)
109 - slot_mapping = torch.tensor(slot_mapping, dtype=torch.int, device='cuda')
109 + slot_mapping = torch.tensor(slot_mapping, dtype=torch.int, device="cuda")
110
111 qkv = torch.randn(num_tokens,
112 3,
113 num_heads,
114 head_size,
115 dtype=dtype,
116 - device='cuda')
116 + device="cuda")
117 _, key, value = qkv.unbind(dim=1)
118
119 # Create the KV caches.
@@ -132,7 +132,7 @@ def test_reshape_and_cache(
132
133 # Run the reference implementation.
134 reshaped_key = key.reshape(num_tokens, *key_cache[0, :, :, 0, :].shape)
135 - block_indicies = torch.div(slot_mapping, block_size, rounding_mode='floor')
135 + block_indicies = torch.div(slot_mapping, block_size, rounding_mode="floor")
136 block_indicies = block_indicies.cpu().tolist()
137 block_offsets = slot_mapping % block_size
138 block_offsets = block_offsets.cpu().tolist()
tests/kernels/test_pos_encoding.py
+1 -1
@@ -140,7 +140,7 @@ def test_rotary_embedding(
140 cos = freqs.cos()
141 sin = freqs.sin()
142 cos_sin_cache = torch.cat((cos, sin), dim=-1)
143 - cos_sin_cache = cos_sin_cache.to(dtype=dtype, device='cuda')
143 + cos_sin_cache = cos_sin_cache.to(dtype=dtype, device="cuda")
144
145 # Run the kernel. The kernel is in-place, so we need to clone the inputs.
146 out_query = query.clone()
tests/samplers/test_sampler.py
+2 -1
@@ -1,3 +1,4 @@
1 +# pylint: disable=protected-access
2 import pytest
3 import random
4 from typing import Tuple
@@ -108,7 +109,7 @@ def test_sampler_all_random(seed: int):
109 def test_sampler_all_beam(seed: int):
110 set_random_seed(seed)
111 batch_size = random.randint(1, 256)
111 - input_tensor, fake_logits, sampler, worker = _prepare_test(batch_size)
112 + input_tensor, _, sampler, worker = _prepare_test(batch_size)
113
114 seq_group_metadata_list = []
115 for i in range(batch_size):
vllm/model_executor/layers/quantized_linear/__init__.py
+2 -2
@@ -1,7 +1,7 @@
1 from vllm.model_executor.layers.quantized_linear.awq import (
2 AWQColumnParallelLinear, AWQRowParallelLinear)
3 -from vllm.model_executor.parallel_utils.tensor_parallel import (
4 - ColumnParallelLinear, RowParallelLinear)
3 +from vllm.model_executor.parallel_utils.layers import (ColumnParallelLinear,
4 + RowParallelLinear)
5
6 _QUANTIZED_LINEAR_REGISTRY = {
7 "awq": (AWQColumnParallelLinear, AWQRowParallelLinear),
vllm/model_executor/layers/quantized_linear/awq.py
+2 -2
@@ -4,8 +4,8 @@ import torch
4 from torch.nn.parameter import Parameter
5
6 from vllm import quantization_ops
7 -from vllm.model_executor.parallel_utils.tensor_parallel.layers import (
8 - ColumnParallelLinear, RowParallelLinear)
7 +from vllm.model_executor.parallel_utils.layers import (ColumnParallelLinear,
8 + RowParallelLinear)
9
10
11 class AWQColumnParallelLinear(ColumnParallelLinear):
vllm/model_executor/layers/sampler.py
+3 -3
@@ -5,8 +5,8 @@ import torch
5 import torch.nn as nn
6
7 from vllm.model_executor.input_metadata import InputMetadata
8 -from vllm.model_executor.parallel_utils.tensor_parallel import (
9 - gather_from_tensor_model_parallel_region)
8 +from vllm.model_executor.parallel_utils.communication_op import (
9 + tensor_model_parallel_all_gather)
10 from vllm.sampling_params import SamplingParams, SamplingType
11 from vllm.sequence import SamplerOutput, SequenceData, SequenceOutputs
12
@@ -92,7 +92,7 @@ def _get_logits(hidden_states: torch.Tensor, embedding: torch.Tensor,
92 logits = torch.matmul(hidden_states, embedding.t())
93 if embedding_bias is not None:
94 logits += embedding_bias
95 - logits = gather_from_tensor_model_parallel_region(logits)
95 + logits = tensor_model_parallel_all_gather(logits)
96 # Remove paddings in vocab (if any).
97 logits = logits[:, :vocab_size]
98 return logits
vllm/model_executor/models/aquila.py
+22 -20
@@ -39,8 +39,9 @@ from vllm.model_executor.weight_utils import (
39 load_tensor_parallel_weights)
40 from vllm.model_executor.parallel_utils.parallel_state import (
41 get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
42 -from vllm.model_executor.parallel_utils.tensor_parallel import (
43 - VocabParallelEmbedding, ColumnParallelLinear, RowParallelLinear)
42 +from vllm.model_executor.parallel_utils.layers import (VocabParallelEmbedding,
43 + ColumnParallelLinear,
44 + RowParallelLinear)
45 from vllm.sequence import SamplerOutput
46 from vllm.transformers_utils.configs.aquila import AquilaConfig
47
@@ -56,16 +57,18 @@ class AquilaMLP(nn.Module):
57 hidden_act: str,
58 ):
59 super().__init__()
59 - self.gate_up_proj = ColumnParallelLinear(hidden_size,
60 - 2 * intermediate_size,
61 - bias=False,
62 - gather_output=False,
63 - perform_initialization=False)
64 - self.down_proj = RowParallelLinear(intermediate_size,
65 - hidden_size,
66 - bias=False,
67 - input_is_parallel=True,
68 - perform_initialization=False)
60 + self.gate_up_proj = ColumnParallelLinear(
61 + hidden_size,
62 + 2 * intermediate_size,
63 + bias=False,
64 + gather_output=False,
65 + )
66 + self.down_proj = RowParallelLinear(
67 + intermediate_size,
68 + hidden_size,
69 + bias=False,
70 + input_is_parallel=True,
71 + )
72 if hidden_act != "silu":
73 raise ValueError(f"Unsupported activation: {hidden_act}. "
74 "Only silu is supported for now.")
@@ -130,14 +133,12 @@ class AquilaAttention(nn.Module):
133 self.head_dim,
134 bias=False,
135 gather_output=False,
133 - perform_initialization=False,
136 )
137 self.o_proj = RowParallelLinear(
138 self.total_num_heads * self.head_dim,
139 hidden_size,
140 bias=False,
141 input_is_parallel=True,
140 - perform_initialization=False,
142 )
143 self.attn = PagedAttentionWithRoPE(
144 self.num_heads,
@@ -230,7 +231,7 @@ class AquilaModel(nn.Module):
231 self.embed_tokens = VocabParallelEmbedding(
232 config.vocab_size,
233 config.hidden_size,
233 - perform_initialization=False)
234 + )
235 self.layers = nn.ModuleList([
236 AquilaDecoderLayer(config) for _ in range(config.num_hidden_layers)
237 ])
@@ -270,11 +271,12 @@ class AquilaForCausalLM(nn.Module):
271 self.config = config
272 self.model = AquilaModel(config)
273 vocab_size = ((config.vocab_size + 63) // 64) * 64
273 - self.lm_head = ColumnParallelLinear(config.hidden_size,
274 - vocab_size,
275 - bias=False,
276 - gather_output=False,
277 - perform_initialization=False)
274 + self.lm_head = ColumnParallelLinear(
275 + config.hidden_size,
276 + vocab_size,
277 + bias=False,
278 + gather_output=False,
279 + )
280 self.sampler = Sampler(config.vocab_size)
281
282 def forward(
vllm/model_executor/models/baichuan.py
+22 -20
@@ -39,8 +39,9 @@ from vllm.model_executor.weight_utils import (
39 load_padded_tensor_parallel_vocab, load_tensor_parallel_weights)
40 from vllm.model_executor.parallel_utils.parallel_state import (
41 get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
42 -from vllm.model_executor.parallel_utils.tensor_parallel import (
43 - VocabParallelEmbedding, ColumnParallelLinear, RowParallelLinear)
42 +from vllm.model_executor.parallel_utils.layers import (VocabParallelEmbedding,
43 + ColumnParallelLinear,
44 + RowParallelLinear)
45 from vllm.sequence import SamplerOutput
46 from vllm.transformers_utils.configs.baichuan import BaiChuanConfig
47
@@ -81,16 +82,18 @@ class BaiChuanMLP(nn.Module):
82 hidden_act: str,
83 ):
84 super().__init__()
84 - self.gate_up_proj = ColumnParallelLinear(hidden_size,
85 - 2 * intermediate_size,
86 - bias=False,
87 - gather_output=False,
88 - perform_initialization=False)
89 - self.down_proj = RowParallelLinear(intermediate_size,
90 - hidden_size,
91 - bias=False,
92 - input_is_parallel=True,
93 - perform_initialization=False)
85 + self.gate_up_proj = ColumnParallelLinear(
86 + hidden_size,
87 + 2 * intermediate_size,
88 + bias=False,
89 + gather_output=False,
90 + )
91 + self.down_proj = RowParallelLinear(
92 + intermediate_size,
93 + hidden_size,
94 + bias=False,
95 + input_is_parallel=True,
96 + )
97 if hidden_act != "silu":
98 raise ValueError(f"Unsupported activation: {hidden_act}. "
99 "Only silu is supported for now.")
@@ -133,14 +136,12 @@ class BaiChuanAttention(nn.Module):
136 3 * hidden_size,
137 bias=False,
138 gather_output=False,
136 - perform_initialization=False,
139 )
140 self.o_proj = RowParallelLinear(
141 self.total_num_heads * self.head_dim,
142 hidden_size,
143 bias=False,
144 input_is_parallel=True,
143 - perform_initialization=False,
145 )
146 # Create the alibi slopes and slice them.
147 if self.postion_embedding == "ALIBI":
@@ -249,7 +250,7 @@ class BaiChuanModel(nn.Module):
250 self.embed_tokens = VocabParallelEmbedding(
251 config.vocab_size,
252 config.hidden_size,
252 - perform_initialization=False)
253 + )
254 self.layers = nn.ModuleList([
255 BaiChuanDecoderLayer(config, position_embedding)
256 for _ in range(config.num_hidden_layers)
@@ -288,11 +289,12 @@ class BaiChuanBaseForCausalLM(nn.Module):
289 super().__init__()
290 self.config = config
291 self.model = BaiChuanModel(config, position_embedding)
291 - self.lm_head = ColumnParallelLinear(config.hidden_size,
292 - config.vocab_size,
293 - bias=False,
294 - gather_output=False,
295 - perform_initialization=False)
292 + self.lm_head = ColumnParallelLinear(
293 + config.hidden_size,
294 + config.vocab_size,
295 + bias=False,
296 + gather_output=False,
297 + )
298 self.sampler = Sampler(config.vocab_size)
299
300 def forward(
vllm/model_executor/models/bloom.py
+16 -13
@@ -35,8 +35,9 @@ from vllm.model_executor.weight_utils import (hf_model_weights_iterator,
35 load_tensor_parallel_weights)
36 from vllm.model_executor.parallel_utils.parallel_state import (
37 get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
38 -from vllm.model_executor.parallel_utils.tensor_parallel import (
39 - VocabParallelEmbedding, ColumnParallelLinear, RowParallelLinear)
38 +from vllm.model_executor.parallel_utils.layers import (VocabParallelEmbedding,
39 + ColumnParallelLinear,
40 + RowParallelLinear)
41 from vllm.sequence import SamplerOutput
42
43 KVCache = Tuple[torch.Tensor, torch.Tensor]
@@ -85,14 +86,12 @@ class BloomAttention(nn.Module):
86 3 * self.hidden_size,
87 bias=True,
88 gather_output=False,
88 - perform_initialization=False,
89 )
90 self.dense = RowParallelLinear(
91 self.hidden_size,
92 self.hidden_size,
93 bias=True,
94 input_is_parallel=True,
95 - perform_initialization=False,
95 )
96
97 # Create the alibi slopes and slice them.
@@ -129,15 +128,17 @@ class BloomMLP(nn.Module):
128 def __init__(self, config: BloomConfig):
129 super().__init__()
130 hidden_size = config.hidden_size
132 - self.dense_h_to_4h = ColumnParallelLinear(hidden_size,
133 - 4 * hidden_size,
134 - gather_output=False,
135 - perform_initialization=False)
131 + self.dense_h_to_4h = ColumnParallelLinear(
132 + hidden_size,
133 + 4 * hidden_size,
134 + gather_output=False,
135 + )
136 self.act = get_act_fn("gelu")
137 - self.dense_4h_to_h = RowParallelLinear(4 * hidden_size,
138 - hidden_size,
139 - input_is_parallel=True,
140 - perform_initialization=False)
137 + self.dense_4h_to_h = RowParallelLinear(
138 + 4 * hidden_size,
139 + hidden_size,
140 + input_is_parallel=True,
141 + )
142
143 def forward(self, x: torch.Tensor) -> torch.Tensor:
144 x, _ = self.dense_h_to_4h(x)
@@ -208,7 +209,9 @@ class BloomModel(nn.Module):
209
210 # Embedding + LN Embedding
211 self.word_embeddings = VocabParallelEmbedding(
211 - config.vocab_size, self.embed_dim, perform_initialization=False)
212 + config.vocab_size,
213 + self.embed_dim,
214 + )
215 self.word_embeddings_layernorm = nn.LayerNorm(
216 self.embed_dim, eps=config.layer_norm_epsilon)
217
vllm/model_executor/models/falcon.py
+15 -16
@@ -36,9 +36,11 @@ from vllm.model_executor.weight_utils import (convert_pyslice_to_tensor,
36 load_tensor_parallel_weights)
37 from vllm.model_executor.parallel_utils.parallel_state import (
38 get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
39 -from vllm.model_executor.parallel_utils.tensor_parallel import (
40 - VocabParallelEmbedding, ColumnParallelLinear, RowParallelLinear,
41 - reduce_from_tensor_model_parallel_region)
39 +from vllm.model_executor.parallel_utils.layers import (VocabParallelEmbedding,
40 + ColumnParallelLinear,
41 + RowParallelLinear)
42 +from vllm.model_executor.parallel_utils.communication_op import (
43 + tensor_model_parallel_all_reduce)
44 from vllm.sequence import SamplerOutput
45 from vllm.transformers_utils.configs import RWConfig
46
@@ -109,7 +111,6 @@ class FalconAttention(nn.Module):
111 self.head_dim,
112 bias=config.bias,
113 gather_output=False,
112 - perform_initialization=False,
114 skip_bias_add=True,
115 )
116 elif self.multi_query:
@@ -120,7 +121,6 @@ class FalconAttention(nn.Module):
121 self.total_num_heads * self.head_dim,
122 bias=config.bias,
123 gather_output=False,
123 - perform_initialization=False,
124 skip_bias_add=True,
125 )
126 self.key_value = FalconLinear(self.hidden_size,
@@ -135,7 +135,6 @@ class FalconAttention(nn.Module):
135 self.head_dim,
136 bias=config.bias,
137 gather_output=False,
138 - perform_initialization=False,
138 skip_bias_add=True,
139 )
140
@@ -151,7 +150,6 @@ class FalconAttention(nn.Module):
150 self.hidden_size,
151 bias=config.bias,
152 input_is_parallel=True,
154 - perform_initialization=False,
153 skip_bias_add=True,
154 reduce_results=self.reduce_row_parallel_results)
155
@@ -231,7 +229,6 @@ class FalconMLP(nn.Module):
229 4 * hidden_size,
230 bias=config.bias,
231 gather_output=False,
234 - perform_initialization=False,
232 skip_bias_add=True)
233 self.act = nn.GELU()
234 self.reduce_row_parallel_results = not (config.new_decoder_architecture
@@ -241,7 +238,6 @@ class FalconMLP(nn.Module):
238 hidden_size,
239 bias=config.bias,
240 input_is_parallel=True,
244 - perform_initialization=False,
241 skip_bias_add=True,
242 reduce_results=self.reduce_row_parallel_results)
243
@@ -325,7 +321,7 @@ class FalconDecoderLayer(nn.Module):
321 # only one all-reduce operator to reduce the results from
322 # both MLP and Attention layers.
323 mlp_output += attention_output
328 - mlp_output = reduce_from_tensor_model_parallel_region(mlp_output)
324 + mlp_output = tensor_model_parallel_all_reduce(mlp_output)
325 if attention_bias is not None:
326 mlp_output += attention_bias
327 if mlp_bias is not None:
@@ -347,7 +343,9 @@ class FalconModel(nn.Module):
343
344 # Embedding + LN Embedding
345 self.word_embeddings = VocabParallelEmbedding(
350 - config.vocab_size, self.embed_dim, perform_initialization=False)
346 + config.vocab_size,
347 + self.embed_dim,
348 + )
349
350 # Transformer blocks
351 self.h = nn.ModuleList([
@@ -389,11 +387,12 @@ class FalconForCausalLM(nn.Module):
387 super().__init__()
388 self.config = config
389 self.transformer = FalconModel(config)
392 - self.lm_head = ColumnParallelLinear(config.hidden_size,
393 - config.vocab_size,
394 - bias=False,
395 - gather_output=False,
396 - perform_initialization=False)
390 + self.lm_head = ColumnParallelLinear(
391 + config.hidden_size,
392 + config.vocab_size,
393 + bias=False,
394 + gather_output=False,
395 + )
396 self.sampler = Sampler(config.vocab_size)
397
398 def forward(
vllm/model_executor/models/gpt2.py
+27 -22
@@ -36,8 +36,9 @@ from vllm.model_executor.weight_utils import (
36 load_padded_tensor_parallel_vocab, load_tensor_parallel_weights)
37 from vllm.model_executor.parallel_utils.parallel_state import (
38 get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
39 -from vllm.model_executor.parallel_utils.tensor_parallel import (
40 - VocabParallelEmbedding, ColumnParallelLinear, RowParallelLinear)
39 +from vllm.model_executor.parallel_utils.layers import (VocabParallelEmbedding,
40 + ColumnParallelLinear,
41 + RowParallelLinear)
42 from vllm.sequence import SamplerOutput
43
44 KVCache = Tuple[torch.Tensor, torch.Tensor]
@@ -56,16 +57,18 @@ class GPT2Attention(nn.Module):
57 self.head_dim = self.hidden_size // total_num_heads
58 self.scale = self.head_dim**-0.5
59
59 - self.c_attn = ColumnParallelLinear(self.hidden_size,
60 - 3 * self.hidden_size,
61 - bias=True,
62 - gather_output=False,
63 - perform_initialization=False)
64 - self.c_proj = RowParallelLinear(self.hidden_size,
65 - self.hidden_size,
66 - bias=True,
67 - input_is_parallel=True,
68 - perform_initialization=False)
60 + self.c_attn = ColumnParallelLinear(
61 + self.hidden_size,
62 + 3 * self.hidden_size,
63 + bias=True,
64 + gather_output=False,
65 + )
66 + self.c_proj = RowParallelLinear(
67 + self.hidden_size,
68 + self.hidden_size,
69 + bias=True,
70 + input_is_parallel=True,
71 + )
72 self.attn = PagedAttention(self.num_heads,
73 self.head_dim,
74 scale=self.scale)
@@ -95,16 +98,18 @@ class GPT2MLP(nn.Module):
98 ):
99 super().__init__()
100 hidden_size = config.hidden_size
98 - self.c_fc = ColumnParallelLinear(hidden_size,
99 - intermediate_size,
100 - bias=True,
101 - gather_output=False,
102 - perform_initialization=False)
103 - self.c_proj = RowParallelLinear(intermediate_size,
104 - hidden_size,
105 - bias=True,
106 - input_is_parallel=True,
107 - perform_initialization=False)
101 + self.c_fc = ColumnParallelLinear(
102 + hidden_size,
103 + intermediate_size,
104 + bias=True,
105 + gather_output=False,
106 + )
107 + self.c_proj = RowParallelLinear(
108 + intermediate_size,
109 + hidden_size,
110 + bias=True,
111 + input_is_parallel=True,
112 + )
113 self.act = get_act_fn(config.activation_function)
114
115 def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
vllm/model_executor/models/gpt_bigcode.py
+34 -29
@@ -37,8 +37,9 @@ from vllm.model_executor.weight_utils import (
37 load_padded_tensor_parallel_vocab, load_tensor_parallel_weights)
38 from vllm.model_executor.parallel_utils.parallel_state import (
39 get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
40 -from vllm.model_executor.parallel_utils.tensor_parallel import (
41 - VocabParallelEmbedding, ColumnParallelLinear, RowParallelLinear)
40 +from vllm.model_executor.parallel_utils.layers import (VocabParallelEmbedding,
41 + ColumnParallelLinear,
42 + RowParallelLinear)
43 from vllm.sequence import SamplerOutput
44
45 KVCache = Tuple[torch.Tensor, torch.Tensor]
@@ -62,29 +63,31 @@ class GPTBigCodeAttention(nn.Module):
63 if self.multi_query:
64 self.num_kv_heads = 1
65 self.kv_dim = self.head_dim
65 - self.c_attn_q = ColumnParallelLinear(self.hidden_size,
66 - self.hidden_size,
67 - bias=True,
68 - gather_output=False,
69 - perform_initialization=False)
66 + self.c_attn_q = ColumnParallelLinear(
67 + self.hidden_size,
68 + self.hidden_size,
69 + bias=True,
70 + gather_output=False,
71 + )
72 self.c_attn_kv = nn.Linear(self.hidden_size,
73 2 * self.kv_dim,
74 bias=True)
75 else:
76 self.num_kv_heads = self.num_heads
77 self.kv_dim = self.num_kv_heads * self.head_dim
76 - self.c_attn = ColumnParallelLinear(self.hidden_size,
77 - self.hidden_size +
78 - 2 * self.kv_dim,
79 - bias=True,
80 - gather_output=False,
81 - perform_initialization=False)
82 -
83 - self.c_proj = RowParallelLinear(self.hidden_size,
84 - self.hidden_size,
85 - bias=True,
86 - input_is_parallel=True,
87 - perform_initialization=False)
78 + self.c_attn = ColumnParallelLinear(
79 + self.hidden_size,
80 + self.hidden_size + 2 * self.kv_dim,
81 + bias=True,
82 + gather_output=False,
83 + )
84 +
85 + self.c_proj = RowParallelLinear(
86 + self.hidden_size,
87 + self.hidden_size,
88 + bias=True,
89 + input_is_parallel=True,
90 + )
91 self.attn = PagedAttention(self.num_heads,
92 self.head_dim,
93 scale=self.scale,
@@ -124,16 +127,18 @@ class GPTBigMLP(nn.Module):
127 ):
128 super().__init__()
129 hidden_size = config.hidden_size
127 - self.c_fc = ColumnParallelLinear(hidden_size,
128 - intermediate_size,
129 - bias=True,
130 - gather_output=False,
131 - perform_initialization=False)
132 - self.c_proj = RowParallelLinear(intermediate_size,
133 - hidden_size,
134 - bias=True,
135 - input_is_parallel=True,
136 - perform_initialization=False)
130 + self.c_fc = ColumnParallelLinear(
131 + hidden_size,
132 + intermediate_size,
133 + bias=True,
134 + gather_output=False,
135 + )
136 + self.c_proj = RowParallelLinear(
137 + intermediate_size,
138 + hidden_size,
139 + bias=True,
140 + input_is_parallel=True,
141 + )
142 self.act = get_act_fn(config.activation_function)
143
144 def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
vllm/model_executor/models/gpt_j.py
+34 -27
@@ -34,8 +34,9 @@ from vllm.model_executor.weight_utils import (hf_model_weights_iterator,
34 load_tensor_parallel_weights)
35 from vllm.model_executor.parallel_utils.parallel_state import (
36 get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
37 -from vllm.model_executor.parallel_utils.tensor_parallel import (
38 - VocabParallelEmbedding, ColumnParallelLinear, RowParallelLinear)
37 +from vllm.model_executor.parallel_utils.layers import (VocabParallelEmbedding,
38 + ColumnParallelLinear,
39 + RowParallelLinear)
40 from vllm.sequence import SamplerOutput
41
42 KVCache = Tuple[torch.Tensor, torch.Tensor]
@@ -49,16 +50,18 @@ class GPTJAttention(nn.Module):
50 self.hidden_size = config.hidden_size
51 self.head_size = self.hidden_size // self.total_num_heads
52
52 - self.qkv_proj = ColumnParallelLinear(config.hidden_size,
53 - 3 * config.hidden_size,
54 - bias=False,
55 - gather_output=False,
56 - perform_initialization=False)
57 - self.out_proj = RowParallelLinear(config.hidden_size,
58 - config.hidden_size,
59 - bias=False,
60 - input_is_parallel=True,
61 - perform_initialization=False)
53 + self.qkv_proj = ColumnParallelLinear(
54 + config.hidden_size,
55 + 3 * config.hidden_size,
56 + bias=False,
57 + gather_output=False,
58 + )
59 + self.out_proj = RowParallelLinear(
60 + config.hidden_size,
61 + config.hidden_size,
62 + bias=False,
63 + input_is_parallel=True,
64 + )
65
66 tp_world_size = get_tensor_model_parallel_world_size()
67 assert self.total_num_heads % tp_world_size == 0
@@ -102,14 +105,16 @@ class GPTJMLP(nn.Module):
105 def __init__(self, intermediate_size: int, config: GPTJConfig):
106 super().__init__()
107 hidden_size = config.n_embd
105 - self.fc_in = ColumnParallelLinear(hidden_size,
106 - intermediate_size,
107 - gather_output=False,
108 - perform_initialization=False)
109 - self.fc_out = RowParallelLinear(intermediate_size,
110 - hidden_size,
111 - input_is_parallel=True,
112 - perform_initialization=False)
108 + self.fc_in = ColumnParallelLinear(
109 + hidden_size,
110 + intermediate_size,
111 + gather_output=False,
112 + )
113 + self.fc_out = RowParallelLinear(
114 + intermediate_size,
115 + hidden_size,
116 + input_is_parallel=True,
117 + )
118 self.act = get_act_fn(config.activation_function)
119
120 def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
@@ -159,9 +164,10 @@ class GPTJModel(nn.Module):
164 super().__init__()
165 self.config = config
166 self.embed_dim = config.n_embd
162 - self.wte = VocabParallelEmbedding(config.vocab_size,
163 - self.embed_dim,
164 - perform_initialization=False)
167 + self.wte = VocabParallelEmbedding(
168 + config.vocab_size,
169 + self.embed_dim,
170 + )
171 self.h = nn.ModuleList(
172 [GPTJBlock(config) for _ in range(config.n_layer)])
173 self.ln_f = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_epsilon)
@@ -199,10 +205,11 @@ class GPTJForCausalLM(nn.Module):
205 self.config = config
206 assert not config.tie_word_embeddings
207 self.transformer = GPTJModel(config)
202 - self.lm_head = ColumnParallelLinear(config.n_embd,
203 - config.vocab_size,
204 - gather_output=False,
205 - perform_initialization=False)
208 + self.lm_head = ColumnParallelLinear(
209 + config.n_embd,
210 + config.vocab_size,
211 + gather_output=False,
212 + )
213 self.sampler = Sampler(config.vocab_size)
214
215 def forward(
vllm/model_executor/models/gpt_neox.py
+29 -23
@@ -34,8 +34,9 @@ from vllm.model_executor.weight_utils import (hf_model_weights_iterator,
34 load_tensor_parallel_weights)
35 from vllm.model_executor.parallel_utils.parallel_state import (
36 get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
37 -from vllm.model_executor.parallel_utils.tensor_parallel import (
38 - VocabParallelEmbedding, ColumnParallelLinear, RowParallelLinear)
37 +from vllm.model_executor.parallel_utils.layers import (VocabParallelEmbedding,
38 + ColumnParallelLinear,
39 + RowParallelLinear)
40 from vllm.sequence import SamplerOutput
41
42 KVCache = Tuple[torch.Tensor, torch.Tensor]
@@ -59,11 +60,12 @@ class GPTNeoXAttention(nn.Module):
60 config.hidden_size,
61 3 * config.hidden_size,
62 gather_output=False,
62 - perform_initialization=False)
63 - self.dense = RowParallelLinear(config.hidden_size,
64 - config.hidden_size,
65 - input_is_parallel=True,
66 - perform_initialization=False)
63 + )
64 + self.dense = RowParallelLinear(
65 + config.hidden_size,
66 + config.hidden_size,
67 + input_is_parallel=True,
68 + )
69
70 scaling = self.head_size**-0.5
71 rotary_dim = int(self.head_size * config.rotary_pct)
@@ -100,14 +102,16 @@ class GPTNeoXMLP(nn.Module):
102
103 def __init__(self, config: GPTNeoXConfig):
104 super().__init__()
103 - self.dense_h_to_4h = ColumnParallelLinear(config.hidden_size,
104 - config.intermediate_size,
105 - gather_output=False,
106 - perform_initialization=False)
107 - self.dense_4h_to_h = RowParallelLinear(config.intermediate_size,
108 - config.hidden_size,
109 - input_is_parallel=True,
110 - perform_initialization=False)
105 + self.dense_h_to_4h = ColumnParallelLinear(
106 + config.hidden_size,
107 + config.intermediate_size,
108 + gather_output=False,
109 + )
110 + self.dense_4h_to_h = RowParallelLinear(
111 + config.intermediate_size,
112 + config.hidden_size,
113 + input_is_parallel=True,
114 + )
115 self.act = get_act_fn(config.hidden_act)
116
117 def forward(self, hidden_states):
@@ -169,9 +173,10 @@ class GPTNeoXModel(nn.Module):
173 super().__init__()
174 self.config = config
175
172 - self.embed_in = VocabParallelEmbedding(config.vocab_size,
173 - config.hidden_size,
174 - perform_initialization=False)
176 + self.embed_in = VocabParallelEmbedding(
177 + config.vocab_size,
178 + config.hidden_size,
179 + )
180 self.layers = nn.ModuleList(
181 [GPTNeoXLayer(config) for _ in range(config.num_hidden_layers)])
182 self.final_layer_norm = nn.LayerNorm(config.hidden_size,
@@ -209,11 +214,12 @@ class GPTNeoXForCausalLM(nn.Module):
214 super().__init__()
215 self.config = config
216 self.gpt_neox = GPTNeoXModel(config)
212 - self.embed_out = ColumnParallelLinear(config.hidden_size,
213 - config.vocab_size,
214 - bias=False,
215 - gather_output=False,
216 - perform_initialization=False)
217 + self.embed_out = ColumnParallelLinear(
218 + config.hidden_size,
219 + config.vocab_size,
220 + bias=False,
221 + gather_output=False,
222 + )
223 self.sampler = Sampler(config.vocab_size)
224
225 def forward(
vllm/model_executor/models/internlm.py
+24 -20
@@ -12,8 +12,9 @@ from vllm.model_executor.layers.layernorm import RMSNorm
12 from vllm.model_executor.layers.sampler import Sampler
13 from vllm.model_executor.parallel_utils.parallel_state import (
14 get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
15 -from vllm.model_executor.parallel_utils.tensor_parallel import (
16 - ColumnParallelLinear, RowParallelLinear, VocabParallelEmbedding)
15 +from vllm.model_executor.parallel_utils.layers import (ColumnParallelLinear,
16 + RowParallelLinear,
17 + VocabParallelEmbedding)
18 from vllm.model_executor.weight_utils import (
19 hf_model_weights_iterator, load_padded_tensor_parallel_vocab,
20 load_tensor_parallel_weights)
@@ -31,16 +32,18 @@ class InternLMMLP(nn.Module):
32 hidden_act: str,
33 ):
34 super().__init__()
34 - self.gate_up_proj = ColumnParallelLinear(hidden_size,
35 - 2 * intermediate_size,
36 - bias=False,
37 - gather_output=False,
38 - perform_initialization=False)
39 - self.down_proj = RowParallelLinear(intermediate_size,
40 - hidden_size,
41 - bias=False,
42 - input_is_parallel=True,
43 - perform_initialization=False)
35 + self.gate_up_proj = ColumnParallelLinear(
36 + hidden_size,
37 + 2 * intermediate_size,
38 + bias=False,
39 + gather_output=False,
40 + )
41 + self.down_proj = RowParallelLinear(
42 + intermediate_size,
43 + hidden_size,
44 + bias=False,
45 + input_is_parallel=True,
46 + )
47 if hidden_act != "silu":
48 raise ValueError(f"Unsupported activation: {hidden_act}. "
49 "Only silu is supported for now.")
@@ -80,14 +83,12 @@ class InternLMAttention(nn.Module):
83 3 * self.total_num_heads * self.head_dim,
84 bias=True,
85 gather_output=False,
83 - perform_initialization=False,
86 )
87 self.o_proj = RowParallelLinear(
88 self.total_num_heads * self.head_dim,
89 hidden_size,
90 bias=True,
91 input_is_parallel=True,
90 - perform_initialization=False,
92 )
93 self.attn = PagedAttentionWithRoPE(
94 self.num_heads,
@@ -176,7 +177,9 @@ class InternLMModel(nn.Module):
177
178 vocab_size = ((config.vocab_size + 63) // 64) * 64
179 self.embed_tokens = VocabParallelEmbedding(
179 - vocab_size, config.hidden_size, perform_initialization=False)
180 + vocab_size,
181 + config.hidden_size,
182 + )
183 self.layers = nn.ModuleList([
184 InternLMDecoderLayer(config)
185 for _ in range(config.num_hidden_layers)
@@ -216,11 +219,12 @@ class InternLMForCausalLM(nn.Module):
219 self.config = config
220 self.model = InternLMModel(config)
221 vocab_size = ((config.vocab_size + 63) // 64) * 64
219 - self.lm_head = ColumnParallelLinear(config.hidden_size,
220 - vocab_size,
221 - bias=False,
222 - gather_output=False,
223 - perform_initialization=False)
222 + self.lm_head = ColumnParallelLinear(
223 + config.hidden_size,
224 + vocab_size,
225 + bias=False,
226 + gather_output=False,
227 + )
228 self.sampler = Sampler(config.vocab_size)
229
230 def forward(
vllm/model_executor/models/llama.py
+4 -8
@@ -39,8 +39,7 @@ from vllm.model_executor.layers.sampler import Sampler
39 from vllm.model_executor.layers.quantized_linear import ParallelLinear
40 from vllm.model_executor.parallel_utils.parallel_state import (
41 get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
42 -from vllm.model_executor.parallel_utils.tensor_parallel import (
43 - VocabParallelEmbedding)
42 +from vllm.model_executor.parallel_utils.layers import VocabParallelEmbedding
43 from vllm.model_executor.quantization_utils import QuantizationConfig
44 from vllm.model_executor.weight_utils import (
45 convert_pyslice_to_tensor, hf_model_weights_iterator,
@@ -64,13 +63,11 @@ class LlamaMLP(nn.Module):
63 2 * intermediate_size,
64 bias=False,
65 gather_output=False,
67 - perform_initialization=False,
66 quant_config=quant_config)
67 self.down_proj = ParallelLinear.row(intermediate_size,
68 hidden_size,
69 bias=False,
70 input_is_parallel=True,
73 - perform_initialization=False,
71 quant_config=quant_config)
72 if hidden_act != "silu":
73 raise ValueError(f"Unsupported activation: {hidden_act}. "
@@ -127,7 +124,6 @@ class LlamaAttention(nn.Module):
124 self.head_dim,
125 bias=False,
126 gather_output=False,
130 - perform_initialization=False,
127 quant_config=quant_config,
128 )
129 self.o_proj = ParallelLinear.row(
@@ -135,7 +131,6 @@ class LlamaAttention(nn.Module):
131 hidden_size,
132 bias=False,
133 input_is_parallel=True,
138 - perform_initialization=False,
134 quant_config=quant_config,
135 )
136 self.attn = PagedAttentionWithRoPE(
@@ -241,7 +236,9 @@ class LlamaModel(nn.Module):
236
237 vocab_size = ((config.vocab_size + 63) // 64) * 64
238 self.embed_tokens = VocabParallelEmbedding(
244 - vocab_size, config.hidden_size, perform_initialization=False)
239 + vocab_size,
240 + config.hidden_size,
241 + )
242 self.layers = nn.ModuleList([
243 LlamaDecoderLayer(config, quant_config)
244 for _ in range(config.num_hidden_layers)
@@ -291,7 +288,6 @@ class LlamaForCausalLM(nn.Module):
288 vocab_size,
289 bias=False,
290 gather_output=False,
294 - perform_initialization=False,
291 quant_config=None)
292 self.sampler = Sampler(config.vocab_size)
293
vllm/model_executor/models/mistral.py
+4 -8
@@ -38,8 +38,7 @@ from vllm.model_executor.layers.sampler import Sampler
38 from vllm.model_executor.layers.quantized_linear import ParallelLinear
39 from vllm.model_executor.parallel_utils.parallel_state import (
40 get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
41 -from vllm.model_executor.parallel_utils.tensor_parallel import (
42 - VocabParallelEmbedding)
41 +from vllm.model_executor.parallel_utils.layers import VocabParallelEmbedding
42 from vllm.model_executor.quantization_utils import QuantizationConfig
43 from vllm.model_executor.weight_utils import (
44 convert_pyslice_to_tensor, hf_model_weights_iterator,
@@ -64,13 +63,11 @@ class MistralMLP(nn.Module):
63 2 * intermediate_size,
64 bias=False,
65 gather_output=False,
67 - perform_initialization=False,
66 quant_config=quant_config)
67 self.down_proj = ParallelLinear.row(intermediate_size,
68 hidden_size,
69 bias=False,
70 input_is_parallel=True,
73 - perform_initialization=False,
71 quant_config=quant_config)
72 if hidden_act != "silu":
73 raise ValueError(f"Unsupported activation: {hidden_act}. "
@@ -116,7 +113,6 @@ class MistralAttention(nn.Module):
113 self.head_dim,
114 bias=False,
115 gather_output=False,
119 - perform_initialization=False,
116 quant_config=quant_config,
117 )
118 self.o_proj = ParallelLinear.row(
@@ -124,7 +120,6 @@ class MistralAttention(nn.Module):
120 hidden_size,
121 bias=False,
122 input_is_parallel=True,
127 - perform_initialization=False,
123 quant_config=quant_config,
124 )
125 self.attn = PagedAttentionWithRoPE(self.num_heads,
@@ -225,7 +220,9 @@ class MistralModel(nn.Module):
220
221 vocab_size = ((config.vocab_size + 63) // 64) * 64
222 self.embed_tokens = VocabParallelEmbedding(
228 - vocab_size, config.hidden_size, perform_initialization=False)
223 + vocab_size,
224 + config.hidden_size,
225 + )
226 self.layers = nn.ModuleList([
227 MistralDecoderLayer(config, quant_config)
228 for _ in range(config.num_hidden_layers)
@@ -275,7 +272,6 @@ class MistralForCausalLM(nn.Module):
272 vocab_size,
273 bias=False,
274 gather_output=False,
278 - perform_initialization=False,
275 quant_config=None)
276 self.sampler = Sampler(config.vocab_size)
277
vllm/model_executor/models/mpt.py
+19 -17
@@ -15,8 +15,9 @@ from vllm.model_executor.weight_utils import (convert_pyslice_to_tensor,
15 load_tensor_parallel_weights)
16 from vllm.model_executor.parallel_utils.parallel_state import (
17 get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
18 -from vllm.model_executor.parallel_utils.tensor_parallel import (
19 - VocabParallelEmbedding, ColumnParallelLinear, RowParallelLinear)
18 +from vllm.model_executor.parallel_utils.layers import (VocabParallelEmbedding,
19 + ColumnParallelLinear,
20 + RowParallelLinear)
21 from vllm.sequence import SamplerOutput
22 from vllm.transformers_utils.configs.mpt import MPTConfig
23
@@ -53,7 +54,6 @@ class MPTAttention(nn.Module):
54 3 * self.d_model,
55 bias=not config.no_bias,
56 gather_output=False,
56 - perform_initialization=False,
57 )
58 if self.qk_ln:
59 self.q_ln = nn.LayerNorm(self.d_model)
@@ -63,7 +63,6 @@ class MPTAttention(nn.Module):
63 self.d_model,
64 bias=not config.no_bias,
65 input_is_parallel=True,
66 - perform_initialization=False,
66 )
67
68 tp_world_size = get_tensor_model_parallel_world_size()
@@ -113,17 +112,19 @@ class MPTMLP(nn.Module):
112 hidden_size = config.d_model
113 expansion_ratio = config.expansion_ratio
114 intermediate_size = expansion_ratio * hidden_size
116 - self.up_proj = ColumnParallelLinear(hidden_size,
117 - intermediate_size,
118 - bias=not config.no_bias,
119 - gather_output=False,
120 - perform_initialization=False)
115 + self.up_proj = ColumnParallelLinear(
116 + hidden_size,
117 + intermediate_size,
118 + bias=not config.no_bias,
119 + gather_output=False,
120 + )
121 self.act = get_act_fn("gelu")
122 - self.down_proj = RowParallelLinear(intermediate_size,
123 - hidden_size,
124 - bias=not config.no_bias,
125 - input_is_parallel=True,
126 - perform_initialization=False)
122 + self.down_proj = RowParallelLinear(
123 + intermediate_size,
124 + hidden_size,
125 + bias=not config.no_bias,
126 + input_is_parallel=True,
127 + )
128
129 def forward(self, x: torch.Tensor) -> torch.Tensor:
130 x, _ = self.up_proj(x)
@@ -172,9 +173,10 @@ class MPTModel(nn.Module):
173 assert config.embedding_fraction == 1.0
174 assert config.norm_type == "low_precision_layernorm"
175
175 - self.wte = VocabParallelEmbedding(config.vocab_size,
176 - config.d_model,
177 - perform_initialization=False)
176 + self.wte = VocabParallelEmbedding(
177 + config.vocab_size,
178 + config.d_model,
179 + )
180 self.blocks = nn.ModuleList(
181 [MPTBlock(config) for _ in range(config.n_layers)])
182 self.norm_f = nn.LayerNorm(config.d_model)
vllm/model_executor/models/opt.py
+28 -23
@@ -35,8 +35,9 @@ from vllm.model_executor.weight_utils import (hf_model_weights_iterator,
35 load_tensor_parallel_weights)
36 from vllm.model_executor.parallel_utils.parallel_state import (
37 get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
38 -from vllm.model_executor.parallel_utils.tensor_parallel import (
39 - VocabParallelEmbedding, ColumnParallelLinear, RowParallelLinear)
38 +from vllm.model_executor.parallel_utils.layers import (VocabParallelEmbedding,
39 + ColumnParallelLinear,
40 + RowParallelLinear)
41 from vllm.sequence import SamplerOutput
42
43 KVCache = Tuple[torch.Tensor, torch.Tensor]
@@ -73,16 +74,18 @@ class OPTAttention(nn.Module):
74 self.head_dim = embed_dim // total_num_heads
75 self.scaling = self.head_dim**-0.5
76
76 - self.qkv_proj = ColumnParallelLinear(embed_dim,
77 - 3 * embed_dim,
78 - bias=bias,
79 - gather_output=False,
80 - perform_initialization=False)
81 - self.out_proj = RowParallelLinear(embed_dim,
82 - embed_dim,
83 - bias=bias,
84 - input_is_parallel=True,
85 - perform_initialization=False)
77 + self.qkv_proj = ColumnParallelLinear(
78 + embed_dim,
79 + 3 * embed_dim,
80 + bias=bias,
81 + gather_output=False,
82 + )
83 + self.out_proj = RowParallelLinear(
84 + embed_dim,
85 + embed_dim,
86 + bias=bias,
87 + input_is_parallel=True,
88 + )
89 self.attn = PagedAttention(self.num_heads,
90 self.head_dim,
91 scale=self.scaling)
@@ -120,16 +123,18 @@ class OPTDecoderLayer(nn.Module):
123 self.self_attn_layer_norm = nn.LayerNorm(
124 self.embed_dim,
125 elementwise_affine=config.layer_norm_elementwise_affine)
123 - self.fc1 = ColumnParallelLinear(self.embed_dim,
124 - config.ffn_dim,
125 - bias=config.enable_bias,
126 - gather_output=False,
127 - perform_initialization=False)
128 - self.fc2 = RowParallelLinear(config.ffn_dim,
129 - self.embed_dim,
130 - bias=config.enable_bias,
131 - input_is_parallel=True,
132 - perform_initialization=False)
126 + self.fc1 = ColumnParallelLinear(
127 + self.embed_dim,
128 + config.ffn_dim,
129 + bias=config.enable_bias,
130 + gather_output=False,
131 + )
132 + self.fc2 = RowParallelLinear(
133 + config.ffn_dim,
134 + self.embed_dim,
135 + bias=config.enable_bias,
136 + input_is_parallel=True,
137 + )
138 self.final_layer_norm = nn.LayerNorm(
139 self.embed_dim,
140 elementwise_affine=config.layer_norm_elementwise_affine)
@@ -182,7 +187,7 @@ class OPTDecoder(nn.Module):
187 self.embed_tokens = VocabParallelEmbedding(
188 config.vocab_size,
189 config.word_embed_proj_dim,
185 - perform_initialization=False)
190 + )
191 # Positional embeddings are replicated (not sharded).
192 self.embed_positions = OPTLearnedPositionalEmbedding(
193 config.max_position_embeddings, config.hidden_size)
vllm/model_executor/models/qwen.py
+5 -9
@@ -28,7 +28,7 @@ from vllm.model_executor.parallel_utils.parallel_state import (
28 get_tensor_model_parallel_rank,
29 get_tensor_model_parallel_world_size,
30 )
31 -from vllm.model_executor.parallel_utils.tensor_parallel import (
31 +from vllm.model_executor.parallel_utils.layers import (
32 VocabParallelEmbedding,
33 ColumnParallelLinear,
34 RowParallelLinear,
@@ -53,14 +53,12 @@ class QWenMLP(nn.Module):
53 2 * intermediate_size,
54 bias=False,
55 gather_output=False,
56 - perform_initialization=False,
56 )
57 self.c_proj = RowParallelLinear(
58 intermediate_size,
59 hidden_size,
60 bias=False,
61 input_is_parallel=True,
63 - perform_initialization=False,
62 )
63 if hidden_act != "silu":
64 raise ValueError(f"Unsupported activation: {hidden_act}. "
@@ -98,14 +96,12 @@ class QWenAttention(nn.Module):
96 3 * hidden_size,
97 bias=True,
98 gather_output=False,
101 - perform_initialization=False,
99 )
100 self.c_proj = RowParallelLinear(
101 self.total_num_heads * self.head_dim,
102 hidden_size,
103 bias=False,
104 input_is_parallel=True,
108 - perform_initialization=False,
105 )
106 self.scaling = self.head_dim**-0.5
107 self.attn = PagedAttentionWithRoPE(
@@ -190,9 +186,10 @@ class QWenModel(nn.Module):
186 self.vocab_size = config.vocab_size
187
188 vocab_size = ((config.vocab_size + 63) // 64) * 64
193 - self.wte = VocabParallelEmbedding(vocab_size,
194 - config.hidden_size,
195 - perform_initialization=False)
189 + self.wte = VocabParallelEmbedding(
190 + vocab_size,
191 + config.hidden_size,
192 + )
193 self.h = nn.ModuleList(
194 [QWenBlock(config) for _ in range(config.num_hidden_layers)])
195 self.ln_f = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon)
@@ -235,7 +232,6 @@ class QWenLMHeadModel(nn.Module):
232 vocab_size,
233 bias=False,
234 gather_output=False,
238 - perform_initialization=False,
235 )
236 self.sampler = Sampler(config.vocab_size)
237
vllm/model_executor/parallel_utils/__init__.py
-7
@@ -1,7 +0,0 @@
1 -import vllm.model_executor.parallel_utils.parallel_state
2 -import vllm.model_executor.parallel_utils.tensor_parallel
3 -
4 -__all__ = [
5 - "parallel_state",
6 - "tensor_parallel",
7 -]
vllm/model_executor/parallel_utils/communication_op.py new
+47
@@ -0,0 +1,47 @@
1 +import torch
2 +
3 +from vllm.model_executor.parallel_utils.parallel_state import (
4 + get_tensor_model_parallel_world_size,
5 + get_tensor_model_parallel_group,
6 +)
7 +
8 +
9 +def tensor_model_parallel_all_reduce(input_):
10 + """All-reduce the input tensor across model parallel group.
11 +
12 + Note: This operation is applied in-place on the input tensor.
13 + """
14 + # Bypass the function if we are using only 1 GPU.
15 + if get_tensor_model_parallel_world_size() == 1:
16 + return input_
17 + # All-reduce.
18 + torch.distributed.all_reduce(input_,
19 + group=get_tensor_model_parallel_group())
20 + return input_
21 +
22 +
23 +def tensor_model_parallel_all_gather(input_, dim=-1):
24 + """All-gather the input tensor across model parallel group."""
25 + world_size = get_tensor_model_parallel_world_size()
26 + # Bypass the function if we are using only 1 GPU.
27 + if world_size == 1:
28 + return input_
29 + assert -input_.dim() <= dim < input_.dim(), (
30 + f"Invalid dim ({dim}) for input tensor with shape {input_.size()}")
31 + if dim < 0:
32 + # Convert negative dim to positive.
33 + dim += input_.dim()
34 + input_size = input_.size()
35 + # Allocate output tensor.
36 + output_tensor = torch.empty((world_size, ) + input_size,
37 + dtype=input_.dtype,
38 + device=input_.device)
39 + # All-gather.
40 + torch.distributed.all_gather_into_tensor(
41 + output_tensor, input_, group=get_tensor_model_parallel_group())
42 + # Reshape
43 + output_tensor = output_tensor.movedim(0, dim)
44 + output_tensor = output_tensor.reshape(input_size[:dim] +
45 + (world_size * input_size[dim], ) +
46 + input_size[dim + 1:])
47 + return output_tensor
vllm/model_executor/parallel_utils/layers.py new
+303
@@ -0,0 +1,303 @@
1 +# Copyright 2023 The vLLM team.
2 +# Adapted from
3 +# https://github.com/NVIDIA/Megatron-LM/blob/main/megatron/core/tensor_parallel/layers.py
4 +# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
5 +
6 +# Parts of the code here are adapted from PyTorch
7 +# repo: https://github.com/pytorch/pytorch
8 +from typing import Optional
9 +
10 +import torch
11 +import torch.nn.functional as F
12 +from torch.nn.parameter import Parameter
13 +
14 +from vllm.model_executor.parallel_utils.parallel_state import (
15 + get_tensor_model_parallel_rank,
16 + get_tensor_model_parallel_world_size,
17 +)
18 +from vllm.model_executor.quantization_utils import QuantizationConfig
19 +from vllm.model_executor.parallel_utils.communication_op import (
20 + tensor_model_parallel_all_reduce, tensor_model_parallel_all_gather)
21 +
22 +from vllm.model_executor.parallel_utils.utils import (
23 + divide,
24 + VocabUtility,
25 + split_tensor_along_last_dim,
26 +)
27 +
28 +
29 +class VocabParallelEmbedding(torch.nn.Module):
30 + """Embedding parallelized in the vocabulary dimension.
31 +
32 + This is mainly adapted from torch.nn.Embedding and all the default
33 + values are kept.
34 + Arguments:
35 + num_embeddings: vocabulary size.
36 + embedding_dim: size of hidden state.
37 + params_dtype: type of the parameters.
38 + """
39 +
40 + def __init__(self,
41 + num_embeddings: int,
42 + embedding_dim: int,
43 + params_dtype: Optional[torch.dtype] = None):
44 + super().__init__()
45 +
46 + # Keep the input dimensions.
47 + self.num_embeddings = num_embeddings
48 + self.embedding_dim = embedding_dim
49 + if params_dtype is None:
50 + params_dtype = torch.get_default_dtype()
51 +
52 + self.tp_size = get_tensor_model_parallel_world_size()
53 + # TODO: Handle vocab padding here.
54 + # Divide the weight matrix along the vocaburaly dimension.
55 + self.vocab_start_index, self.vocab_end_index = (
56 + VocabUtility.vocab_range_from_global_vocab_size(
57 + self.num_embeddings, get_tensor_model_parallel_rank(),
58 + self.tp_size))
59 + self.num_embeddings_per_partition = (self.vocab_end_index -
60 + self.vocab_start_index)
61 +
62 + self.weight = Parameter(
63 + torch.empty(self.num_embeddings_per_partition,
64 + self.embedding_dim,
65 + device=torch.cuda.current_device(),
66 + dtype=params_dtype))
67 +
68 + def forward(self, input_):
69 + if self.tp_size > 1:
70 + # Build the mask.
71 + input_mask = ((input_ < self.vocab_start_index) |
72 + (input_ >= self.vocab_end_index))
73 + # Mask the input.
74 + masked_input = input_.clone() - self.vocab_start_index
75 + masked_input[input_mask] = 0
76 + else:
77 + masked_input = input_
78 + # Get the embeddings.
79 + output_parallel = F.embedding(masked_input, self.weight)
80 + # Mask the output embedding.
81 + if self.tp_size > 1:
82 + output_parallel[input_mask, :] = 0.0
83 + # Reduce across all the model parallel GPUs.
84 + output = tensor_model_parallel_all_reduce(output_parallel)
85 + return output
86 +
87 +
88 +class ColumnParallelLinear(torch.nn.Module):
89 + """Linear layer with column parallelism.
90 +
91 + The linear layer is defined as Y = XA + b. A is parallelized along
92 + its second dimension as A = [A_1, ..., A_p].
93 +
94 + Arguments:
95 + input_size: first dimension of matrix A.
96 + output_size: second dimension of matrix A.
97 +
98 + Keyword Arguments
99 + bias: If true, add bias
100 + gather_output: If true, call all-gather on output and make Y available
101 + to all GPUs, otherwise, every GPU will have its output
102 + which is Y_i = XA_i
103 + skip_bias_add: This was added to enable performance optimizations where
104 + bias can be fused with other element-wise operations. we
105 + skip adding bias but instead return it.
106 + params_dtype: Data type for the parameters.
107 + quant_config: Quantization configuration.
108 + """
109 +
110 + def __init__(
111 + self,
112 + input_size: int,
113 + output_size: int,
114 + bias: bool = True,
115 + gather_output: bool = True,
116 + skip_bias_add: bool = False,
117 + params_dtype: Optional[torch.dtype] = None,
118 + quant_config: Optional[QuantizationConfig] = None,
119 + ):
120 + super().__init__()
121 +
122 + # Keep input parameters
123 + self.input_size = input_size
124 + self.output_size = output_size
125 + self.gather_output = gather_output
126 + # Divide the weight matrix along the last dimension.
127 + self.tp_size = get_tensor_model_parallel_world_size()
128 + self.output_size_per_partition = divide(output_size, self.tp_size)
129 + self.skip_bias_add = skip_bias_add
130 + self.quant_config = quant_config
131 +
132 + if params_dtype is None:
133 + params_dtype = torch.get_default_dtype()
134 +
135 + # Parameters.
136 + # Note: torch.nn.functional.linear performs XA^T + b and as a result
137 + # we allocate the transpose.
138 + self.create_weights(params_dtype)
139 +
140 + if bias:
141 + self.bias = Parameter(
142 + torch.empty(self.output_size_per_partition,
143 + device=torch.cuda.current_device(),
144 + dtype=params_dtype))
145 + else:
146 + self.register_parameter('bias', None)
147 +
148 + def create_weights(self, dtype: torch.dtype) -> None:
149 + self.weight = Parameter(
150 + torch.empty(self.output_size_per_partition,
151 + self.input_size,
152 + device=torch.cuda.current_device(),
153 + dtype=dtype))
154 +
155 + def apply_weights(
156 + self,
157 + x: torch.Tensor,
158 + bias: Optional[torch.Tensor],
159 + ) -> torch.Tensor:
160 + return F.linear(x, self.weight, bias)
161 +
162 + def forward(self, input_):
163 + """Forward of ColumnParallelLinear
164 +
165 + Args:
166 + input_: Tensor whose last dimension is `input_size`.
167 +
168 + Returns:
169 + - output
170 + - bias
171 + """
172 + bias = self.bias if not self.skip_bias_add else None
173 +
174 + input_parallel = input_
175 + # Matrix multiply.
176 + output_parallel = self.apply_weights(input_parallel, bias)
177 + if self.gather_output:
178 + # All-gather across the partitions.
179 + output = tensor_model_parallel_all_gather(output_parallel)
180 + else:
181 + output = output_parallel
182 + output_bias = self.bias if self.skip_bias_add else None
183 + return output, output_bias
184 +
185 +
186 +class RowParallelLinear(torch.nn.Module):
187 + """Linear layer with row parallelism.
188 +
189 + The linear layer is defined as Y = XA + b. A is parallelized along
190 + its first dimension and X along its second dimension as:
191 + - -
192 + | A_1 |
193 + | . |
194 + A = | . | X = [X_1, ..., X_p]
195 + | . |
196 + | A_p |
197 + - -
198 + Arguments:
199 + input_size: first dimension of matrix A.
200 + output_size: second dimension of matrix A.
201 +
202 + Keyword Arguments:
203 + bias: If true, add bias. Note that bias is not parallelized.
204 + input_is_parallel: If true, we assume that the input is already
205 + split across the GPUs and we do not split
206 + again.
207 + skip_bias_add: This was added to enable performance optimization where
208 + bias can be fused with other element-wise operations.
209 + We skip adding bias but instead return it.
210 + params_dtype: Data type for the parameters.
211 + quant_config: Quantization configuration.
212 + """
213 +
214 + def __init__(
215 + self,
216 + input_size: int,
217 + output_size: int,
218 + bias: bool = True,
219 + input_is_parallel: bool = False,
220 + skip_bias_add: bool = False,
221 + params_dtype: Optional[torch.dtype] = None,
222 + reduce_results: bool = True,
223 + quant_config: Optional[QuantizationConfig] = None,
224 + ):
225 + super().__init__()
226 + # Keep input parameters
227 + self.input_size = input_size
228 + self.output_size = output_size
229 + self.input_is_parallel = input_is_parallel
230 + self.reduce_results = reduce_results
231 + if params_dtype is None:
232 + params_dtype = torch.get_default_dtype()
233 +
234 + # Divide the weight matrix along the last dimension.
235 + self.tp_size = get_tensor_model_parallel_world_size()
236 + self.input_size_per_partition = divide(input_size, self.tp_size)
237 + self.skip_bias_add = skip_bias_add
238 + self.quant_config = quant_config
239 +
240 + self.create_weights(params_dtype)
241 +
242 + if not reduce_results and (bias and not skip_bias_add):
243 + raise ValueError('When not reduce the results, adding bias to the '
244 + 'results can lead to incorrect results')
245 +
246 + if bias:
247 + self.bias = Parameter(
248 + torch.empty(self.output_size,
249 + device=torch.cuda.current_device(),
250 + dtype=params_dtype))
251 +
252 + # Always initialize bias to zero.
253 + with torch.no_grad():
254 + self.bias.zero_()
255 + else:
256 + self.register_parameter('bias', None)
257 +
258 + def create_weights(self, dtype: torch.dtype) -> None:
259 + self.weight = Parameter(
260 + torch.empty(self.output_size,
261 + self.input_size_per_partition,
262 + device=torch.cuda.current_device(),
263 + dtype=dtype))
264 +
265 + def apply_weights(self, x: torch.Tensor) -> torch.Tensor:
266 + return F.linear(x, self.weight)
267 +
268 + def forward(self, input_):
269 + """Forward of RowParallelLinear
270 +
271 + Args:
272 + input_: tensor whose last dimension is `input_size`. If
273 + `input_is_parallel` is set, then the last dimension
274 + is `input_size // tp_size`.
275 +
276 + Returns:
277 + - output
278 + - bias
279 + """
280 + # Set up backprop all-reduce.
281 + if self.input_is_parallel:
282 + input_parallel = input_
283 + else:
284 + # TODO: simplify code below
285 + tp_rank = get_tensor_model_parallel_rank()
286 + splitted_input = split_tensor_along_last_dim(
287 + input_, num_partitions=self.tp_size)
288 + input_parallel = splitted_input[tp_rank].contiguous()
289 +
290 + # Matrix multiply.
291 + output_parallel = self.apply_weights(input_parallel)
292 + if self.reduce_results and self.tp_size > 1:
293 + output_ = tensor_model_parallel_all_reduce(output_parallel)
294 + else:
295 + output_ = output_parallel
296 +
297 + if not self.skip_bias_add:
298 + output = output_ + self.bias if self.bias is not None else output_
299 + output_bias = None
300 + else:
301 + output = output_
302 + output_bias = self.bias
303 + return output, output_bias
vllm/model_executor/parallel_utils/parallel_state.py
+56 -376
@@ -1,78 +1,42 @@
1 # Copyright 2023 The vLLM team.
2 -# Adapted from https://github.com/NVIDIA/Megatron-LM/blob/main/megatron/core/parallel_state.py
2 +# Adapted from
3 +# https://github.com/NVIDIA/Megatron-LM/blob/main/megatron/core/parallel_state.py
4 # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
4 -
5 """Model and data parallel groups."""
6
7 import torch
8 -from typing import Optional
8
10 -# Intra-layer model parallel group that the current rank belongs to.
9 +# Tensor model parallel group that the current rank belongs to.
10 _TENSOR_MODEL_PARALLEL_GROUP = None
12 -# Inter-layer model parallel group that the current rank belongs to.
11 +# Pipeline model parallel group that the current rank belongs to.
12 _PIPELINE_MODEL_PARALLEL_GROUP = None
14 -# Model parallel group (both intra- and pipeline) that the current rank belongs to.
15 -_MODEL_PARALLEL_GROUP = None
16 -# Embedding group.
17 -_EMBEDDING_GROUP = None
18 -# Position embedding group.
19 -_POSITION_EMBEDDING_GROUP = None
20 -# Data parallel group that the current rank belongs to.
21 -_DATA_PARALLEL_GROUP = None
22 -
23 -_VIRTUAL_PIPELINE_MODEL_PARALLEL_RANK = None
24 -_VIRTUAL_PIPELINE_MODEL_PARALLEL_WORLD_SIZE = None
25 -_PIPELINE_MODEL_PARALLEL_SPLIT_RANK = None
26 -
27 -# These values enable us to change the mpu sizes on the fly.
28 -_MPU_TENSOR_MODEL_PARALLEL_WORLD_SIZE = None
29 -_MPU_PIPELINE_MODEL_PARALLEL_WORLD_SIZE = None
30 -_MPU_TENSOR_MODEL_PARALLEL_RANK = None
31 -_MPU_PIPELINE_MODEL_PARALLEL_RANK = None
32 -
33 -# A list of ranks that have a copy of the embedding.
34 -_EMBEDDING_GLOBAL_RANKS = None
35 -
36 -# A list of ranks that have a copy of the position embedding.
37 -_POSITION_EMBEDDING_GLOBAL_RANKS = None
38 -
39 -# A list of global ranks for each pipeline group to ease calculation of the source
40 -# rank when broadcasting from the first or last pipeline stage.
41 -_PIPELINE_GLOBAL_RANKS = None
13
43 -# A list of global ranks for each data parallel group to ease calculation of the source
44 -# rank when broadcasting weights from src to all other data parallel ranks
45 -_DATA_PARALLEL_GLOBAL_RANKS = None
14 +# A list of global ranks for each pipeline group to ease calculation of the
15 +# source rank when broadcasting from the first or last pipeline stage.
16 +_PIPELINE_GLOBAL_RANKS = None
17
18
19 def initialize_model_parallel(
20 tensor_model_parallel_size: int = 1,
21 pipeline_model_parallel_size: int = 1,
51 - virtual_pipeline_model_parallel_size: Optional[int] = None,
52 - pipeline_model_parallel_split_rank: Optional[int] = None,
22 ) -> None:
23 """
55 - Initialize model data parallel groups.
24 + Initialize model parallel groups.
25
26 Arguments:
58 - tensor_model_parallel_size: number of GPUs used for tensor model parallelism.
59 - pipeline_model_parallel_size: number of GPUs used for pipeline model parallelism.
60 - virtual_pipeline_model_parallel_size: number of virtual stages (interleaved
61 - pipeline).
62 - pipeline_model_parallel_split_rank: for models with both encoder and decoder,
63 - rank in pipeline with split point.
64 -
65 - Let's say we have a total of 16 GPUs denoted by g0 ... g15 and we
27 + tensor_model_parallel_size: number of GPUs used for tensor model
28 + parallelism.
29 + pipeline_model_parallel_size: number of GPUs used for pipeline model
30 + parallelism.
31 +
32 + Let's say we have a total of 8 GPUs denoted by g0 ... g7 and we
33 use 2 GPUs to parallelize the model tensor, and 4 GPUs to parallelize
34 the model pipeline. The present function will
68 - create 8 tensor model-parallel groups, 4 pipeline model-parallel groups
69 - and 8 data-parallel groups as:
70 - 8 data_parallel groups:
71 - [g0, g2], [g1, g3], [g4, g6], [g5, g7], [g8, g10], [g9, g11], [g12, g14], [g13, g15]
72 - 8 tensor model-parallel groups:
73 - [g0, g1], [g2, g3], [g4, g5], [g6, g7], [g8, g9], [g10, g11], [g12, g13], [g14, g15]
74 - 4 pipeline model-parallel groups:
75 - [g0, g4, g8, g12], [g1, g5, g9, g13], [g2, g6, g10, g14], [g3, g7, g11, g15]
35 + create 4 tensor model-parallel groups and 2 pipeline model-parallel groups:
36 + 4 tensor model-parallel groups:
37 + [g0, g1], [g2, g3], [g4, g5], [g6, g7]
38 + 2 pipeline model-parallel groups:
39 + [g0, g2, g4, g6], [g1, g3, g5, g7]
40 Note that for efficiency, the caller should make sure adjacent ranks
41 are on the same DGX box. For example if we are using 2 DGX-1 boxes
42 with a total of 16 GPUs, rank 0 to 7 belong to the first box and
@@ -82,64 +46,23 @@ def initialize_model_parallel(
46 assert torch.distributed.is_initialized()
47 world_size: int = torch.distributed.get_world_size()
48
85 - if world_size % (tensor_model_parallel_size * pipeline_model_parallel_size) != 0:
49 + if (world_size !=
50 + tensor_model_parallel_size * pipeline_model_parallel_size):
51 raise RuntimeError(
87 - f"world_size ({world_size}) is not divisible by tensor_model_parallel_size "
88 - f"({tensor_model_parallel_size}) x pipeline_model_parallel_size ({pipeline_model_parallel_size})"
89 - )
90 -
91 - data_parallel_size: int = world_size // (tensor_model_parallel_size *
92 - pipeline_model_parallel_size)
93 -
94 - num_tensor_model_parallel_groups: int = world_size // tensor_model_parallel_size
95 - num_pipeline_model_parallel_groups: int = world_size // pipeline_model_parallel_size
96 - num_data_parallel_groups: int = world_size // data_parallel_size
97 -
98 - if virtual_pipeline_model_parallel_size is not None:
99 - if not pipeline_model_parallel_size > 2:
100 - raise RuntimeError("pipeline-model-parallel size should be greater than 2 with "
101 - "interleaved schedule")
102 - global _VIRTUAL_PIPELINE_MODEL_PARALLEL_RANK
103 - global _VIRTUAL_PIPELINE_MODEL_PARALLEL_WORLD_SIZE
104 - _VIRTUAL_PIPELINE_MODEL_PARALLEL_RANK = 0
105 - _VIRTUAL_PIPELINE_MODEL_PARALLEL_WORLD_SIZE = virtual_pipeline_model_parallel_size
106 -
107 - if pipeline_model_parallel_split_rank is not None:
108 - global _PIPELINE_MODEL_PARALLEL_SPLIT_RANK
109 - _PIPELINE_MODEL_PARALLEL_SPLIT_RANK = pipeline_model_parallel_split_rank
110 -
52 + f"world_size ({world_size}) is not equal to "
53 + f"tensor_model_parallel_size ({tensor_model_parallel_size}) x "
54 + f"pipeline_model_parallel_size ({pipeline_model_parallel_size})")
55 +
56 + num_tensor_model_parallel_groups: int = (world_size //
57 + tensor_model_parallel_size)
58 + num_pipeline_model_parallel_groups: int = (world_size //
59 + pipeline_model_parallel_size)
60 rank = torch.distributed.get_rank()
61
113 - # Build the data-parallel groups.
114 - global _DATA_PARALLEL_GROUP
115 - global _DATA_PARALLEL_GLOBAL_RANKS
116 - assert _DATA_PARALLEL_GROUP is None, 'data parallel group is already initialized'
117 - all_data_parallel_group_ranks = []
118 - for i in range(pipeline_model_parallel_size):
119 - start_rank = i * num_pipeline_model_parallel_groups
120 - end_rank = (i + 1) * num_pipeline_model_parallel_groups
121 - for j in range(tensor_model_parallel_size):
122 - ranks = range(start_rank + j, end_rank, tensor_model_parallel_size)
123 - all_data_parallel_group_ranks.append(list(ranks))
124 - group = torch.distributed.new_group(ranks)
125 - if rank in ranks:
126 - _DATA_PARALLEL_GROUP = group
127 - _DATA_PARALLEL_GLOBAL_RANKS = ranks
128 -
129 - # Build the model-parallel groups.
130 - global _MODEL_PARALLEL_GROUP
131 - assert _MODEL_PARALLEL_GROUP is None, 'model parallel group is already initialized'
132 - for i in range(data_parallel_size):
133 - ranks = [data_parallel_group_ranks[i]
134 - for data_parallel_group_ranks in all_data_parallel_group_ranks]
135 - group = torch.distributed.new_group(ranks)
136 - if rank in ranks:
137 - _MODEL_PARALLEL_GROUP = group
138 -
62 # Build the tensor model-parallel groups.
63 global _TENSOR_MODEL_PARALLEL_GROUP
141 - assert _TENSOR_MODEL_PARALLEL_GROUP is None, \
142 - 'tensor model parallel group is already initialized'
64 + assert _TENSOR_MODEL_PARALLEL_GROUP is None, (
65 + "tensor model parallel group is already initialized")
66 for i in range(num_tensor_model_parallel_groups):
67 ranks = range(i * tensor_model_parallel_size,
68 (i + 1) * tensor_model_parallel_size)
@@ -147,268 +70,60 @@ def initialize_model_parallel(
70 if rank in ranks:
71 _TENSOR_MODEL_PARALLEL_GROUP = group
72
150 - # Build the pipeline model-parallel groups and embedding groups
151 - # (first and last rank in each pipeline model-parallel group).
73 + # Build the pipeline model-parallel groups.
74 global _PIPELINE_MODEL_PARALLEL_GROUP
75 global _PIPELINE_GLOBAL_RANKS
154 - assert _PIPELINE_MODEL_PARALLEL_GROUP is None, \
155 - 'pipeline model parallel group is already initialized'
156 - global _EMBEDDING_GROUP
157 - global _EMBEDDING_GLOBAL_RANKS
158 - assert _EMBEDDING_GROUP is None, 'embedding group is already initialized'
159 - global _POSITION_EMBEDDING_GROUP
160 - global _POSITION_EMBEDDING_GLOBAL_RANKS
161 - assert _POSITION_EMBEDDING_GROUP is None, \
162 - 'position embedding group is already initialized'
76 + assert _PIPELINE_MODEL_PARALLEL_GROUP is None, (
77 + "pipeline model parallel group is already initialized")
78 for i in range(num_pipeline_model_parallel_groups):
79 ranks = range(i, world_size, num_pipeline_model_parallel_groups)
80 group = torch.distributed.new_group(ranks)
81 if rank in ranks:
82 _PIPELINE_MODEL_PARALLEL_GROUP = group
83 _PIPELINE_GLOBAL_RANKS = ranks
169 - # Setup embedding group (to exchange gradients between
170 - # first and last stages).
171 - if len(ranks) > 1:
172 - embedding_ranks = [ranks[0], ranks[-1]]
173 - position_embedding_ranks = [ranks[0]]
174 - if pipeline_model_parallel_split_rank is not None:
175 - if ranks[pipeline_model_parallel_split_rank] not in embedding_ranks:
176 - embedding_ranks = [ranks[0],
177 - ranks[pipeline_model_parallel_split_rank],
178 - ranks[-1]]
179 - if ranks[pipeline_model_parallel_split_rank] not in position_embedding_ranks:
180 - position_embedding_ranks = [ranks[0],
181 - ranks[pipeline_model_parallel_split_rank]]
182 - else:
183 - embedding_ranks = ranks
184 - position_embedding_ranks = ranks
185 -
186 - group = torch.distributed.new_group(embedding_ranks)
187 - if rank in embedding_ranks:
188 - _EMBEDDING_GROUP = group
189 - if rank in ranks:
190 - _EMBEDDING_GLOBAL_RANKS = embedding_ranks
84
192 - group = torch.distributed.new_group(position_embedding_ranks)
193 - if rank in position_embedding_ranks:
194 - _POSITION_EMBEDDING_GROUP = group
195 - if rank in ranks:
196 - _POSITION_EMBEDDING_GLOBAL_RANKS = position_embedding_ranks
85
86 def model_parallel_is_initialized():
87 """Check if model and data parallel groups are initialized."""
200 - if _TENSOR_MODEL_PARALLEL_GROUP is None or \
201 - _PIPELINE_MODEL_PARALLEL_GROUP is None or \
202 - _DATA_PARALLEL_GROUP is None:
203 - return False
204 - return True
205 -
206 -
207 -def get_model_parallel_group():
208 - """Get the model parallel group the caller rank belongs to."""
209 - assert _MODEL_PARALLEL_GROUP is not None, \
210 - 'model parallel group is not initialized'
211 - return _MODEL_PARALLEL_GROUP
88 + return (_TENSOR_MODEL_PARALLEL_GROUP is not None
89 + and _PIPELINE_MODEL_PARALLEL_GROUP is not None)
90
91
92 def get_tensor_model_parallel_group():
93 """Get the tensor model parallel group the caller rank belongs to."""
216 - assert _TENSOR_MODEL_PARALLEL_GROUP is not None, \
217 - 'intra_layer_model parallel group is not initialized'
94 + assert _TENSOR_MODEL_PARALLEL_GROUP is not None, (
95 + "tenosr model parallel group is not initialized")
96 return _TENSOR_MODEL_PARALLEL_GROUP
97
98
99 def get_pipeline_model_parallel_group():
100 """Get the pipeline model parallel group the caller rank belongs to."""
223 - assert _PIPELINE_MODEL_PARALLEL_GROUP is not None, \
224 - 'pipeline_model parallel group is not initialized'
101 + assert _PIPELINE_MODEL_PARALLEL_GROUP is not None, (
102 + "pipeline model parallel group is not initialized")
103 return _PIPELINE_MODEL_PARALLEL_GROUP
104
105
228 -def get_data_parallel_group():
229 - """Get the data parallel group the caller rank belongs to."""
230 - assert _DATA_PARALLEL_GROUP is not None, \
231 - 'data parallel group is not initialized'
232 - return _DATA_PARALLEL_GROUP
233 -
234 -
235 -def get_embedding_group():
236 - """Get the embedding group the caller rank belongs to."""
237 - assert _EMBEDDING_GROUP is not None, \
238 - 'embedding group is not initialized'
239 - return _EMBEDDING_GROUP
240 -
241 -
242 -def get_position_embedding_group():
243 - """Get the position embedding group the caller rank belongs to."""
244 - assert _POSITION_EMBEDDING_GROUP is not None, \
245 - 'position embedding group is not initialized'
246 - return _POSITION_EMBEDDING_GROUP
247 -
248 -
249 -def set_tensor_model_parallel_world_size(world_size):
250 - """Set the tensor model parallel size"""
251 - global _MPU_TENSOR_MODEL_PARALLEL_WORLD_SIZE
252 - _MPU_TENSOR_MODEL_PARALLEL_WORLD_SIZE = world_size
253 -
254 -
255 -def set_pipeline_model_parallel_world_size(world_size):
256 - """Set the pipeline model parallel size"""
257 - global _MPU_PIPELINE_MODEL_PARALLEL_WORLD_SIZE
258 - _MPU_PIPELINE_MODEL_PARALLEL_WORLD_SIZE = world_size
259 -
260 -
106 def get_tensor_model_parallel_world_size():
107 """Return world size for the tensor model parallel group."""
263 - global _MPU_TENSOR_MODEL_PARALLEL_WORLD_SIZE
264 - if _MPU_TENSOR_MODEL_PARALLEL_WORLD_SIZE is not None:
265 - return _MPU_TENSOR_MODEL_PARALLEL_WORLD_SIZE
266 - return torch.distributed.get_world_size(group=get_tensor_model_parallel_group())
108 + return torch.distributed.get_world_size(
109 + group=get_tensor_model_parallel_group())
110
111
112 def get_pipeline_model_parallel_world_size():
113 """Return world size for the pipeline model parallel group."""
271 - global _MPU_PIPELINE_MODEL_PARALLEL_WORLD_SIZE
272 - if _MPU_PIPELINE_MODEL_PARALLEL_WORLD_SIZE is not None:
273 - return _MPU_PIPELINE_MODEL_PARALLEL_WORLD_SIZE
274 - return torch.distributed.get_world_size(group=get_pipeline_model_parallel_group())
275 -
276 -
277 -def set_tensor_model_parallel_rank(rank):
278 - """Set tensor model parallel rank."""
279 - global _MPU_TENSOR_MODEL_PARALLEL_RANK
280 - _MPU_TENSOR_MODEL_PARALLEL_RANK = rank
281 -
282 -
283 -def set_pipeline_model_parallel_rank(rank):
284 - """Set pipeline model parallel rank."""
285 - global _MPU_PIPELINE_MODEL_PARALLEL_RANK
286 - _MPU_PIPELINE_MODEL_PARALLEL_RANK = rank
287 -
288 -
289 -def set_pipeline_model_parallel_split_rank(rank):
290 - """Set pipeline model parallel split rank."""
291 - global _MPU_PIPELINE_MODEL_PARALLEL_SPLIT_RANK
292 - _MPU_PIPELINE_MODEL_PARALLEL_SPLIT_RANK = rank
114 + return torch.distributed.get_world_size(
115 + group=get_pipeline_model_parallel_group())
116
117
118 def get_tensor_model_parallel_rank():
119 """Return my rank for the tensor model parallel group."""
297 - global _MPU_TENSOR_MODEL_PARALLEL_RANK
298 - if _MPU_TENSOR_MODEL_PARALLEL_RANK is not None:
299 - return _MPU_TENSOR_MODEL_PARALLEL_RANK
120 return torch.distributed.get_rank(group=get_tensor_model_parallel_group())
121
122
123 def get_pipeline_model_parallel_rank():
124 """Return my rank for the pipeline model parallel group."""
305 - global _MPU_PIPELINE_MODEL_PARALLEL_RANK
306 - if _MPU_PIPELINE_MODEL_PARALLEL_RANK is not None:
307 - return _MPU_PIPELINE_MODEL_PARALLEL_RANK
308 - return torch.distributed.get_rank(group=get_pipeline_model_parallel_group())
309 -
310 -
311 -
312 -def is_pipeline_first_stage(ignore_virtual=False):
313 - """Return True if in the first pipeline model-parallel stage, False otherwise."""
314 - if not ignore_virtual:
315 - if get_virtual_pipeline_model_parallel_world_size() is not None and \
316 - get_virtual_pipeline_model_parallel_rank() != 0:
317 - return False
318 - return get_pipeline_model_parallel_rank() == 0
319 -
320 -
321 -def is_pipeline_last_stage(ignore_virtual=False):
322 - """Return True if in the last pipeline model-parallel stage, False otherwise."""
323 - if not ignore_virtual:
324 - virtual_pipeline_model_parallel_world_size = \
325 - get_virtual_pipeline_model_parallel_world_size()
326 - if virtual_pipeline_model_parallel_world_size is not None and \
327 - get_virtual_pipeline_model_parallel_rank() != (
328 - virtual_pipeline_model_parallel_world_size - 1):
329 - return False
330 - return get_pipeline_model_parallel_rank() == (
331 - get_pipeline_model_parallel_world_size() - 1)
332 -
333 -
334 -def is_rank_in_embedding_group(ignore_virtual=False):
335 - """Return true if current rank is in embedding group, False otherwise."""
336 - rank = torch.distributed.get_rank()
337 - global _EMBEDDING_GLOBAL_RANKS
338 - if ignore_virtual:
339 - return rank in _EMBEDDING_GLOBAL_RANKS
340 - if rank in _EMBEDDING_GLOBAL_RANKS:
341 - if rank == _EMBEDDING_GLOBAL_RANKS[0]:
342 - return is_pipeline_first_stage(ignore_virtual=False)
343 - elif rank == _EMBEDDING_GLOBAL_RANKS[-1]:
344 - return is_pipeline_last_stage(ignore_virtual=False)
345 - else:
346 - return True
347 - return False
348 -
349 -
350 -def is_rank_in_position_embedding_group():
351 - """Return true if current rank is in position embedding group, False otherwise."""
352 - rank = torch.distributed.get_rank()
353 - global _POSITION_EMBEDDING_GLOBAL_RANKS
354 - return rank in _POSITION_EMBEDDING_GLOBAL_RANKS
355 -
356 -
357 -def is_pipeline_stage_before_split(rank=None):
358 - """Return True if pipeline stage executes encoder block for a model
359 - with both encoder and decoder."""
360 - if get_pipeline_model_parallel_world_size() == 1:
361 - return True
362 - if rank is None:
363 - rank = get_pipeline_model_parallel_rank()
364 - global _PIPELINE_MODEL_PARALLEL_SPLIT_RANK
365 - if _PIPELINE_MODEL_PARALLEL_SPLIT_RANK is None:
366 - return True
367 - if rank < _PIPELINE_MODEL_PARALLEL_SPLIT_RANK:
368 - return True
369 - return False
370 -
371 -
372 -def is_pipeline_stage_after_split(rank=None):
373 - """Return True if pipeline stage executes decoder block for a model
374 - with both encoder and decoder."""
375 - if get_pipeline_model_parallel_world_size() == 1:
376 - return True
377 - if rank is None:
378 - rank = get_pipeline_model_parallel_rank()
379 - global _PIPELINE_MODEL_PARALLEL_SPLIT_RANK
380 - if _PIPELINE_MODEL_PARALLEL_SPLIT_RANK is None:
381 - return True
382 - if rank >= _PIPELINE_MODEL_PARALLEL_SPLIT_RANK:
383 - return True
384 - return False
385 -
386 -
387 -def is_pipeline_stage_at_split():
388 - """Return true if pipeline stage executes decoder block and next
389 - stage executes encoder block for a model with both encoder and
390 - decoder."""
391 - rank = get_pipeline_model_parallel_rank()
392 - return is_pipeline_stage_before_split(rank) and \
393 - is_pipeline_stage_after_split(rank+1)
394 -
395 -
396 -def get_virtual_pipeline_model_parallel_rank():
397 - """Return the virtual pipeline-parallel rank."""
398 - global _VIRTUAL_PIPELINE_MODEL_PARALLEL_RANK
399 - return _VIRTUAL_PIPELINE_MODEL_PARALLEL_RANK
400 -
401 -
402 -def set_virtual_pipeline_model_parallel_rank(rank):
403 - """Set the virtual pipeline-parallel rank."""
404 - global _VIRTUAL_PIPELINE_MODEL_PARALLEL_RANK
405 - _VIRTUAL_PIPELINE_MODEL_PARALLEL_RANK = rank
406 -
407 -
408 -def get_virtual_pipeline_model_parallel_world_size():
409 - """Return the virtual pipeline-parallel world size."""
410 - global _VIRTUAL_PIPELINE_MODEL_PARALLEL_WORLD_SIZE
411 - return _VIRTUAL_PIPELINE_MODEL_PARALLEL_WORLD_SIZE
125 + return torch.distributed.get_rank(
126 + group=get_pipeline_model_parallel_group())
127
128
129 def get_tensor_model_parallel_src_rank():
@@ -419,35 +134,27 @@ def get_tensor_model_parallel_src_rank():
134 return (global_rank // local_world_size) * local_world_size
135
136
422 -def get_data_parallel_src_rank():
423 - """Calculate the global rank corresponding to the first local rank
424 - in the data parallel group."""
425 - assert _DATA_PARALLEL_GLOBAL_RANKS is not None, \
426 - "Data parallel group is not initialized"
427 - return _DATA_PARALLEL_GLOBAL_RANKS[0]
428 -
429 -
137 def get_pipeline_model_parallel_first_rank():
138 """Return the global rank of the first process in the pipeline for the
139 current tensor parallel group"""
433 - assert _PIPELINE_GLOBAL_RANKS is not None, \
434 - "Pipeline parallel group is not initialized"
140 + assert _PIPELINE_GLOBAL_RANKS is not None, (
141 + "Pipeline parallel group is not initialized")
142 return _PIPELINE_GLOBAL_RANKS[0]
143
144
145 def get_pipeline_model_parallel_last_rank():
146 """Return the global rank of the last process in the pipeline for the
147 current tensor parallel group"""
441 - assert _PIPELINE_GLOBAL_RANKS is not None, \
442 - "Pipeline parallel group is not initialized"
148 + assert _PIPELINE_GLOBAL_RANKS is not None, (
149 + "Pipeline parallel group is not initialized")
150 last_rank_local = get_pipeline_model_parallel_world_size() - 1
151 return _PIPELINE_GLOBAL_RANKS[last_rank_local]
152
153
154 def get_pipeline_model_parallel_next_rank():
155 """Return the global rank that follows the caller in the pipeline"""
449 - assert _PIPELINE_GLOBAL_RANKS is not None, \
450 - "Pipeline parallel group is not initialized"
156 + assert _PIPELINE_GLOBAL_RANKS is not None, (
157 + "Pipeline parallel group is not initialized")
158 rank_in_pipeline = get_pipeline_model_parallel_rank()
159 world_size = get_pipeline_model_parallel_world_size()
160 return _PIPELINE_GLOBAL_RANKS[(rank_in_pipeline + 1) % world_size]
@@ -455,45 +162,18 @@ def get_pipeline_model_parallel_next_rank():
162
163 def get_pipeline_model_parallel_prev_rank():
164 """Return the global rank that preceeds the caller in the pipeline"""
458 - assert _PIPELINE_GLOBAL_RANKS is not None, \
459 - "Pipeline parallel group is not initialized"
165 + assert _PIPELINE_GLOBAL_RANKS is not None, (
166 + "Pipeline parallel group is not initialized")
167 rank_in_pipeline = get_pipeline_model_parallel_rank()
168 world_size = get_pipeline_model_parallel_world_size()
169 return _PIPELINE_GLOBAL_RANKS[(rank_in_pipeline - 1) % world_size]
170
171
465 -def get_data_parallel_world_size():
466 - """Return world size for the data parallel group."""
467 - return torch.distributed.get_world_size(group=get_data_parallel_group())
468 -
469 -
470 -def get_data_parallel_rank():
471 - """Return my rank for the data parallel group."""
472 - return torch.distributed.get_rank(group=get_data_parallel_group())
473 -
172 def destroy_model_parallel():
173 """Set the groups to none."""
476 - global _MODEL_PARALLEL_GROUP
477 - _MODEL_PARALLEL_GROUP = None
174 global _TENSOR_MODEL_PARALLEL_GROUP
175 _TENSOR_MODEL_PARALLEL_GROUP = None
176 global _PIPELINE_MODEL_PARALLEL_GROUP
177 _PIPELINE_MODEL_PARALLEL_GROUP = None
482 - global _DATA_PARALLEL_GROUP
483 - _DATA_PARALLEL_GROUP = None
484 - global _EMBEDDING_GROUP
485 - _EMBEDDING_GROUP = None
486 - global _POSITION_EMBEDDING_GROUP
487 - _POSITION_EMBEDDING_GROUP = None
488 - global _VIRTUAL_PIPELINE_MODEL_PARALLEL_RANK
489 - _VIRTUAL_PIPELINE_MODEL_PARALLEL_RANK = None
490 - global _VIRTUAL_PIPELINE_MODEL_PARALLEL_WORLD_SIZE
491 - _VIRTUAL_PIPELINE_MODEL_PARALLEL_WORLD_SIZE = None
492 - global _MPU_TENSOR_MODEL_PARALLEL_WORLD_SIZE
493 - _MPU_TENSOR_MODEL_PARALLEL_WORLD_SIZE = None
494 - global _MPU_PIPELINE_MODEL_PARALLEL_WORLD_SIZE
495 - _MPU_PIPELINE_MODEL_PARALLEL_WORLD_SIZE = None
496 - global _MPU_TENSOR_MODEL_PARALLEL_RANK
497 - _MPU_TENSOR_MODEL_PARALLEL_RANK = None
498 - global _MPU_PIPELINE_MODEL_PARALLEL_RANK
499 - _MPU_PIPELINE_MODEL_PARALLEL_RANK = None
178 + global _PIPELINE_GLOBAL_RANKS
179 + _PIPELINE_GLOBAL_RANKS = None
vllm/model_executor/parallel_utils/tensor_parallel/__init__.py deleted
-50
@@ -1,50 +0,0 @@
1 -from .layers import (
2 - ColumnParallelLinear,
3 - RowParallelLinear,
4 - VocabParallelEmbedding,
5 - set_tensor_model_parallel_attributes,
6 - set_defaults_if_not_set_tensor_model_parallel_attributes,
7 - copy_tensor_model_parallel_attributes,
8 - param_is_not_tensor_parallel_duplicate,
9 -)
10 -
11 -from .mappings import (
12 - copy_to_tensor_model_parallel_region,
13 - gather_from_tensor_model_parallel_region,
14 - gather_from_sequence_parallel_region,
15 - reduce_from_tensor_model_parallel_region,
16 - scatter_to_tensor_model_parallel_region,
17 - scatter_to_sequence_parallel_region,
18 -)
19 -
20 -from .random import (
21 - get_cuda_rng_tracker,
22 - model_parallel_cuda_manual_seed,
23 -)
24 -
25 -from .utils import (
26 - split_tensor_along_last_dim,
27 -)
28 -
29 -__all__ = [
30 - #layers.py
31 - "ColumnParallelLinear",
32 - "RowParallelLinear",
33 - "VocabParallelEmbedding",
34 - "set_tensor_model_parallel_attributes",
35 - "set_defaults_if_not_set_tensor_model_parallel_attributes",
36 - "copy_tensor_model_parallel_attributes",
37 - "param_is_not_tensor_parallel_duplicate",
38 - # mappings.py
39 - "copy_to_tensor_model_parallel_region",
40 - "gather_from_tensor_model_parallel_region",
41 - "gather_from_sequence_parallel_region",
42 - "reduce_from_tensor_model_parallel_region",
43 - "scatter_to_tensor_model_parallel_region",
44 - "scatter_to_sequence_parallel_region",
45 - # random.py
46 - "get_cuda_rng_tracker",
47 - "model_parallel_cuda_manual_seed",
48 - # utils.py
49 - "split_tensor_along_last_dim",
50 -]
vllm/model_executor/parallel_utils/tensor_parallel/layers.py deleted
-366
@@ -1,366 +0,0 @@
1 -# Copyright 2023 The vLLM team.
2 -# Adapted from https://github.com/NVIDIA/Megatron-LM/blob/main/megatron/core/tensor_parallel/layers.py
3 -# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
4 -
5 -# Parts of the code here are adapted from PyTorch
6 -# repo: https://github.com/pytorch/pytorch
7 -from typing import Optional
8 -
9 -import torch
10 -import torch.nn.functional as F
11 -import torch.nn.init as init
12 -from torch.nn.parameter import Parameter
13 -
14 -from vllm.model_executor.parallel_utils.parallel_state import (
15 - get_tensor_model_parallel_rank,
16 - get_tensor_model_parallel_world_size,
17 -)
18 -from .mappings import (
19 - gather_from_tensor_model_parallel_region,
20 - reduce_from_tensor_model_parallel_region,
21 - scatter_to_tensor_model_parallel_region,
22 -)
23 -
24 -from .utils import (
25 - divide,
26 - VocabUtility,
27 -)
28 -
29 -_MODEL_PARALLEL_ATTRIBUTE_DEFAULTS = {'tensor_model_parallel': False,
30 - 'partition_dim': -1,
31 - 'partition_stride': 1}
32 -
33 -def param_is_not_tensor_parallel_duplicate(param):
34 - return (hasattr(param, 'tensor_model_parallel') and
35 - param.tensor_model_parallel) or (
36 - get_tensor_model_parallel_rank() == 0)
37 -
38 -
39 -def set_tensor_model_parallel_attributes(tensor, is_parallel, dim, stride):
40 - # Make sure the attributes are not set.
41 - for attribute in _MODEL_PARALLEL_ATTRIBUTE_DEFAULTS:
42 - assert not hasattr(tensor, attribute)
43 - # Set the attributes.
44 - setattr(tensor, 'tensor_model_parallel', is_parallel)
45 - setattr(tensor, 'partition_dim', dim)
46 - setattr(tensor, 'partition_stride', stride)
47 -
48 -
49 -def set_defaults_if_not_set_tensor_model_parallel_attributes(tensor):
50 - def maybe_set(attribute, value):
51 - if not hasattr(tensor, attribute):
52 - setattr(tensor, attribute, value)
53 - for attribute in _MODEL_PARALLEL_ATTRIBUTE_DEFAULTS:
54 - maybe_set(attribute, _MODEL_PARALLEL_ATTRIBUTE_DEFAULTS[attribute])
55 -
56 -
57 -def copy_tensor_model_parallel_attributes(destination_tensor, source_tensor):
58 - def maybe_copy(attribute):
59 - if hasattr(source_tensor, attribute):
60 - setattr(destination_tensor, attribute,
61 - getattr(source_tensor, attribute))
62 - for attribute in _MODEL_PARALLEL_ATTRIBUTE_DEFAULTS:
63 - maybe_copy(attribute)
64 -
65 -
66 -class VocabParallelEmbedding(torch.nn.Module):
67 - """Embedding parallelized in the vocabulary dimension.
68 -
69 - This is mainly adapted from torch.nn.Embedding and all the default
70 - values are kept.
71 - Arguments:
72 - num_embeddings: vocabulary size.
73 - embedding_dim: size of hidden state.
74 -
75 - Keyword Arguments:
76 - init_method: method to initialize weights.
77 - params_dtype
78 - use_cpu_initialization
79 - perform_initialization
80 - """
81 -
82 - def __init__(self, num_embeddings: int, embedding_dim: int, *,
83 - init_method=init.xavier_normal_,
84 - params_dtype: torch.dtype=None,
85 - use_cpu_initialization: bool=False,
86 - perform_initialization: bool=False):
87 - super(VocabParallelEmbedding, self).__init__()
88 - assert not perform_initialization
89 - assert not use_cpu_initialization
90 -
91 - # Keep the input dimensions.
92 - self.num_embeddings = num_embeddings
93 - self.embedding_dim = embedding_dim
94 - if params_dtype is None:
95 - params_dtype = torch.get_default_dtype()
96 -
97 - # Set the defaults for compatibility.
98 - self.padding_idx = None
99 - self.max_norm = None
100 - self.norm_type = 2.
101 - self.scale_grad_by_freq = False
102 - self.sparse = False
103 - self._weight = None
104 - self.tensor_model_parallel_size = get_tensor_model_parallel_world_size()
105 - # Divide the weight matrix along the vocaburaly dimension.
106 - self.vocab_start_index, self.vocab_end_index = \
107 - VocabUtility.vocab_range_from_global_vocab_size(
108 - self.num_embeddings, get_tensor_model_parallel_rank(),
109 - self.tensor_model_parallel_size)
110 - self.num_embeddings_per_partition = self.vocab_end_index - \
111 - self.vocab_start_index
112 -
113 - self.weight = Parameter(torch.empty(
114 - self.num_embeddings_per_partition, self.embedding_dim,
115 - device=torch.cuda.current_device(), dtype=params_dtype))
116 -
117 - def forward(self, input_):
118 - if self.tensor_model_parallel_size > 1:
119 - # Build the mask.
120 - input_mask = (input_ < self.vocab_start_index) | \
121 - (input_ >= self.vocab_end_index)
122 - # Mask the input.
123 - masked_input = input_.clone() - self.vocab_start_index
124 - masked_input[input_mask] = 0
125 - else:
126 - masked_input = input_
127 - # Get the embeddings.
128 - output_parallel = F.embedding(masked_input, self.weight,
129 - self.padding_idx, self.max_norm,
130 - self.norm_type, self.scale_grad_by_freq,
131 - self.sparse)
132 - # Mask the output embedding.
133 - if self.tensor_model_parallel_size > 1:
134 - output_parallel[input_mask, :] = 0.0
135 - # Reduce across all the model parallel GPUs.
136 - output = reduce_from_tensor_model_parallel_region(output_parallel)
137 - return output
138 -
139 -
140 -class ColumnParallelLinear(torch.nn.Module):
141 - """Linear layer with column parallelism.
142 -
143 - The linear layer is defined as Y = XA + b. A is parallelized along
144 - its second dimension as A = [A_1, ..., A_p].
145 -
146 - Arguments:
147 - input_size: first dimension of matrix A.
148 - output_size: second dimension of matrix A.
149 -
150 - Keyword Arguments
151 - bias: If true, add bias
152 - gather_output: If true, call all-gather on output and make Y available
153 - to all GPUs, otherwise, every GPU will have its output
154 - which is Y_i = XA_i
155 - init_method: method to initialize weights. Note that bias is always set
156 - to zero.
157 - stride: For the strided linear layers.
158 - keep_master_weight_for_test: This was added for testing and should be
159 - set to False. It returns the master weights
160 - used for initialization.
161 - skip_bias_add: This was added to enable performance optimations where bias
162 - can be fused with other elementwise operations. we skip
163 - adding bias but instead return it.
164 - params_dtype:
165 - use_cpu_initialization:
166 - """
167 -
168 - def __init__(self, input_size, output_size, *,
169 - bias=True, gather_output=True,
170 - init_method=init.xavier_normal_, stride=1,
171 - keep_master_weight_for_test=False,
172 - skip_bias_add=False,
173 - params_dtype=None,
174 - use_cpu_initialization=False,
175 - perform_initialization=False,
176 - quant_config=None,
177 - ):
178 - super(ColumnParallelLinear, self).__init__()
179 - assert not perform_initialization
180 - assert not use_cpu_initialization
181 -
182 - # Keep input parameters
183 - self.input_size = input_size
184 - self.output_size = output_size
185 - self.gather_output = gather_output
186 - # Divide the weight matrix along the last dimension.
187 - self.world_size = get_tensor_model_parallel_world_size()
188 - self.output_size_per_partition = divide(output_size, self.world_size)
189 - self.skip_bias_add = skip_bias_add
190 - self.quant_config = quant_config
191 -
192 - if params_dtype is None:
193 - params_dtype = torch.get_default_dtype()
194 -
195 - # Parameters.
196 - # Note: torch.nn.functional.linear performs XA^T + b and as a result
197 - # we allocate the transpose.
198 - self.create_weights(params_dtype)
199 -
200 - if bias:
201 - self.bias = Parameter(torch.empty(
202 - self.output_size_per_partition,
203 - device=torch.cuda.current_device(),
204 - dtype=params_dtype))
205 - set_tensor_model_parallel_attributes(self.bias, True, 0, stride)
206 - # Always initialize bias to zero.
207 - with torch.no_grad():
208 - self.bias.zero_()
209 - else:
210 - self.register_parameter('bias', None)
211 -
212 - def create_weights(self, dtype: torch.dtype) -> None:
213 - self.weight = Parameter(torch.empty(
214 - self.output_size_per_partition, self.input_size,
215 - device=torch.cuda.current_device(), dtype=dtype))
216 -
217 - def apply_weights(
218 - self,
219 - x: torch.Tensor,
220 - bias: Optional[torch.Tensor],
221 - ) -> torch.Tensor:
222 - return F.linear(x, self.weight, bias)
223 -
224 - def forward(self, input_):
225 - """Forward of ColumnParallelLinear
226 -
227 - Args:
228 - input_: 3D tensor whose order of dimension is [sequence, batch, hidden]
229 -
230 - Returns:
231 - - output
232 - - bias
233 - """
234 - bias = self.bias if not self.skip_bias_add else None
235 -
236 - input_parallel = input_
237 - # Matrix multiply.
238 - output_parallel = self.apply_weights(input_parallel, bias)
239 - if self.gather_output:
240 - # All-gather across the partitions.
241 - output = gather_from_tensor_model_parallel_region(output_parallel)
242 - else:
243 - output = output_parallel
244 - output_bias = self.bias if self.skip_bias_add else None
245 - return output, output_bias
246 -
247 -
248 -class RowParallelLinear(torch.nn.Module):
249 - """Linear layer with row parallelism.
250 -
251 - The linear layer is defined as Y = XA + b. A is parallelized along
252 - its first dimension and X along its second dimension as:
253 - - -
254 - | A_1 |
255 - | . |
256 - A = | . | X = [X_1, ..., X_p]
257 - | . |
258 - | A_p |
259 - - -
260 - Arguments:
261 - input_size: first dimension of matrix A.
262 - output_size: second dimension of matrix A.
263 -
264 - Keyword Arguments:
265 - bias: If true, add bias. Note that bias is not parallelized.
266 - input_is_parallel: If true, we assume that the input is already
267 - split across the GPUs and we do not split
268 - again.
269 - init_method: method to initialize weights. Note that bias is always set
270 - to zero.
271 - stride: For the strided linear layers.
272 - keep_master_weight_for_test: This was added for testing and should be
273 - set to False. It returns the master weights
274 - used for initialization.
275 - skip_bias_add: This was added to enable performance optimization where bias
276 - can be fused with other elementwise operations. We skip
277 - adding bias but instead return it.
278 - params_dtype:
279 - use_cpu_initialization:
280 - perform_initialization:
281 - reduce_results:
282 - """
283 -
284 - def __init__(self, input_size, output_size, *,
285 - bias=True, input_is_parallel=False,
286 - init_method=init.xavier_normal_, stride=1,
287 - keep_master_weight_for_test=False,
288 - skip_bias_add=False,
289 - params_dtype=None,
290 - use_cpu_initialization=False,
291 - perform_initialization=False,
292 - reduce_results=True,
293 - quant_config=None,
294 - ):
295 - super(RowParallelLinear, self).__init__()
296 - assert not perform_initialization
297 - assert not use_cpu_initialization
298 -
299 - # Keep input parameters
300 - self.input_size = input_size
301 - self.output_size = output_size
302 - self.input_is_parallel = input_is_parallel
303 - self.reduce_results = reduce_results
304 - if params_dtype is None:
305 - params_dtype = torch.get_default_dtype()
306 -
307 - # Divide the weight matrix along the last dimension.
308 - self.world_size = get_tensor_model_parallel_world_size()
309 - self.input_size_per_partition = divide(input_size, self.world_size)
310 - self.skip_bias_add = skip_bias_add
311 - self.quant_config = quant_config
312 -
313 - self.create_weights(params_dtype)
314 -
315 - if not reduce_results and (bias and not skip_bias_add):
316 - raise ValueError("When not reduce the results, adding bias to the "
317 - "results can lead to incorrect results")
318 -
319 - if bias:
320 - self.bias = Parameter(torch.empty(
321 - self.output_size, device=torch.cuda.current_device(),
322 - dtype=params_dtype))
323 -
324 - # Always initialize bias to zero.
325 - with torch.no_grad():
326 - self.bias.zero_()
327 - else:
328 - self.register_parameter('bias', None)
329 -
330 - def create_weights(self, dtype: torch.dtype) -> None:
331 - self.weight = Parameter(torch.empty(
332 - self.output_size, self.input_size_per_partition,
333 - device=torch.cuda.current_device(), dtype=dtype))
334 -
335 - def apply_weights(self, x: torch.Tensor) -> torch.Tensor:
336 - return F.linear(x, self.weight)
337 -
338 - def forward(self, input_):
339 - """Forward of RowParallelLinear
340 -
341 - Args:
342 - input_: 3D tensor whose order of dimension is [sequence, batch, hidden]
343 -
344 - Returns:
345 - - output
346 - - bias
347 - """
348 - # Set up backprop all-reduce.
349 - if self.input_is_parallel:
350 - input_parallel = input_
351 - else:
352 - input_parallel = scatter_to_tensor_model_parallel_region(input_)
353 - # Matrix multiply.
354 - output_parallel = self.apply_weights(input_parallel)
355 - if self.reduce_results and self.world_size > 1:
356 - output_ = reduce_from_tensor_model_parallel_region(output_parallel)
357 - else:
358 - output_ = output_parallel
359 -
360 - if not self.skip_bias_add:
361 - output = output_ + self.bias if self.bias is not None else output_
362 - output_bias = None
363 - else:
364 - output = output_
365 - output_bias = self.bias
366 - return output, output_bias
vllm/model_executor/parallel_utils/tensor_parallel/mappings.py deleted
-281
@@ -1,281 +0,0 @@
1 -# Copyright 2023 The vLLM team.
2 -# Adapted from https://github.com/NVIDIA/Megatron-LM/blob/main/megatron/core/tensor_parallel/mappings.py
3 -# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
4 -
5 -import torch
6 -
7 -from vllm.model_executor.parallel_utils.parallel_state import (
8 - get_tensor_model_parallel_rank,
9 - get_tensor_model_parallel_world_size,
10 - get_tensor_model_parallel_group,
11 -)
12 -from .utils import split_tensor_along_last_dim
13 -
14 -
15 -def _reduce(input_):
16 - """All-reduce the input tensor across model parallel group."""
17 -
18 - # Bypass the function if we are using only 1 GPU.
19 - if get_tensor_model_parallel_world_size()==1:
20 - return input_
21 -
22 - # All-reduce.
23 - torch.distributed.all_reduce(input_, group=get_tensor_model_parallel_group())
24 -
25 - return input_
26 -
27 -
28 -def _split_along_last_dim(input_):
29 - """Split the tensor along its last dimension and keep the
30 - corresponding slice."""
31 -
32 - world_size = get_tensor_model_parallel_world_size()
33 - # Bypass the function if we are using only 1 GPU.
34 - if world_size == 1:
35 - return input_
36 -
37 - # Split along last dimension.
38 - input_list = split_tensor_along_last_dim(input_, world_size)
39 -
40 - # Note: torch.split does not create contiguous tensors by default.
41 - rank = get_tensor_model_parallel_rank()
42 - output = input_list[rank].contiguous()
43 -
44 - return output
45 -
46 -
47 -def _split_along_first_dim(input_):
48 - """Split the tensor along its first dimension and keep the
49 - corresponding slice."""
50 -
51 - world_size = get_tensor_model_parallel_world_size()
52 - # Bypass the function if we are using only 1 GPU.
53 - if world_size == 1:
54 - return input_
55 -
56 - # Split along first dimension.
57 - dim_size = input_.size()[0]
58 - assert dim_size % world_size == 0, \
59 - "First dimension of the tensor should be divisible by tensor parallel size"
60 - local_dim_size = dim_size // world_size
61 - rank = get_tensor_model_parallel_rank()
62 - dim_offset = rank * local_dim_size
63 -
64 - output = input_[dim_offset:dim_offset+local_dim_size].contiguous()
65 -
66 - return output
67 -
68 -
69 -def _gather_along_last_dim(input_):
70 - """Gather tensors and concatinate along the last dimension."""
71 -
72 - world_size = get_tensor_model_parallel_world_size()
73 - # Bypass the function if we are using only 1 GPU.
74 - if world_size == 1:
75 - return input_
76 -
77 - # Size and dimension.
78 - last_dim = input_.dim() - 1
79 - rank = get_tensor_model_parallel_rank()
80 -
81 - tensor_list = [torch.empty_like(input_) for _ in range(world_size)]
82 - tensor_list[rank] = input_
83 - torch.distributed.all_gather(tensor_list, input_, group=get_tensor_model_parallel_group())
84 -
85 - # Note: torch.cat already creates a contiguous tensor.
86 - output = torch.cat(tensor_list, dim=last_dim).contiguous()
87 -
88 - return output
89 -
90 -
91 -def _gather_along_first_dim(input_):
92 - """Gather tensors and concatinate along the first dimension."""
93 -
94 - world_size = get_tensor_model_parallel_world_size()
95 - # Bypass the function if we are using only 1 GPU.
96 - if world_size == 1:
97 - return input_
98 -
99 - dim_size = list(input_.size())
100 - dim_size[0] = dim_size[0] * world_size
101 -
102 - output = torch.empty(dim_size, dtype=input_.dtype,
103 - device=torch.cuda.current_device())
104 - torch.distributed._all_gather_base(output, input_.contiguous(),
105 - group=get_tensor_model_parallel_group())
106 -
107 - return output
108 -
109 -def _reduce_scatter_along_first_dim(input_):
110 - """Reduce-scatter the input tensor across model parallel group."""
111 - world_size = get_tensor_model_parallel_world_size()
112 - # Bypass the function if we are using only 1 GPU.
113 - if world_size == 1:
114 - return input_
115 -
116 - dim_size = list(input_.size())
117 - assert dim_size[0] % world_size == 0, \
118 - "First dimension of the tensor should be divisible by tensor parallel size"
119 -
120 - dim_size[0] = dim_size[0] // world_size
121 -
122 - output = torch.empty(dim_size, dtype=input_.dtype,
123 - device=torch.cuda.current_device())
124 - torch.distributed._reduce_scatter_base(output, input_.contiguous(),
125 - group=get_tensor_model_parallel_group())
126 - return output
127 -
128 -
129 -class _CopyToModelParallelRegion(torch.autograd.Function):
130 - """Pass the input to the model parallel region."""
131 -
132 - @staticmethod
133 - def symbolic(graph, input_):
134 - return input_
135 -
136 - @staticmethod
137 - def forward(ctx, input_):
138 - return input_
139 -
140 - @staticmethod
141 - def backward(ctx, grad_output):
142 - return _reduce(grad_output)
143 -
144 -
145 -class _ReduceFromModelParallelRegion(torch.autograd.Function):
146 - """All-reduce the input from the model parallel region."""
147 -
148 - @staticmethod
149 - def symbolic(graph, input_):
150 - return _reduce(input_)
151 -
152 - @staticmethod
153 - def forward(ctx, input_):
154 - return _reduce(input_)
155 -
156 - @staticmethod
157 - def backward(ctx, grad_output):
158 - return grad_output
159 -
160 -
161 -class _ScatterToModelParallelRegion(torch.autograd.Function):
162 - """Split the input and keep only the corresponding chuck to the rank."""
163 -
164 - @staticmethod
165 - def symbolic(graph, input_):
166 - return _split_along_last_dim(input_)
167 -
168 - @staticmethod
169 - def forward(ctx, input_):
170 - return _split_along_last_dim(input_)
171 -
172 - @staticmethod
173 - def backward(ctx, grad_output):
174 - return _gather_along_last_dim(grad_output)
175 -
176 -
177 -class _GatherFromModelParallelRegion(torch.autograd.Function):
178 - """Gather the input from model parallel region and concatinate."""
179 -
180 - @staticmethod
181 - def symbolic(graph, input_):
182 - return _gather_along_last_dim(input_)
183 -
184 - @staticmethod
185 - def forward(ctx, input_):
186 - return _gather_along_last_dim(input_)
187 -
188 - @staticmethod
189 - def backward(ctx, grad_output):
190 - return _split_along_last_dim(grad_output)
191 -
192 -
193 -class _ScatterToSequenceParallelRegion(torch.autograd.Function):
194 - """Split the input and keep only the corresponding chuck to the rank."""
195 -
196 - @staticmethod
197 - def symbolic(graph, input_):
198 - return _split_along_first_dim(input_)
199 -
200 - @staticmethod
201 - def forward(ctx, input_):
202 - return _split_along_first_dim(input_)
203 -
204 - @staticmethod
205 - def backward(ctx, grad_output):
206 - return _gather_along_first_dim(grad_output)
207 -
208 -
209 -class _GatherFromSequenceParallelRegion(torch.autograd.Function):
210 - """Gather the input from sequence parallel region and concatinate."""
211 -
212 - @staticmethod
213 - def symbolic(graph, input_, tensor_parallel_output_grad=True):
214 - return _gather_along_first_dim(input_)
215 -
216 - @staticmethod
217 - def forward(ctx, input_, tensor_parallel_output_grad=True):
218 - ctx.tensor_parallel_output_grad = tensor_parallel_output_grad
219 - return _gather_along_first_dim(input_)
220 -
221 - @staticmethod
222 - def backward(ctx, grad_output):
223 - tensor_parallel_output_grad = ctx.tensor_parallel_output_grad
224 -
225 - # If the computation graph after the gather operation is
226 - # in the tensor parallel mode, output gradients need to reduce
227 - # scattered and whereas if the computation is duplicated,
228 - # output gradients need to be scattered.
229 - if tensor_parallel_output_grad:
230 - return _reduce_scatter_along_first_dim(grad_output), None
231 - else:
232 - return _split_along_first_dim(grad_output), None
233 -
234 -
235 -class _ReduceScatterToSequenceParallelRegion(torch.autograd.Function):
236 - """Reduce scatter the input from the model parallel region."""
237 -
238 - @staticmethod
239 - def symbolic(graph, input_):
240 - return _reduce_scatter_along_first_dim(input_)
241 -
242 - @staticmethod
243 - def forward(ctx, input_):
244 - return _reduce_scatter_along_first_dim(input_)
245 -
246 - @staticmethod
247 - def backward(ctx, grad_output):
248 - return _gather_along_first_dim(grad_output)
249 -
250 -
251 -# -----------------
252 -# Helper functions.
253 -# -----------------
254 -
255 -def copy_to_tensor_model_parallel_region(input_):
256 - return _CopyToModelParallelRegion.apply(input_)
257 -
258 -
259 -def reduce_from_tensor_model_parallel_region(input_):
260 - return _ReduceFromModelParallelRegion.apply(input_)
261 -
262 -
263 -def scatter_to_tensor_model_parallel_region(input_):
264 - return _ScatterToModelParallelRegion.apply(input_)
265 -
266 -
267 -def gather_from_tensor_model_parallel_region(input_):
268 - return _GatherFromModelParallelRegion.apply(input_)
269 -
270 -
271 -def scatter_to_sequence_parallel_region(input_):
272 - return _ScatterToSequenceParallelRegion.apply(input_)
273 -
274 -
275 -def gather_from_sequence_parallel_region(input_, tensor_parallel_output_grad=True):
276 - return _GatherFromSequenceParallelRegion.apply(input_, tensor_parallel_output_grad)
277 -
278 -
279 -def reduce_scatter_to_sequence_parallel_region(input_):
280 - return _ReduceScatterToSequenceParallelRegion.apply(input_)
281 -
vllm/model_executor/parallel_utils/tensor_parallel/random.py deleted
-164
@@ -1,164 +0,0 @@
1 -# Copyright 2023 The vLLM team.
2 -# Adapted from https://github.com/NVIDIA/Megatron-LM/blob/main/megatron/core/tensor_parallel/random.py
3 -# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
4 -
5 -# Parts of the code here are adapted from PyTorch
6 -# repo: https://github.com/pytorch/pytorch
7 -
8 -import contextlib
9 -
10 -import torch
11 -from torch import _C
12 -from torch.cuda import _lazy_call, device as device_ctx_manager
13 -
14 -from vllm.model_executor.parallel_utils.parallel_state import (
15 - get_tensor_model_parallel_rank,
16 -)
17 -
18 -# Default name for the model parallel rng tracker.
19 -_MODEL_PARALLEL_RNG_TRACKER_NAME = 'model-parallel-rng'
20 -
21 -
22 -def _set_cuda_rng_state(new_state, device=-1):
23 - """Sets the random number generator state of the current GPU.
24 -
25 - Argumentss:
26 - new_state (torch.ByteTensor): The desired state
27 - This function is adapted from PyTorch repo (torch.cuda.set_rng_state)
28 - with a single change: the input state is not cloned. Cloning caused
29 - major performance issues for +4 GPU cases.
30 - """
31 - if hasattr(_C, '_cuda_setRNGState') and callable(_C._cuda_setRNGState):
32 - # older PyTorch
33 - def cb():
34 - with device_ctx_manager(device):
35 - _C._cuda_setRNGState(new_state)
36 - else:
37 - # newer PyTorch
38 - if device == -1:
39 - device = torch.device('cuda')
40 - elif isinstance(device, str):
41 - device = torch.device(device)
42 - elif isinstance(device, int):
43 - device = torch.device('cuda', device)
44 -
45 - def cb():
46 - idx = device.index
47 - if idx is None:
48 - idx = torch.cuda.current_device()
49 - default_generator = torch.cuda.default_generators[idx]
50 - default_generator.set_state(new_state)
51 -
52 - _lazy_call(cb)
53 -
54 -
55 -
56 -class CudaRNGStatesTracker:
57 - """Tracker for the cuda RNG states.
58 -
59 - Using the `add` method, a cuda rng state is initialized based on
60 - the input `seed` and is assigned to `name`. Later, by forking the
61 - rng state, we can perform operations and return to our starting
62 - cuda state.
63 - """
64 -
65 - def __init__(self):
66 - # Map from a string name to the cuda rng state.
67 - self.states_ = {}
68 - # Seeds are just for book keeping and ensure no seed is set twice.
69 - self.seeds_ = set()
70 -
71 - def reset(self):
72 - """Set to the initial state (no tracker)."""
73 - self.states_ = {}
74 - self.seeds_ = set()
75 -
76 - def get_states(self):
77 - """Get rng states. Copy the dictionary so we have direct
78 - pointers to the states, not just a pointer to the dictionary."""
79 - states = {}
80 - for name in self.states_:
81 - states[name] = self.states_[name]
82 - return states
83 -
84 - def set_states(self, states):
85 - """Set the rng states. For efficiency purposes, we do not check
86 - the size of seed for compatibility."""
87 - self.states_ = states
88 -
89 - def add(self, name, seed):
90 - """Track the rng state."""
91 - # Check seed is not already used.
92 - if seed in self.seeds_:
93 - raise Exception('seed {} already exists'.format(seed))
94 - self.seeds_.add(seed)
95 - # Check that state is not already defined.
96 - if name in self.states_:
97 - raise Exception('cuda rng state {} already exists'.format(name))
98 - # Get the current rng state.
99 - orig_rng_state = torch.cuda.get_rng_state()
100 - # Set the new state and store it.
101 - torch.cuda.manual_seed(seed)
102 - self.states_[name] = torch.cuda.get_rng_state()
103 - # Reset rng state to what it was.
104 - _set_cuda_rng_state(orig_rng_state)
105 -
106 - @contextlib.contextmanager
107 - def fork(self, name=_MODEL_PARALLEL_RNG_TRACKER_NAME):
108 - """Fork the cuda rng state, perform operations, and exit with
109 - the original state."""
110 - # Check if we have added the state
111 - if name not in self.states_:
112 - raise Exception('cuda rng state {} is not added'.format(name))
113 - # Store current rng state.
114 - orig_cuda_rng_state = torch.cuda.get_rng_state()
115 - # Set rng state to the desired one
116 - _set_cuda_rng_state(self.states_[name])
117 - # Do the stuff we wanted to do.
118 - try:
119 - yield
120 - finally:
121 - # Update the current rng state for later use.
122 - self.states_[name] = torch.cuda.get_rng_state()
123 - # And set the state to the original state we started with.
124 - _set_cuda_rng_state(orig_cuda_rng_state)
125 -
126 -
127 -# RNG tracker object.
128 -_CUDA_RNG_STATE_TRACKER = CudaRNGStatesTracker()
129 -
130 -
131 -def get_cuda_rng_tracker():
132 - """Get cuda rng tracker."""
133 - return _CUDA_RNG_STATE_TRACKER
134 -
135 -
136 -def model_parallel_cuda_manual_seed(seed):
137 - """Initialize model parallel cuda seed.
138 -
139 - This function should be called after the model parallel is
140 - initialized. Also, no torch.cuda.manual_seed should be called
141 - after this function. Basically, this is replacement for that
142 - function.
143 - Two set of RNG states are tracked:
144 - default state: This is for data parallelism and is the same among a
145 - set of model parallel GPUs but different across
146 - different model paralle groups. This is used for
147 - example for dropout in the non-tensor-model-parallel regions.
148 - tensor-model-parallel state: This state is different among a set of model
149 - parallel GPUs, but the same across data parallel
150 - groups. This is used for example for dropout in
151 - model parallel regions.
152 - """
153 - # 2718 is just for fun and any POSITIVE value will work.
154 - offset = seed + 2718
155 - tensor_model_parallel_seed = offset + get_tensor_model_parallel_rank()
156 - # Data parallel gets the original seed.
157 - data_parallel_seed = seed
158 -
159 - _CUDA_RNG_STATE_TRACKER.reset()
160 - # Set the default state.
161 - torch.cuda.manual_seed(data_parallel_seed)
162 - # and model parallel state.
163 - _CUDA_RNG_STATE_TRACKER.add(_MODEL_PARALLEL_RNG_TRACKER_NAME,
164 - tensor_model_parallel_seed)
vllm/model_executor/parallel_utils/utils.py renamed
+9 -9
@@ -1,15 +1,16 @@
1 # Copyright 2023 The vLLM team.
2 -# Adapted from https://github.com/NVIDIA/Megatron-LM/blob/main/megatron/core/tensor_parallel/utils.py
2 +# Adapted from
3 +# https://github.com/NVIDIA/Megatron-LM/blob/main/megatron/core/tensor_parallel/utils.py
4 # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
5 +from typing import List, Sequence
6
7 import torch
6 -from typing import List, Sequence
8 +
9
10 def ensure_divisibility(numerator, denominator):
11 """Ensure that numerator is divisible by the denominator."""
12 assert numerator % denominator == 0, "{} is not divisible by {}".format(
11 - numerator, denominator
12 - )
13 + numerator, denominator)
14
15
16 def divide(numerator, denominator):
@@ -56,15 +57,14 @@ class VocabUtility:
57
58 @staticmethod
59 def vocab_range_from_per_partition_vocab_size(
59 - per_partition_vocab_size: int, rank, world_size: int
60 - ) -> Sequence[int]:
60 + per_partition_vocab_size: int, rank: int) -> Sequence[int]:
61 index_f = rank * per_partition_vocab_size
62 index_l = index_f + per_partition_vocab_size
63 return index_f, index_l
64
65 @staticmethod
66 - def vocab_range_from_global_vocab_size(global_vocab_size: int, rank: int, world_size: int) -> Sequence[int]:
66 + def vocab_range_from_global_vocab_size(global_vocab_size: int, rank: int,
67 + world_size: int) -> Sequence[int]:
68 per_partition_vocab_size = divide(global_vocab_size, world_size)
69 return VocabUtility.vocab_range_from_per_partition_vocab_size(
69 - per_partition_vocab_size, rank, world_size
70 - )
70 + per_partition_vocab_size, rank)
vllm/model_executor/utils.py
-6
@@ -4,9 +4,6 @@ import random
4 import numpy as np
5 import torch
6
7 -from vllm.model_executor.parallel_utils.parallel_state import model_parallel_is_initialized
8 -from vllm.model_executor.parallel_utils.tensor_parallel import model_parallel_cuda_manual_seed
9 -
7
8 def set_random_seed(seed: int) -> None:
9 random.seed(seed)
@@ -14,6 +11,3 @@ def set_random_seed(seed: int) -> None:
11 torch.manual_seed(seed)
12 if torch.cuda.is_available():
13 torch.cuda.manual_seed_all(seed)
17 -
18 - if model_parallel_is_initialized():
19 - model_parallel_cuda_manual_seed(seed)