A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
[Core][Refactor] move parallel_utils into vllm/distributed (#3950)
[WIP][Core][Refactor] move vllm/model_executor/parallel_utils into vllm/distributed and vllm/device_communicators (#3950)
youkaichao committed
Apr 10, 2024 at 15:33 UTC
63e7176f265be43dcc425f5ab4ab45c90234f5c3
52 files changed
+111
-141
tests/conftest.py
+1
-2
@@ -11,8 +11,7 @@ from transformers import (AutoModelForCausalLM, AutoProcessor,
11
12
from vllm import LLM, SamplingParams
13
from vllm.config import TokenizerPoolConfig, VisionLanguageConfig
14
-from vllm.model_executor.parallel_utils.parallel_state import (
15
- destroy_model_parallel)
14
+from vllm.distributed import destroy_model_parallel
15
from vllm.sequence import MultiModalData
16
from vllm.transformers_utils.tokenizer import get_tokenizer
17
tests/distributed/test_comm_ops.py
+3
-3
@@ -8,9 +8,9 @@ import pytest
8
import ray
9
import torch
10
11
-from vllm.model_executor.parallel_utils.communication_op import (
12
- broadcast_tensor_dict, tensor_model_parallel_all_gather,
13
- tensor_model_parallel_all_reduce)
11
+from vllm.distributed import (broadcast_tensor_dict,
12
+ tensor_model_parallel_all_gather,
13
+ tensor_model_parallel_all_reduce)
14
from vllm.test_utils import (init_test_distributed_environment,
15
multi_process_tensor_parallel)
16
tests/distributed/test_custom_all_reduce.py
+6
-7
@@ -6,9 +6,8 @@ import ray
6
import torch
7
import torch.distributed as dist
8
9
-from vllm.model_executor.parallel_utils import custom_all_reduce as custom_ar
10
-from vllm.model_executor.parallel_utils.communication_op import (
11
- tensor_model_parallel_all_reduce)
9
+from vllm.distributed import tensor_model_parallel_all_reduce
10
+from vllm.distributed.device_communicators import custom_all_reduce
11
from vllm.test_utils import (init_test_distributed_environment,
12
multi_process_tensor_parallel)
13
@@ -26,10 +25,10 @@ def graph_allreduce(world_size, rank, distributed_init_port):
25
init_test_distributed_environment(1, world_size, rank,
26
distributed_init_port)
27
29
- custom_ar.init_custom_ar()
28
+ custom_all_reduce.init_custom_all_reduce()
29
for sz in test_sizes:
30
for dtype in [torch.float32, torch.float16, torch.bfloat16]:
32
- with custom_ar.capture():
31
+ with custom_all_reduce.capture():
32
# use integers so result matches NCCL exactly
33
inp1 = torch.randint(1,
34
16, (sz, ),
@@ -62,8 +61,8 @@ def eager_allreduce(world_size, rank, distributed_init_port):
61
distributed_init_port)
62
63
sz = 1024
65
- custom_ar.init_custom_ar()
66
- fa = custom_ar.get_handle()
64
+ custom_all_reduce.init_custom_all_reduce()
65
+ fa = custom_all_reduce.get_handle()
66
inp = torch.ones(sz, dtype=torch.float32, device=device)
67
out = fa.all_reduce_unreg(inp)
68
assert torch.allclose(out, inp * world_size)
tests/distributed/test_pynccl.py
+2
-2
@@ -4,8 +4,8 @@ import os
4
import pytest
5
import torch
6
7
-from vllm.model_executor.parallel_utils.pynccl import (NCCLCommunicator,
8
- ncclGetUniqueId)
7
+from vllm.distributed.device_communicators.pynccl import (NCCLCommunicator,
8
+ ncclGetUniqueId)
9
10
11
def distributed_run(fn, world_size):
tests/lora/conftest.py
+1
-2
@@ -12,6 +12,7 @@ from huggingface_hub import snapshot_download
12
13
import vllm
14
from vllm.config import LoRAConfig
15
+from vllm.distributed import destroy_model_parallel, initialize_model_parallel
16
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
17
MergedColumnParallelLinear,
18
RowParallelLinear)
@@ -19,8 +20,6 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
20
from vllm.model_executor.layers.sampler import Sampler
21
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
22
from vllm.model_executor.model_loader import get_model
22
-from vllm.model_executor.parallel_utils.parallel_state import (
23
- destroy_model_parallel, initialize_model_parallel)
23
24
25
def cleanup():
vllm/distributed/__init__.py
new
+3
@@ -0,0 +1,3 @@
1
+from .communication_op import *
2
+from .parallel_state import *
3
+from .utils import *
vllm/distributed/communication_op.py
renamed
+8
-6
@@ -4,12 +4,10 @@ from typing import Any, Dict, List, Optional, Union
4
import torch
5
from torch.distributed import ProcessGroup
6
7
-from vllm.model_executor.parallel_utils import pynccl_utils
8
-from vllm.model_executor.parallel_utils.custom_all_reduce import (
9
- custom_all_reduce)
10
-from vllm.model_executor.parallel_utils.parallel_state import (
11
- get_tensor_model_parallel_group, get_tensor_model_parallel_rank,
12
- get_tensor_model_parallel_world_size, is_pynccl_enabled_for_all_reduce)
7
+from .parallel_state import (get_tensor_model_parallel_group,
8
+ get_tensor_model_parallel_rank,
9
+ get_tensor_model_parallel_world_size,
10
+ is_pynccl_enabled_for_all_reduce)
11
12
13
def tensor_model_parallel_all_reduce(input_: torch.Tensor) -> torch.Tensor:
@@ -24,6 +22,10 @@ def tensor_model_parallel_all_reduce(input_: torch.Tensor) -> torch.Tensor:
22
TLDR: always assume this function modifies its input, but use the return
23
value as the output.
24
"""
25
+ from vllm.distributed.device_communicators import pynccl_utils
26
+ from vllm.distributed.device_communicators.custom_all_reduce import (
27
+ custom_all_reduce)
28
+
29
# Bypass the function if we are using only 1 GPU.
30
if get_tensor_model_parallel_world_size() == 1:
31
return input_
vllm/distributed/device_communicators/__init__.py
renamed
vllm/distributed/device_communicators/custom_all_reduce.py
renamed
+3
-2
@@ -5,8 +5,6 @@ import torch
5
import torch.distributed as dist
6
7
from vllm.logger import init_logger
8
-from vllm.model_executor.parallel_utils.parallel_state import (
9
- get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
8
9
try:
10
import pynvml
@@ -25,6 +23,9 @@ _SUPPORTED_WORLD_SIZES = [2, 4, 6, 8]
23
24
25
def init_custom_ar() -> None:
26
+ from vllm.distributed import (get_tensor_model_parallel_rank,
27
+ get_tensor_model_parallel_world_size)
28
+
29
global _CA_HANDLE
30
if _CA_HANDLE is not None:
31
return
vllm/distributed/device_communicators/pynccl.py
renamed
vllm/distributed/device_communicators/pynccl_utils.py
renamed
+2
-2
@@ -9,8 +9,8 @@ from vllm.logger import init_logger
9
logger = init_logger(__name__)
10
11
try:
12
- from vllm.model_executor.parallel_utils.pynccl import (NCCLCommunicator,
13
- ncclGetVersion)
12
+ from vllm.distributed.device_communicators.pynccl import (NCCLCommunicator,
13
+ ncclGetVersion)
14
except Exception as e:
15
# in non-NVIDIA environments, we can't import the nccl module
16
# e.g. when running on machines with AMD GPUs
vllm/distributed/parallel_state.py
renamed
+2
-2
@@ -8,8 +8,6 @@ from typing import Optional
8
9
import torch
10
11
-from vllm.model_executor.parallel_utils import pynccl_utils
12
-
11
# Tensor model parallel group that the current rank belongs to.
12
_TENSOR_MODEL_PARALLEL_GROUP = None
13
# Pipeline model parallel group that the current rank belongs to.
@@ -266,6 +264,7 @@ def destroy_model_parallel():
264
_PIPELINE_MODEL_PARALLEL_GROUP = None
265
global _PIPELINE_GLOBAL_RANKS
266
_PIPELINE_GLOBAL_RANKS = None
267
+ from vllm.distributed.device_communicators import pynccl_utils
268
269
# Destroy the pynccl states if any.
270
pynccl_utils.destroy_process_group()
@@ -279,6 +278,7 @@ _ENABLE_PYNCCL_FOR_ALL_REDUCE = False
278
279
@contextlib.contextmanager
280
def with_pynccl_for_all_reduce():
281
+ from vllm.distributed.device_communicators import pynccl_utils
282
"""use pynccl instead of torch.distributed for all reduce"""
283
tp_size = get_tensor_model_parallel_world_size()
284
if tp_size == 1:
vllm/distributed/utils.py
renamed
vllm/lora/layers.py
+6
-7
@@ -10,6 +10,12 @@ import torch.nn.functional as F
10
from transformers import PretrainedConfig
11
12
from vllm.config import LoRAConfig
13
+from vllm.distributed import (get_tensor_model_parallel_rank,
14
+ get_tensor_model_parallel_world_size,
15
+ split_tensor_along_last_dim,
16
+ tensor_model_parallel_all_gather,
17
+ tensor_model_parallel_all_reduce,
18
+ tensor_model_parallel_gather)
19
from vllm.lora.punica import add_lora, add_lora_slice, bgmv
20
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
21
MergedColumnParallelLinear,
@@ -18,13 +24,6 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
24
from vllm.model_executor.layers.logits_processor import LogitsProcessor
25
from vllm.model_executor.layers.vocab_parallel_embedding import (
26
ParallelLMHead, VocabParallelEmbedding)
21
-from vllm.model_executor.parallel_utils.communication_op import (
22
- tensor_model_parallel_all_gather, tensor_model_parallel_all_reduce,
23
- tensor_model_parallel_gather)
24
-from vllm.model_executor.parallel_utils.parallel_state import (
25
- get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
26
-from vllm.model_executor.parallel_utils.utils import (
27
- split_tensor_along_last_dim)
27
28
if TYPE_CHECKING:
29
pass
vllm/model_executor/layers/activation.py
+2
-3
@@ -7,10 +7,9 @@ import torch.nn as nn
7
import torch.nn.functional as F
8
9
from vllm._C import ops
10
+from vllm.distributed import (divide, get_tensor_model_parallel_rank,
11
+ get_tensor_model_parallel_world_size)
12
from vllm.model_executor.layers.quantization import QuantizationConfig
11
-from vllm.model_executor.parallel_utils.parallel_state import (
12
- get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
13
-from vllm.model_executor.parallel_utils.utils import divide
13
from vllm.model_executor.utils import set_weight_attrs
14
15
vllm/model_executor/layers/linear.py
+5
-6
@@ -5,13 +5,12 @@ import torch
5
import torch.nn.functional as F
6
from torch.nn.parameter import Parameter
7
8
+from vllm.distributed import (divide, get_tensor_model_parallel_rank,
9
+ get_tensor_model_parallel_world_size,
10
+ split_tensor_along_last_dim,
11
+ tensor_model_parallel_all_gather,
12
+ tensor_model_parallel_all_reduce)
13
from vllm.logger import init_logger
9
-from vllm.model_executor.parallel_utils.communication_op import (
10
- tensor_model_parallel_all_gather, tensor_model_parallel_all_reduce)
11
-from vllm.model_executor.parallel_utils.parallel_state import (
12
- get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
13
-from vllm.model_executor.parallel_utils.utils import (
14
- divide, split_tensor_along_last_dim)
14
from vllm.model_executor.utils import set_weight_attrs
15
16
logger = init_logger(__name__)
vllm/model_executor/layers/logits_processor.py
+1
-2
@@ -4,8 +4,7 @@ from typing import Optional
4
import torch
5
import torch.nn as nn
6
7
-from vllm.model_executor.parallel_utils.communication_op import (
8
- tensor_model_parallel_gather)
7
+from vllm.distributed import tensor_model_parallel_gather
8
from vllm.model_executor.sampling_metadata import SamplingMetadata
9
10
vllm/model_executor/layers/vocab_parallel_embedding.py
+3
-5
@@ -4,11 +4,9 @@ import torch
4
import torch.nn.functional as F
5
from torch.nn.parameter import Parameter
6
7
-from vllm.model_executor.parallel_utils.communication_op import (
8
- tensor_model_parallel_all_reduce)
9
-from vllm.model_executor.parallel_utils.parallel_state import (
10
- get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
11
-from vllm.model_executor.parallel_utils.utils import divide
7
+from vllm.distributed import (divide, get_tensor_model_parallel_rank,
8
+ get_tensor_model_parallel_world_size,
9
+ tensor_model_parallel_all_reduce)
10
from vllm.model_executor.utils import set_weight_attrs
11
12
DEFAULT_VOCAB_PADDING_SIZE = 64
vllm/model_executor/models/baichuan.py
+2
-2
@@ -27,6 +27,8 @@ from transformers import PretrainedConfig
27
28
from vllm.attention import Attention, AttentionMetadata
29
from vllm.config import LoRAConfig
30
+from vllm.distributed import (get_tensor_model_parallel_rank,
31
+ get_tensor_model_parallel_world_size)
32
from vllm.model_executor.layers.activation import SiluAndMul
33
from vllm.model_executor.layers.layernorm import RMSNorm
34
from vllm.model_executor.layers.linear import (LinearMethodBase,
@@ -38,8 +40,6 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
40
from vllm.model_executor.layers.sampler import Sampler
41
from vllm.model_executor.layers.vocab_parallel_embedding import (
42
ParallelLMHead, VocabParallelEmbedding)
41
-from vllm.model_executor.parallel_utils.parallel_state import (
42
- get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
43
from vllm.model_executor.sampling_metadata import SamplingMetadata
44
from vllm.model_executor.weight_utils import (default_weight_loader,
45
hf_model_weights_iterator)
vllm/model_executor/models/bloom.py
+2
-2
@@ -24,6 +24,8 @@ from torch import nn
24
from transformers import BloomConfig
25
26
from vllm.attention import Attention, AttentionMetadata
27
+from vllm.distributed import (get_tensor_model_parallel_rank,
28
+ get_tensor_model_parallel_world_size)
29
from vllm.model_executor.layers.activation import get_act_fn
30
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
31
LinearMethodBase,
@@ -33,8 +35,6 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
35
from vllm.model_executor.layers.sampler import Sampler
36
from vllm.model_executor.layers.vocab_parallel_embedding import (
37
VocabParallelEmbedding)
36
-from vllm.model_executor.parallel_utils.parallel_state import (
37
- get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
38
from vllm.model_executor.sampling_metadata import SamplingMetadata
39
from vllm.model_executor.weight_utils import (default_weight_loader,
40
hf_model_weights_iterator)
vllm/model_executor/models/chatglm.py
+1
-2
@@ -10,6 +10,7 @@ from torch.nn import LayerNorm
10
11
from vllm.attention import Attention, AttentionMetadata
12
from vllm.config import LoRAConfig
13
+from vllm.distributed import get_tensor_model_parallel_world_size
14
from vllm.model_executor.layers.activation import SiluAndMul
15
from vllm.model_executor.layers.layernorm import RMSNorm
16
from vllm.model_executor.layers.linear import (LinearMethodBase,
@@ -21,8 +22,6 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
22
from vllm.model_executor.layers.sampler import Sampler
23
from vllm.model_executor.layers.vocab_parallel_embedding import (
24
ParallelLMHead, VocabParallelEmbedding)
24
-from vllm.model_executor.parallel_utils.parallel_state import (
25
- get_tensor_model_parallel_world_size)
25
from vllm.model_executor.sampling_metadata import SamplingMetadata
26
from vllm.model_executor.weight_utils import (default_weight_loader,
27
hf_model_weights_iterator)
vllm/model_executor/models/commandr.py
+2
-2
@@ -29,6 +29,8 @@ from torch.nn.parameter import Parameter
29
from transformers import CohereConfig
30
31
from vllm.attention import Attention, AttentionMetadata
32
+from vllm.distributed import (get_tensor_model_parallel_rank,
33
+ get_tensor_model_parallel_world_size)
34
from vllm.model_executor.layers.activation import SiluAndMul
35
from vllm.model_executor.layers.linear import (LinearMethodBase,
36
MergedColumnParallelLinear,
@@ -39,8 +41,6 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
41
from vllm.model_executor.layers.sampler import Sampler
42
from vllm.model_executor.layers.vocab_parallel_embedding import (
43
VocabParallelEmbedding)
42
-from vllm.model_executor.parallel_utils.parallel_state import (
43
- get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
44
from vllm.model_executor.sampling_metadata import SamplingMetadata
45
from vllm.model_executor.utils import set_weight_attrs
46
from vllm.model_executor.weight_utils import (default_weight_loader,
vllm/model_executor/models/dbrx.py
+3
-4
@@ -5,6 +5,9 @@ import torch
5
import torch.nn as nn
6
7
from vllm.attention import Attention, AttentionMetadata
8
+from vllm.distributed import (get_tensor_model_parallel_rank,
9
+ get_tensor_model_parallel_world_size,
10
+ tensor_model_parallel_all_reduce)
11
from vllm.model_executor.layers.fused_moe import fused_moe
12
from vllm.model_executor.layers.linear import (LinearMethodBase,
13
QKVParallelLinear,
@@ -15,10 +18,6 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
18
from vllm.model_executor.layers.sampler import Sampler
19
from vllm.model_executor.layers.vocab_parallel_embedding import (
20
DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
18
-from vllm.model_executor.parallel_utils.communication_op import (
19
- tensor_model_parallel_all_reduce)
20
-from vllm.model_executor.parallel_utils.parallel_state import (
21
- get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
21
from vllm.model_executor.sampling_metadata import SamplingMetadata
22
from vllm.model_executor.utils import set_weight_attrs
23
from vllm.model_executor.weight_utils import (default_weight_loader,
vllm/model_executor/models/deepseek.py
+3
-4
@@ -28,6 +28,9 @@ from torch import nn
28
from transformers import PretrainedConfig
29
30
from vllm.attention import Attention, AttentionMetadata
31
+from vllm.distributed import (get_tensor_model_parallel_rank,
32
+ get_tensor_model_parallel_world_size,
33
+ tensor_model_parallel_all_reduce)
34
from vllm.model_executor.layers.activation import SiluAndMul
35
from vllm.model_executor.layers.fused_moe import fused_moe
36
from vllm.model_executor.layers.layernorm import RMSNorm
@@ -41,10 +44,6 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
44
from vllm.model_executor.layers.sampler import Sampler
45
from vllm.model_executor.layers.vocab_parallel_embedding import (
46
ParallelLMHead, VocabParallelEmbedding)
44
-from vllm.model_executor.parallel_utils.communication_op import (
45
- tensor_model_parallel_all_reduce)
46
-from vllm.model_executor.parallel_utils.parallel_state import (
47
- get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
47
from vllm.model_executor.sampling_metadata import SamplingMetadata
48
from vllm.model_executor.weight_utils import (default_weight_loader,
49
hf_model_weights_iterator)
vllm/model_executor/models/falcon.py
+3
-4
@@ -27,6 +27,9 @@ from torch.nn import LayerNorm
27
from transformers import FalconConfig as HF_FalconConfig
28
29
from vllm.attention import Attention, AttentionMetadata
30
+from vllm.distributed import (get_tensor_model_parallel_rank,
31
+ get_tensor_model_parallel_world_size,
32
+ tensor_model_parallel_all_reduce)
33
from vllm.model_executor.layers.activation import get_act_fn
34
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
35
LinearMethodBase,
@@ -37,10 +40,6 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
40
from vllm.model_executor.layers.sampler import Sampler
41
from vllm.model_executor.layers.vocab_parallel_embedding import (
42
VocabParallelEmbedding)
40
-from vllm.model_executor.parallel_utils.communication_op import (
41
- tensor_model_parallel_all_reduce)
42
-from vllm.model_executor.parallel_utils.parallel_state import (
43
- get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
43
from vllm.model_executor.sampling_metadata import SamplingMetadata
44
from vllm.model_executor.weight_utils import (default_weight_loader,
45
hf_model_weights_iterator)
vllm/model_executor/models/gemma.py
+1
-2
@@ -23,6 +23,7 @@ from transformers import GemmaConfig
23
24
from vllm.attention import Attention, AttentionMetadata
25
from vllm.config import LoRAConfig
26
+from vllm.distributed import get_tensor_model_parallel_world_size
27
from vllm.logger import init_logger
28
from vllm.model_executor.layers.activation import GeluAndMul
29
from vllm.model_executor.layers.layernorm import RMSNorm
@@ -35,8 +36,6 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
36
from vllm.model_executor.layers.sampler import Sampler
37
from vllm.model_executor.layers.vocab_parallel_embedding import (
38
VocabParallelEmbedding)
38
-from vllm.model_executor.parallel_utils.parallel_state import (
39
- get_tensor_model_parallel_world_size)
39
from vllm.model_executor.sampling_metadata import SamplingMetadata
40
from vllm.model_executor.weight_utils import (default_weight_loader,
41
hf_model_weights_iterator)
vllm/model_executor/models/gpt2.py
+1
-2
@@ -24,6 +24,7 @@ from torch import nn
24
from transformers import GPT2Config
25
26
from vllm.attention import Attention, AttentionMetadata
27
+from vllm.distributed import get_tensor_model_parallel_world_size
28
from vllm.model_executor.layers.activation import get_act_fn
29
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
30
LinearMethodBase,
@@ -33,8 +34,6 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
34
from vllm.model_executor.layers.sampler import Sampler
35
from vllm.model_executor.layers.vocab_parallel_embedding import (
36
VocabParallelEmbedding)
36
-from vllm.model_executor.parallel_utils.parallel_state import (
37
- get_tensor_model_parallel_world_size)
37
from vllm.model_executor.sampling_metadata import SamplingMetadata
38
from vllm.model_executor.weight_utils import (default_weight_loader,
39
hf_model_weights_iterator)
vllm/model_executor/models/gpt_bigcode.py
+1
-2
@@ -25,6 +25,7 @@ from torch import nn
25
from transformers import GPTBigCodeConfig
26
27
from vllm.attention import Attention, AttentionMetadata
28
+from vllm.distributed import get_tensor_model_parallel_world_size
29
from vllm.model_executor.layers.activation import get_act_fn
30
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
31
LinearMethodBase,
@@ -34,8 +35,6 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
35
from vllm.model_executor.layers.sampler import Sampler
36
from vllm.model_executor.layers.vocab_parallel_embedding import (
37
VocabParallelEmbedding)
37
-from vllm.model_executor.parallel_utils.parallel_state import (
38
- get_tensor_model_parallel_world_size)
38
from vllm.model_executor.sampling_metadata import SamplingMetadata
39
from vllm.model_executor.weight_utils import (default_weight_loader,
40
hf_model_weights_iterator)
vllm/model_executor/models/gpt_j.py
+1
-2
@@ -23,6 +23,7 @@ from torch import nn
23
from transformers import GPTJConfig
24
25
from vllm.attention import Attention, AttentionMetadata
26
+from vllm.distributed import get_tensor_model_parallel_world_size
27
from vllm.model_executor.layers.activation import get_act_fn
28
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
29
LinearMethodBase,
@@ -33,8 +34,6 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
34
from vllm.model_executor.layers.sampler import Sampler
35
from vllm.model_executor.layers.vocab_parallel_embedding import (
36
ParallelLMHead, VocabParallelEmbedding)
36
-from vllm.model_executor.parallel_utils.parallel_state import (
37
- get_tensor_model_parallel_world_size)
37
from vllm.model_executor.sampling_metadata import SamplingMetadata
38
from vllm.model_executor.weight_utils import (default_weight_loader,
39
hf_model_weights_iterator)
vllm/model_executor/models/gpt_neox.py
+1
-2
@@ -23,6 +23,7 @@ from torch import nn
23
from transformers import GPTNeoXConfig
24
25
from vllm.attention import Attention, AttentionMetadata
26
+from vllm.distributed import get_tensor_model_parallel_world_size
27
from vllm.model_executor.layers.activation import get_act_fn
28
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
29
LinearMethodBase,
@@ -33,8 +34,6 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
34
from vllm.model_executor.layers.sampler import Sampler
35
from vllm.model_executor.layers.vocab_parallel_embedding import (
36
ParallelLMHead, VocabParallelEmbedding)
36
-from vllm.model_executor.parallel_utils.parallel_state import (
37
- get_tensor_model_parallel_world_size)
37
from vllm.model_executor.sampling_metadata import SamplingMetadata
38
from vllm.model_executor.weight_utils import (default_weight_loader,
39
hf_model_weights_iterator)
vllm/model_executor/models/internlm2.py
+1
-2
@@ -6,6 +6,7 @@ from torch import nn
6
from transformers import PretrainedConfig
7
8
from vllm.attention import Attention, AttentionMetadata
9
+from vllm.distributed import get_tensor_model_parallel_world_size
10
from vllm.model_executor.layers.activation import SiluAndMul
11
from vllm.model_executor.layers.layernorm import RMSNorm
12
from vllm.model_executor.layers.linear import (LinearMethodBase,
@@ -17,8 +18,6 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
18
from vllm.model_executor.layers.sampler import Sampler
19
from vllm.model_executor.layers.vocab_parallel_embedding import (
20
ParallelLMHead, VocabParallelEmbedding)
20
-from vllm.model_executor.parallel_utils.parallel_state import (
21
- get_tensor_model_parallel_world_size)
21
from vllm.model_executor.sampling_metadata import SamplingMetadata
22
from vllm.model_executor.weight_utils import (default_weight_loader,
23
hf_model_weights_iterator)
vllm/model_executor/models/jais.py
+2
-2
@@ -26,6 +26,8 @@ import torch
26
from torch import nn
27
28
from vllm.attention import Attention, AttentionMetadata
29
+from vllm.distributed import (get_tensor_model_parallel_rank,
30
+ get_tensor_model_parallel_world_size)
31
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
32
LinearMethodBase,
33
QKVParallelLinear,
@@ -34,8 +36,6 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
36
from vllm.model_executor.layers.sampler import Sampler
37
from vllm.model_executor.layers.vocab_parallel_embedding import (
38
VocabParallelEmbedding)
37
-from vllm.model_executor.parallel_utils.parallel_state import (
38
- get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
39
from vllm.model_executor.sampling_metadata import SamplingMetadata
40
from vllm.model_executor.weight_utils import (default_weight_loader,
41
hf_model_weights_iterator)
vllm/model_executor/models/llama.py
+2
-2
@@ -29,6 +29,8 @@ from transformers import LlamaConfig
29
30
from vllm.attention import Attention, AttentionMetadata
31
from vllm.config import LoRAConfig
32
+from vllm.distributed import (get_tensor_model_parallel_rank,
33
+ get_tensor_model_parallel_world_size)
34
from vllm.model_executor.layers.activation import SiluAndMul
35
from vllm.model_executor.layers.layernorm import RMSNorm
36
from vllm.model_executor.layers.linear import (LinearMethodBase,
@@ -40,8 +42,6 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
42
from vllm.model_executor.layers.sampler import Sampler
43
from vllm.model_executor.layers.vocab_parallel_embedding import (
44
DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
43
-from vllm.model_executor.parallel_utils.parallel_state import (
44
- get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
45
from vllm.model_executor.sampling_metadata import SamplingMetadata
46
from vllm.model_executor.weight_utils import (default_weight_loader,
47
hf_model_weights_iterator,
vllm/model_executor/models/minicpm.py
+3
-4
@@ -29,6 +29,9 @@ from torch import nn
29
30
from vllm.attention import Attention, AttentionMetadata
31
from vllm.config import LoRAConfig
32
+from vllm.distributed import (get_tensor_model_parallel_rank,
33
+ get_tensor_model_parallel_world_size,
34
+ tensor_model_parallel_all_reduce)
35
from vllm.model_executor.layers.activation import SiluAndMul
36
from vllm.model_executor.layers.fused_moe import fused_moe
37
from vllm.model_executor.layers.layernorm import RMSNorm
@@ -42,10 +45,6 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
45
from vllm.model_executor.layers.sampler import Sampler
46
from vllm.model_executor.layers.vocab_parallel_embedding import (
47
DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
45
-from vllm.model_executor.parallel_utils.communication_op import (
46
- tensor_model_parallel_all_reduce)
47
-from vllm.model_executor.parallel_utils.parallel_state import (
48
- get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
48
from vllm.model_executor.sampling_metadata import SamplingMetadata
49
from vllm.model_executor.utils import set_weight_attrs
50
from vllm.model_executor.weight_utils import (default_weight_loader,
vllm/model_executor/models/mixtral.py
+3
-4
@@ -29,6 +29,9 @@ from transformers import MixtralConfig
29
30
from vllm.attention import Attention, AttentionMetadata
31
from vllm.config import LoRAConfig
32
+from vllm.distributed import (get_tensor_model_parallel_rank,
33
+ get_tensor_model_parallel_world_size,
34
+ tensor_model_parallel_all_reduce)
35
from vllm.model_executor.layers.fused_moe import fused_moe
36
from vllm.model_executor.layers.layernorm import RMSNorm
37
from vllm.model_executor.layers.linear import (LinearMethodBase,
@@ -40,10 +43,6 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
43
from vllm.model_executor.layers.sampler import Sampler
44
from vllm.model_executor.layers.vocab_parallel_embedding import (
45
DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
43
-from vllm.model_executor.parallel_utils.communication_op import (
44
- tensor_model_parallel_all_reduce)
45
-from vllm.model_executor.parallel_utils.parallel_state import (
46
- get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
46
from vllm.model_executor.sampling_metadata import SamplingMetadata
47
from vllm.model_executor.utils import set_weight_attrs
48
from vllm.model_executor.weight_utils import (default_weight_loader,
vllm/model_executor/models/mixtral_quant.py
+3
-4
@@ -30,6 +30,9 @@ from torch import nn
30
from transformers import MixtralConfig
31
32
from vllm.attention import Attention, AttentionMetadata
33
+from vllm.distributed import (get_tensor_model_parallel_rank,
34
+ get_tensor_model_parallel_world_size,
35
+ tensor_model_parallel_all_reduce)
36
from vllm.model_executor.layers.layernorm import RMSNorm
37
from vllm.model_executor.layers.linear import (LinearMethodBase,
38
QKVParallelLinear,
@@ -40,10 +43,6 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
43
from vllm.model_executor.layers.sampler import Sampler
44
from vllm.model_executor.layers.vocab_parallel_embedding import (
45
ParallelLMHead, VocabParallelEmbedding)
43
-from vllm.model_executor.parallel_utils.communication_op import (
44
- tensor_model_parallel_all_reduce)
45
-from vllm.model_executor.parallel_utils.parallel_state import (
46
- get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
46
from vllm.model_executor.sampling_metadata import SamplingMetadata
47
from vllm.model_executor.weight_utils import (default_weight_loader,
48
hf_model_weights_iterator)
vllm/model_executor/models/mpt.py
+2
-2
@@ -7,6 +7,8 @@ import torch
7
import torch.nn as nn
8
9
from vllm.attention import Attention, AttentionMetadata
10
+from vllm.distributed import (get_tensor_model_parallel_rank,
11
+ get_tensor_model_parallel_world_size)
12
from vllm.model_executor.layers.activation import get_act_fn
13
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
14
LinearMethodBase,
@@ -16,8 +18,6 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
18
from vllm.model_executor.layers.sampler import Sampler
19
from vllm.model_executor.layers.vocab_parallel_embedding import (
20
VocabParallelEmbedding)
19
-from vllm.model_executor.parallel_utils.parallel_state import (
20
- get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
21
from vllm.model_executor.sampling_metadata import SamplingMetadata
22
from vllm.model_executor.weight_utils import (default_weight_loader,
23
hf_model_weights_iterator)
vllm/model_executor/models/olmo.py
+1
-2
@@ -44,6 +44,7 @@ from hf_olmo import OLMoConfig
44
from torch import nn
45
46
from vllm.attention import Attention, AttentionMetadata
47
+from vllm.distributed import get_tensor_model_parallel_world_size
48
from vllm.model_executor.layers.activation import SiluAndMul
49
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
50
LinearMethodBase,
@@ -55,8 +56,6 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
56
from vllm.model_executor.layers.sampler import Sampler
57
from vllm.model_executor.layers.vocab_parallel_embedding import (
58
VocabParallelEmbedding)
58
-from vllm.model_executor.parallel_utils.parallel_state import (
59
- get_tensor_model_parallel_world_size)
59
from vllm.model_executor.sampling_metadata import SamplingMetadata
60
from vllm.model_executor.weight_utils import (default_weight_loader,
61
hf_model_weights_iterator)
vllm/model_executor/models/opt.py
+1
-2
@@ -24,6 +24,7 @@ from torch import nn
24
from transformers import OPTConfig
25
26
from vllm.attention import Attention, AttentionMetadata
27
+from vllm.distributed import get_tensor_model_parallel_world_size
28
from vllm.model_executor.layers.activation import get_act_fn
29
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
30
LinearMethodBase,
@@ -34,8 +35,6 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
35
from vllm.model_executor.layers.sampler import Sampler
36
from vllm.model_executor.layers.vocab_parallel_embedding import (
37
VocabParallelEmbedding)
37
-from vllm.model_executor.parallel_utils.parallel_state import (
38
- get_tensor_model_parallel_world_size)
38
from vllm.model_executor.sampling_metadata import SamplingMetadata
39
from vllm.model_executor.weight_utils import (default_weight_loader,
40
hf_model_weights_iterator)
vllm/model_executor/models/orion.py
+1
-2
@@ -11,6 +11,7 @@ from torch import nn
11
from transformers import PretrainedConfig
12
13
from vllm.attention import Attention, AttentionMetadata
14
+from vllm.distributed import get_tensor_model_parallel_world_size
15
from vllm.model_executor.layers.activation import SiluAndMul
16
from vllm.model_executor.layers.linear import (LinearMethodBase,
17
MergedColumnParallelLinear,
@@ -21,8 +22,6 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
22
from vllm.model_executor.layers.sampler import Sampler
23
from vllm.model_executor.layers.vocab_parallel_embedding import (
24
ParallelLMHead, VocabParallelEmbedding)
24
-from vllm.model_executor.parallel_utils.parallel_state import (
25
- get_tensor_model_parallel_world_size)
25
from vllm.model_executor.sampling_metadata import SamplingMetadata
26
from vllm.model_executor.weight_utils import (default_weight_loader,
27
hf_model_weights_iterator)
vllm/model_executor/models/phi.py
+1
-2
@@ -42,6 +42,7 @@ from torch import nn
42
from transformers import PretrainedConfig
43
44
from vllm.attention import Attention, AttentionMetadata
45
+from vllm.distributed import get_tensor_model_parallel_world_size
46
from vllm.model_executor.layers.activation import get_act_fn
47
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
48
LinearMethodBase,
@@ -52,8 +53,6 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
53
from vllm.model_executor.layers.sampler import Sampler
54
from vllm.model_executor.layers.vocab_parallel_embedding import (
55
ParallelLMHead, VocabParallelEmbedding)
55
-from vllm.model_executor.parallel_utils.parallel_state import (
56
- get_tensor_model_parallel_world_size)
56
from vllm.model_executor.sampling_metadata import SamplingMetadata
57
from vllm.model_executor.weight_utils import (default_weight_loader,
58
hf_model_weights_iterator)
vllm/model_executor/models/qwen.py
+1
-2
@@ -11,6 +11,7 @@ from torch import nn
11
from transformers import PretrainedConfig
12
13
from vllm.attention import Attention, AttentionMetadata
14
+from vllm.distributed import get_tensor_model_parallel_world_size
15
from vllm.model_executor.layers.activation import SiluAndMul
16
from vllm.model_executor.layers.layernorm import RMSNorm
17
from vllm.model_executor.layers.linear import (LinearMethodBase,
@@ -22,8 +23,6 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
23
from vllm.model_executor.layers.sampler import Sampler
24
from vllm.model_executor.layers.vocab_parallel_embedding import (
25
ParallelLMHead, VocabParallelEmbedding)
25
-from vllm.model_executor.parallel_utils.parallel_state import (
26
- get_tensor_model_parallel_world_size)
26
from vllm.model_executor.sampling_metadata import SamplingMetadata
27
from vllm.model_executor.weight_utils import (default_weight_loader,
28
hf_model_weights_iterator)
vllm/model_executor/models/qwen2.py
+1
-2
@@ -30,6 +30,7 @@ from transformers import Qwen2Config
30
31
from vllm.attention import Attention, AttentionMetadata
32
from vllm.config import LoRAConfig
33
+from vllm.distributed import get_tensor_model_parallel_world_size
34
from vllm.model_executor.layers.activation import SiluAndMul
35
from vllm.model_executor.layers.layernorm import RMSNorm
36
from vllm.model_executor.layers.linear import (LinearMethodBase,
@@ -41,8 +42,6 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
42
from vllm.model_executor.layers.sampler import Sampler
43
from vllm.model_executor.layers.vocab_parallel_embedding import (
44
ParallelLMHead, VocabParallelEmbedding)
44
-from vllm.model_executor.parallel_utils.parallel_state import (
45
- get_tensor_model_parallel_world_size)
45
from vllm.model_executor.sampling_metadata import SamplingMetadata
46
from vllm.model_executor.weight_utils import (default_weight_loader,
47
hf_model_weights_iterator)
vllm/model_executor/models/qwen2_moe.py
+3
-4
@@ -30,6 +30,9 @@ from torch import nn
30
from transformers import PretrainedConfig
31
32
from vllm.attention import Attention, AttentionMetadata
33
+from vllm.distributed import (get_tensor_model_parallel_rank,
34
+ get_tensor_model_parallel_world_size,
35
+ tensor_model_parallel_all_reduce)
36
from vllm.model_executor.layers.activation import SiluAndMul
37
from vllm.model_executor.layers.fused_moe import fused_moe
38
from vllm.model_executor.layers.layernorm import RMSNorm
@@ -43,10 +46,6 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
46
from vllm.model_executor.layers.sampler import Sampler
47
from vllm.model_executor.layers.vocab_parallel_embedding import (
48
ParallelLMHead, VocabParallelEmbedding)
46
-from vllm.model_executor.parallel_utils.communication_op import (
47
- tensor_model_parallel_all_reduce)
48
-from vllm.model_executor.parallel_utils.parallel_state import (
49
- get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
49
from vllm.model_executor.sampling_metadata import SamplingMetadata
50
from vllm.model_executor.weight_utils import (default_weight_loader,
51
hf_model_weights_iterator)
vllm/model_executor/models/stablelm.py
+1
-2
@@ -26,6 +26,7 @@ from torch import nn
26
from transformers import PretrainedConfig
27
28
from vllm.attention import Attention, AttentionMetadata
29
+from vllm.distributed import get_tensor_model_parallel_world_size
30
from vllm.model_executor.layers.activation import SiluAndMul
31
from vllm.model_executor.layers.linear import (LinearMethodBase,
32
MergedColumnParallelLinear,
@@ -36,8 +37,6 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
37
from vllm.model_executor.layers.sampler import Sampler
38
from vllm.model_executor.layers.vocab_parallel_embedding import (
39
ParallelLMHead, VocabParallelEmbedding)
39
-from vllm.model_executor.parallel_utils.parallel_state import (
40
- get_tensor_model_parallel_world_size)
40
from vllm.model_executor.sampling_metadata import SamplingMetadata
41
from vllm.model_executor.weight_utils import (default_weight_loader,
42
hf_model_weights_iterator)
vllm/model_executor/models/starcoder2.py
+1
-2
@@ -25,6 +25,7 @@ from torch import nn
25
from transformers import Starcoder2Config
26
27
from vllm.attention import Attention, AttentionMetadata
28
+from vllm.distributed import get_tensor_model_parallel_world_size
29
from vllm.model_executor.layers.activation import get_act_fn
30
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
31
LinearMethodBase,
@@ -35,8 +36,6 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
36
from vllm.model_executor.layers.sampler import Sampler
37
from vllm.model_executor.layers.vocab_parallel_embedding import (
38
DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
38
-from vllm.model_executor.parallel_utils.parallel_state import (
39
- get_tensor_model_parallel_world_size)
39
from vllm.model_executor.sampling_metadata import SamplingMetadata
40
from vllm.model_executor.weight_utils import (default_weight_loader,
41
hf_model_weights_iterator)
vllm/model_executor/models/xverse.py
+1
-2
@@ -28,6 +28,7 @@ from transformers import PretrainedConfig
28
29
from vllm.attention import Attention, AttentionMetadata
30
from vllm.config import LoRAConfig
31
+from vllm.distributed import get_tensor_model_parallel_world_size
32
from vllm.model_executor.layers.activation import SiluAndMul
33
from vllm.model_executor.layers.layernorm import RMSNorm
34
from vllm.model_executor.layers.linear import (LinearMethodBase,
@@ -39,8 +40,6 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
40
from vllm.model_executor.layers.sampler import Sampler
41
from vllm.model_executor.layers.vocab_parallel_embedding import (
42
ParallelLMHead, VocabParallelEmbedding)
42
-from vllm.model_executor.parallel_utils.parallel_state import (
43
- get_tensor_model_parallel_world_size)
43
from vllm.model_executor.sampling_metadata import SamplingMetadata
44
from vllm.model_executor.weight_utils import (default_weight_loader,
45
hf_model_weights_iterator)
vllm/model_executor/parallel_utils/README.md
deleted
-1
@@ -1 +0,0 @@
1
-The files in this folder are ported from [Megatron-LM](https://github.com/NVIDIA/Megatron-LM/tree/main/megatron/core). We only keep the codes that are used in inference.
\ No newline at end of file
vllm/test_utils.py
+2
-2
@@ -1,7 +1,7 @@
1
import ray
2
3
-from vllm.model_executor.parallel_utils.parallel_state import (
4
- ensure_model_parallel_initialized, init_distributed_environment)
3
+from vllm.distributed import (ensure_model_parallel_initialized,
4
+ init_distributed_environment)
5
from vllm.utils import get_open_port
6
7
vllm/worker/cpu_worker.py
+3
-4
@@ -7,13 +7,12 @@ import torch.distributed
7
from vllm.attention import get_attn_backend
8
from vllm.config import (CacheConfig, DeviceConfig, LoRAConfig, ModelConfig,
9
ParallelConfig, SchedulerConfig)
10
+from vllm.distributed import (broadcast_tensor_dict,
11
+ ensure_model_parallel_initialized,
12
+ init_distributed_environment)
13
from vllm.logger import init_logger
14
from vllm.model_executor import set_random_seed
15
from vllm.model_executor.model_loader import get_model
13
-from vllm.model_executor.parallel_utils.communication_op import (
14
- broadcast_tensor_dict)
15
-from vllm.model_executor.parallel_utils.parallel_state import (
16
- ensure_model_parallel_initialized, init_distributed_environment)
16
from vllm.sequence import SamplerOutput, SequenceGroupMetadata
17
from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE
18
from vllm.worker.model_runner import ModelRunner
vllm/worker/model_runner.py
+3
-5
@@ -9,17 +9,15 @@ import torch.nn as nn
9
from vllm.attention import AttentionMetadata, get_attn_backend
10
from vllm.config import (DeviceConfig, LoRAConfig, ModelConfig, ParallelConfig,
11
SchedulerConfig, VisionLanguageConfig)
12
+from vllm.distributed import broadcast_tensor_dict, with_pynccl_for_all_reduce
13
+from vllm.distributed.device_communicators import (custom_all_reduce,
14
+ pynccl_utils)
15
from vllm.logger import init_logger
16
from vllm.lora.layers import LoRAMapping
17
from vllm.lora.request import LoRARequest
18
from vllm.lora.worker_manager import LRUCacheWorkerLoRAManager
19
from vllm.model_executor import SamplingMetadata
20
from vllm.model_executor.model_loader import get_model
18
-from vllm.model_executor.parallel_utils import custom_all_reduce, pynccl_utils
19
-from vllm.model_executor.parallel_utils.communication_op import (
20
- broadcast_tensor_dict)
21
-from vllm.model_executor.parallel_utils.parallel_state import (
22
- with_pynccl_for_all_reduce)
21
from vllm.sampling_params import SamplingParams, SamplingType
22
from vllm.sequence import (MultiModalData, SamplerOutput, SequenceData,
23
SequenceGroupMetadata)
vllm/worker/worker.py
+6
-6
@@ -8,14 +8,14 @@ import torch.distributed
8
9
from vllm.config import (CacheConfig, DeviceConfig, LoRAConfig, ModelConfig,
10
ParallelConfig, SchedulerConfig, VisionLanguageConfig)
11
+from vllm.distributed import (broadcast_tensor_dict,
12
+ ensure_model_parallel_initialized,
13
+ init_distributed_environment)
14
+from vllm.distributed.device_communicators import pynccl_utils
15
+from vllm.distributed.device_communicators.custom_all_reduce import (
16
+ init_custom_ar)
17
from vllm.lora.request import LoRARequest
18
from vllm.model_executor import set_random_seed
13
-from vllm.model_executor.parallel_utils import pynccl_utils
14
-from vllm.model_executor.parallel_utils.communication_op import (
15
- broadcast_tensor_dict)
16
-from vllm.model_executor.parallel_utils.custom_all_reduce import init_custom_ar
17
-from vllm.model_executor.parallel_utils.parallel_state import (
18
- ensure_model_parallel_initialized, init_distributed_environment)
19
from vllm.sequence import SamplerOutput, SequenceGroupMetadata
20
from vllm.worker.cache_engine import CacheEngine
21
from vllm.worker.model_runner import ModelRunner