@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 63e7176f26
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 03:00 UTC

[Core][Refactor] move parallel_utils into vllm/distributed (#3950)

[WIP][Core][Refactor] move vllm/model_executor/parallel_utils into vllm/distributed and vllm/device_communicators (#3950)

youkaichao committed Apr 10, 2024 at 15:33 UTC 63e7176f265be43dcc425f5ab4ab45c90234f5c3
52 files changed +111 -141
tests/conftest.py
+1 -2
@@ -11,8 +11,7 @@ from transformers import (AutoModelForCausalLM, AutoProcessor,
11
12 from vllm import LLM, SamplingParams
13 from vllm.config import TokenizerPoolConfig, VisionLanguageConfig
14 -from vllm.model_executor.parallel_utils.parallel_state import (
15 - destroy_model_parallel)
14 +from vllm.distributed import destroy_model_parallel
15 from vllm.sequence import MultiModalData
16 from vllm.transformers_utils.tokenizer import get_tokenizer
17
tests/distributed/test_comm_ops.py
+3 -3
@@ -8,9 +8,9 @@ import pytest
8 import ray
9 import torch
10
11 -from vllm.model_executor.parallel_utils.communication_op import (
12 - broadcast_tensor_dict, tensor_model_parallel_all_gather,
13 - tensor_model_parallel_all_reduce)
11 +from vllm.distributed import (broadcast_tensor_dict,
12 + tensor_model_parallel_all_gather,
13 + tensor_model_parallel_all_reduce)
14 from vllm.test_utils import (init_test_distributed_environment,
15 multi_process_tensor_parallel)
16
tests/distributed/test_custom_all_reduce.py
+6 -7
@@ -6,9 +6,8 @@ import ray
6 import torch
7 import torch.distributed as dist
8
9 -from vllm.model_executor.parallel_utils import custom_all_reduce as custom_ar
10 -from vllm.model_executor.parallel_utils.communication_op import (
11 - tensor_model_parallel_all_reduce)
9 +from vllm.distributed import tensor_model_parallel_all_reduce
10 +from vllm.distributed.device_communicators import custom_all_reduce
11 from vllm.test_utils import (init_test_distributed_environment,
12 multi_process_tensor_parallel)
13
@@ -26,10 +25,10 @@ def graph_allreduce(world_size, rank, distributed_init_port):
25 init_test_distributed_environment(1, world_size, rank,
26 distributed_init_port)
27
29 - custom_ar.init_custom_ar()
28 + custom_all_reduce.init_custom_all_reduce()
29 for sz in test_sizes:
30 for dtype in [torch.float32, torch.float16, torch.bfloat16]:
32 - with custom_ar.capture():
31 + with custom_all_reduce.capture():
32 # use integers so result matches NCCL exactly
33 inp1 = torch.randint(1,
34 16, (sz, ),
@@ -62,8 +61,8 @@ def eager_allreduce(world_size, rank, distributed_init_port):
61 distributed_init_port)
62
63 sz = 1024
65 - custom_ar.init_custom_ar()
66 - fa = custom_ar.get_handle()
64 + custom_all_reduce.init_custom_all_reduce()
65 + fa = custom_all_reduce.get_handle()
66 inp = torch.ones(sz, dtype=torch.float32, device=device)
67 out = fa.all_reduce_unreg(inp)
68 assert torch.allclose(out, inp * world_size)
tests/distributed/test_pynccl.py
+2 -2
@@ -4,8 +4,8 @@ import os
4 import pytest
5 import torch
6
7 -from vllm.model_executor.parallel_utils.pynccl import (NCCLCommunicator,
8 - ncclGetUniqueId)
7 +from vllm.distributed.device_communicators.pynccl import (NCCLCommunicator,
8 + ncclGetUniqueId)
9
10
11 def distributed_run(fn, world_size):
tests/lora/conftest.py
+1 -2
@@ -12,6 +12,7 @@ from huggingface_hub import snapshot_download
12
13 import vllm
14 from vllm.config import LoRAConfig
15 +from vllm.distributed import destroy_model_parallel, initialize_model_parallel
16 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
17 MergedColumnParallelLinear,
18 RowParallelLinear)
@@ -19,8 +20,6 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
20 from vllm.model_executor.layers.sampler import Sampler
21 from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
22 from vllm.model_executor.model_loader import get_model
22 -from vllm.model_executor.parallel_utils.parallel_state import (
23 - destroy_model_parallel, initialize_model_parallel)
23
24
25 def cleanup():
vllm/distributed/__init__.py new
+3
@@ -0,0 +1,3 @@
1 +from .communication_op import *
2 +from .parallel_state import *
3 +from .utils import *
vllm/distributed/communication_op.py renamed
+8 -6
@@ -4,12 +4,10 @@ from typing import Any, Dict, List, Optional, Union
4 import torch
5 from torch.distributed import ProcessGroup
6
7 -from vllm.model_executor.parallel_utils import pynccl_utils
8 -from vllm.model_executor.parallel_utils.custom_all_reduce import (
9 - custom_all_reduce)
10 -from vllm.model_executor.parallel_utils.parallel_state import (
11 - get_tensor_model_parallel_group, get_tensor_model_parallel_rank,
12 - get_tensor_model_parallel_world_size, is_pynccl_enabled_for_all_reduce)
7 +from .parallel_state import (get_tensor_model_parallel_group,
8 + get_tensor_model_parallel_rank,
9 + get_tensor_model_parallel_world_size,
10 + is_pynccl_enabled_for_all_reduce)
11
12
13 def tensor_model_parallel_all_reduce(input_: torch.Tensor) -> torch.Tensor:
@@ -24,6 +22,10 @@ def tensor_model_parallel_all_reduce(input_: torch.Tensor) -> torch.Tensor:
22 TLDR: always assume this function modifies its input, but use the return
23 value as the output.
24 """
25 + from vllm.distributed.device_communicators import pynccl_utils
26 + from vllm.distributed.device_communicators.custom_all_reduce import (
27 + custom_all_reduce)
28 +
29 # Bypass the function if we are using only 1 GPU.
30 if get_tensor_model_parallel_world_size() == 1:
31 return input_
vllm/distributed/device_communicators/__init__.py renamed
vllm/distributed/device_communicators/custom_all_reduce.py renamed
+3 -2
@@ -5,8 +5,6 @@ import torch
5 import torch.distributed as dist
6
7 from vllm.logger import init_logger
8 -from vllm.model_executor.parallel_utils.parallel_state import (
9 - get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
8
9 try:
10 import pynvml
@@ -25,6 +23,9 @@ _SUPPORTED_WORLD_SIZES = [2, 4, 6, 8]
23
24
25 def init_custom_ar() -> None:
26 + from vllm.distributed import (get_tensor_model_parallel_rank,
27 + get_tensor_model_parallel_world_size)
28 +
29 global _CA_HANDLE
30 if _CA_HANDLE is not None:
31 return
vllm/distributed/device_communicators/pynccl.py renamed
vllm/distributed/device_communicators/pynccl_utils.py renamed
+2 -2
@@ -9,8 +9,8 @@ from vllm.logger import init_logger
9 logger = init_logger(__name__)
10
11 try:
12 - from vllm.model_executor.parallel_utils.pynccl import (NCCLCommunicator,
13 - ncclGetVersion)
12 + from vllm.distributed.device_communicators.pynccl import (NCCLCommunicator,
13 + ncclGetVersion)
14 except Exception as e:
15 # in non-NVIDIA environments, we can't import the nccl module
16 # e.g. when running on machines with AMD GPUs
vllm/distributed/parallel_state.py renamed
+2 -2
@@ -8,8 +8,6 @@ from typing import Optional
8
9 import torch
10
11 -from vllm.model_executor.parallel_utils import pynccl_utils
12 -
11 # Tensor model parallel group that the current rank belongs to.
12 _TENSOR_MODEL_PARALLEL_GROUP = None
13 # Pipeline model parallel group that the current rank belongs to.
@@ -266,6 +264,7 @@ def destroy_model_parallel():
264 _PIPELINE_MODEL_PARALLEL_GROUP = None
265 global _PIPELINE_GLOBAL_RANKS
266 _PIPELINE_GLOBAL_RANKS = None
267 + from vllm.distributed.device_communicators import pynccl_utils
268
269 # Destroy the pynccl states if any.
270 pynccl_utils.destroy_process_group()
@@ -279,6 +278,7 @@ _ENABLE_PYNCCL_FOR_ALL_REDUCE = False
278
279 @contextlib.contextmanager
280 def with_pynccl_for_all_reduce():
281 + from vllm.distributed.device_communicators import pynccl_utils
282 """use pynccl instead of torch.distributed for all reduce"""
283 tp_size = get_tensor_model_parallel_world_size()
284 if tp_size == 1:
vllm/distributed/utils.py renamed
vllm/lora/layers.py
+6 -7
@@ -10,6 +10,12 @@ import torch.nn.functional as F
10 from transformers import PretrainedConfig
11
12 from vllm.config import LoRAConfig
13 +from vllm.distributed import (get_tensor_model_parallel_rank,
14 + get_tensor_model_parallel_world_size,
15 + split_tensor_along_last_dim,
16 + tensor_model_parallel_all_gather,
17 + tensor_model_parallel_all_reduce,
18 + tensor_model_parallel_gather)
19 from vllm.lora.punica import add_lora, add_lora_slice, bgmv
20 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
21 MergedColumnParallelLinear,
@@ -18,13 +24,6 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
24 from vllm.model_executor.layers.logits_processor import LogitsProcessor
25 from vllm.model_executor.layers.vocab_parallel_embedding import (
26 ParallelLMHead, VocabParallelEmbedding)
21 -from vllm.model_executor.parallel_utils.communication_op import (
22 - tensor_model_parallel_all_gather, tensor_model_parallel_all_reduce,
23 - tensor_model_parallel_gather)
24 -from vllm.model_executor.parallel_utils.parallel_state import (
25 - get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
26 -from vllm.model_executor.parallel_utils.utils import (
27 - split_tensor_along_last_dim)
27
28 if TYPE_CHECKING:
29 pass
vllm/model_executor/layers/activation.py
+2 -3
@@ -7,10 +7,9 @@ import torch.nn as nn
7 import torch.nn.functional as F
8
9 from vllm._C import ops
10 +from vllm.distributed import (divide, get_tensor_model_parallel_rank,
11 + get_tensor_model_parallel_world_size)
12 from vllm.model_executor.layers.quantization import QuantizationConfig
11 -from vllm.model_executor.parallel_utils.parallel_state import (
12 - get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
13 -from vllm.model_executor.parallel_utils.utils import divide
13 from vllm.model_executor.utils import set_weight_attrs
14
15
vllm/model_executor/layers/linear.py
+5 -6
@@ -5,13 +5,12 @@ import torch
5 import torch.nn.functional as F
6 from torch.nn.parameter import Parameter
7
8 +from vllm.distributed import (divide, get_tensor_model_parallel_rank,
9 + get_tensor_model_parallel_world_size,
10 + split_tensor_along_last_dim,
11 + tensor_model_parallel_all_gather,
12 + tensor_model_parallel_all_reduce)
13 from vllm.logger import init_logger
9 -from vllm.model_executor.parallel_utils.communication_op import (
10 - tensor_model_parallel_all_gather, tensor_model_parallel_all_reduce)
11 -from vllm.model_executor.parallel_utils.parallel_state import (
12 - get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
13 -from vllm.model_executor.parallel_utils.utils import (
14 - divide, split_tensor_along_last_dim)
14 from vllm.model_executor.utils import set_weight_attrs
15
16 logger = init_logger(__name__)
vllm/model_executor/layers/logits_processor.py
+1 -2
@@ -4,8 +4,7 @@ from typing import Optional
4 import torch
5 import torch.nn as nn
6
7 -from vllm.model_executor.parallel_utils.communication_op import (
8 - tensor_model_parallel_gather)
7 +from vllm.distributed import tensor_model_parallel_gather
8 from vllm.model_executor.sampling_metadata import SamplingMetadata
9
10
vllm/model_executor/layers/vocab_parallel_embedding.py
+3 -5
@@ -4,11 +4,9 @@ import torch
4 import torch.nn.functional as F
5 from torch.nn.parameter import Parameter
6
7 -from vllm.model_executor.parallel_utils.communication_op import (
8 - tensor_model_parallel_all_reduce)
9 -from vllm.model_executor.parallel_utils.parallel_state import (
10 - get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
11 -from vllm.model_executor.parallel_utils.utils import divide
7 +from vllm.distributed import (divide, get_tensor_model_parallel_rank,
8 + get_tensor_model_parallel_world_size,
9 + tensor_model_parallel_all_reduce)
10 from vllm.model_executor.utils import set_weight_attrs
11
12 DEFAULT_VOCAB_PADDING_SIZE = 64
vllm/model_executor/models/baichuan.py
+2 -2
@@ -27,6 +27,8 @@ from transformers import PretrainedConfig
27
28 from vllm.attention import Attention, AttentionMetadata
29 from vllm.config import LoRAConfig
30 +from vllm.distributed import (get_tensor_model_parallel_rank,
31 + get_tensor_model_parallel_world_size)
32 from vllm.model_executor.layers.activation import SiluAndMul
33 from vllm.model_executor.layers.layernorm import RMSNorm
34 from vllm.model_executor.layers.linear import (LinearMethodBase,
@@ -38,8 +40,6 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
40 from vllm.model_executor.layers.sampler import Sampler
41 from vllm.model_executor.layers.vocab_parallel_embedding import (
42 ParallelLMHead, VocabParallelEmbedding)
41 -from vllm.model_executor.parallel_utils.parallel_state import (
42 - get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
43 from vllm.model_executor.sampling_metadata import SamplingMetadata
44 from vllm.model_executor.weight_utils import (default_weight_loader,
45 hf_model_weights_iterator)
vllm/model_executor/models/bloom.py
+2 -2
@@ -24,6 +24,8 @@ from torch import nn
24 from transformers import BloomConfig
25
26 from vllm.attention import Attention, AttentionMetadata
27 +from vllm.distributed import (get_tensor_model_parallel_rank,
28 + get_tensor_model_parallel_world_size)
29 from vllm.model_executor.layers.activation import get_act_fn
30 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
31 LinearMethodBase,
@@ -33,8 +35,6 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
35 from vllm.model_executor.layers.sampler import Sampler
36 from vllm.model_executor.layers.vocab_parallel_embedding import (
37 VocabParallelEmbedding)
36 -from vllm.model_executor.parallel_utils.parallel_state import (
37 - get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
38 from vllm.model_executor.sampling_metadata import SamplingMetadata
39 from vllm.model_executor.weight_utils import (default_weight_loader,
40 hf_model_weights_iterator)
vllm/model_executor/models/chatglm.py
+1 -2
@@ -10,6 +10,7 @@ from torch.nn import LayerNorm
10
11 from vllm.attention import Attention, AttentionMetadata
12 from vllm.config import LoRAConfig
13 +from vllm.distributed import get_tensor_model_parallel_world_size
14 from vllm.model_executor.layers.activation import SiluAndMul
15 from vllm.model_executor.layers.layernorm import RMSNorm
16 from vllm.model_executor.layers.linear import (LinearMethodBase,
@@ -21,8 +22,6 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
22 from vllm.model_executor.layers.sampler import Sampler
23 from vllm.model_executor.layers.vocab_parallel_embedding import (
24 ParallelLMHead, VocabParallelEmbedding)
24 -from vllm.model_executor.parallel_utils.parallel_state import (
25 - get_tensor_model_parallel_world_size)
25 from vllm.model_executor.sampling_metadata import SamplingMetadata
26 from vllm.model_executor.weight_utils import (default_weight_loader,
27 hf_model_weights_iterator)
vllm/model_executor/models/commandr.py
+2 -2
@@ -29,6 +29,8 @@ from torch.nn.parameter import Parameter
29 from transformers import CohereConfig
30
31 from vllm.attention import Attention, AttentionMetadata
32 +from vllm.distributed import (get_tensor_model_parallel_rank,
33 + get_tensor_model_parallel_world_size)
34 from vllm.model_executor.layers.activation import SiluAndMul
35 from vllm.model_executor.layers.linear import (LinearMethodBase,
36 MergedColumnParallelLinear,
@@ -39,8 +41,6 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
41 from vllm.model_executor.layers.sampler import Sampler
42 from vllm.model_executor.layers.vocab_parallel_embedding import (
43 VocabParallelEmbedding)
42 -from vllm.model_executor.parallel_utils.parallel_state import (
43 - get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
44 from vllm.model_executor.sampling_metadata import SamplingMetadata
45 from vllm.model_executor.utils import set_weight_attrs
46 from vllm.model_executor.weight_utils import (default_weight_loader,
vllm/model_executor/models/dbrx.py
+3 -4
@@ -5,6 +5,9 @@ import torch
5 import torch.nn as nn
6
7 from vllm.attention import Attention, AttentionMetadata
8 +from vllm.distributed import (get_tensor_model_parallel_rank,
9 + get_tensor_model_parallel_world_size,
10 + tensor_model_parallel_all_reduce)
11 from vllm.model_executor.layers.fused_moe import fused_moe
12 from vllm.model_executor.layers.linear import (LinearMethodBase,
13 QKVParallelLinear,
@@ -15,10 +18,6 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
18 from vllm.model_executor.layers.sampler import Sampler
19 from vllm.model_executor.layers.vocab_parallel_embedding import (
20 DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
18 -from vllm.model_executor.parallel_utils.communication_op import (
19 - tensor_model_parallel_all_reduce)
20 -from vllm.model_executor.parallel_utils.parallel_state import (
21 - get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
21 from vllm.model_executor.sampling_metadata import SamplingMetadata
22 from vllm.model_executor.utils import set_weight_attrs
23 from vllm.model_executor.weight_utils import (default_weight_loader,
vllm/model_executor/models/deepseek.py
+3 -4
@@ -28,6 +28,9 @@ from torch import nn
28 from transformers import PretrainedConfig
29
30 from vllm.attention import Attention, AttentionMetadata
31 +from vllm.distributed import (get_tensor_model_parallel_rank,
32 + get_tensor_model_parallel_world_size,
33 + tensor_model_parallel_all_reduce)
34 from vllm.model_executor.layers.activation import SiluAndMul
35 from vllm.model_executor.layers.fused_moe import fused_moe
36 from vllm.model_executor.layers.layernorm import RMSNorm
@@ -41,10 +44,6 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
44 from vllm.model_executor.layers.sampler import Sampler
45 from vllm.model_executor.layers.vocab_parallel_embedding import (
46 ParallelLMHead, VocabParallelEmbedding)
44 -from vllm.model_executor.parallel_utils.communication_op import (
45 - tensor_model_parallel_all_reduce)
46 -from vllm.model_executor.parallel_utils.parallel_state import (
47 - get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
47 from vllm.model_executor.sampling_metadata import SamplingMetadata
48 from vllm.model_executor.weight_utils import (default_weight_loader,
49 hf_model_weights_iterator)
vllm/model_executor/models/falcon.py
+3 -4
@@ -27,6 +27,9 @@ from torch.nn import LayerNorm
27 from transformers import FalconConfig as HF_FalconConfig
28
29 from vllm.attention import Attention, AttentionMetadata
30 +from vllm.distributed import (get_tensor_model_parallel_rank,
31 + get_tensor_model_parallel_world_size,
32 + tensor_model_parallel_all_reduce)
33 from vllm.model_executor.layers.activation import get_act_fn
34 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
35 LinearMethodBase,
@@ -37,10 +40,6 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
40 from vllm.model_executor.layers.sampler import Sampler
41 from vllm.model_executor.layers.vocab_parallel_embedding import (
42 VocabParallelEmbedding)
40 -from vllm.model_executor.parallel_utils.communication_op import (
41 - tensor_model_parallel_all_reduce)
42 -from vllm.model_executor.parallel_utils.parallel_state import (
43 - get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
43 from vllm.model_executor.sampling_metadata import SamplingMetadata
44 from vllm.model_executor.weight_utils import (default_weight_loader,
45 hf_model_weights_iterator)
vllm/model_executor/models/gemma.py
+1 -2
@@ -23,6 +23,7 @@ from transformers import GemmaConfig
23
24 from vllm.attention import Attention, AttentionMetadata
25 from vllm.config import LoRAConfig
26 +from vllm.distributed import get_tensor_model_parallel_world_size
27 from vllm.logger import init_logger
28 from vllm.model_executor.layers.activation import GeluAndMul
29 from vllm.model_executor.layers.layernorm import RMSNorm
@@ -35,8 +36,6 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
36 from vllm.model_executor.layers.sampler import Sampler
37 from vllm.model_executor.layers.vocab_parallel_embedding import (
38 VocabParallelEmbedding)
38 -from vllm.model_executor.parallel_utils.parallel_state import (
39 - get_tensor_model_parallel_world_size)
39 from vllm.model_executor.sampling_metadata import SamplingMetadata
40 from vllm.model_executor.weight_utils import (default_weight_loader,
41 hf_model_weights_iterator)
vllm/model_executor/models/gpt2.py
+1 -2
@@ -24,6 +24,7 @@ from torch import nn
24 from transformers import GPT2Config
25
26 from vllm.attention import Attention, AttentionMetadata
27 +from vllm.distributed import get_tensor_model_parallel_world_size
28 from vllm.model_executor.layers.activation import get_act_fn
29 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
30 LinearMethodBase,
@@ -33,8 +34,6 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
34 from vllm.model_executor.layers.sampler import Sampler
35 from vllm.model_executor.layers.vocab_parallel_embedding import (
36 VocabParallelEmbedding)
36 -from vllm.model_executor.parallel_utils.parallel_state import (
37 - get_tensor_model_parallel_world_size)
37 from vllm.model_executor.sampling_metadata import SamplingMetadata
38 from vllm.model_executor.weight_utils import (default_weight_loader,
39 hf_model_weights_iterator)
vllm/model_executor/models/gpt_bigcode.py
+1 -2
@@ -25,6 +25,7 @@ from torch import nn
25 from transformers import GPTBigCodeConfig
26
27 from vllm.attention import Attention, AttentionMetadata
28 +from vllm.distributed import get_tensor_model_parallel_world_size
29 from vllm.model_executor.layers.activation import get_act_fn
30 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
31 LinearMethodBase,
@@ -34,8 +35,6 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
35 from vllm.model_executor.layers.sampler import Sampler
36 from vllm.model_executor.layers.vocab_parallel_embedding import (
37 VocabParallelEmbedding)
37 -from vllm.model_executor.parallel_utils.parallel_state import (
38 - get_tensor_model_parallel_world_size)
38 from vllm.model_executor.sampling_metadata import SamplingMetadata
39 from vllm.model_executor.weight_utils import (default_weight_loader,
40 hf_model_weights_iterator)
vllm/model_executor/models/gpt_j.py
+1 -2
@@ -23,6 +23,7 @@ from torch import nn
23 from transformers import GPTJConfig
24
25 from vllm.attention import Attention, AttentionMetadata
26 +from vllm.distributed import get_tensor_model_parallel_world_size
27 from vllm.model_executor.layers.activation import get_act_fn
28 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
29 LinearMethodBase,
@@ -33,8 +34,6 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
34 from vllm.model_executor.layers.sampler import Sampler
35 from vllm.model_executor.layers.vocab_parallel_embedding import (
36 ParallelLMHead, VocabParallelEmbedding)
36 -from vllm.model_executor.parallel_utils.parallel_state import (
37 - get_tensor_model_parallel_world_size)
37 from vllm.model_executor.sampling_metadata import SamplingMetadata
38 from vllm.model_executor.weight_utils import (default_weight_loader,
39 hf_model_weights_iterator)
vllm/model_executor/models/gpt_neox.py
+1 -2
@@ -23,6 +23,7 @@ from torch import nn
23 from transformers import GPTNeoXConfig
24
25 from vllm.attention import Attention, AttentionMetadata
26 +from vllm.distributed import get_tensor_model_parallel_world_size
27 from vllm.model_executor.layers.activation import get_act_fn
28 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
29 LinearMethodBase,
@@ -33,8 +34,6 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
34 from vllm.model_executor.layers.sampler import Sampler
35 from vllm.model_executor.layers.vocab_parallel_embedding import (
36 ParallelLMHead, VocabParallelEmbedding)
36 -from vllm.model_executor.parallel_utils.parallel_state import (
37 - get_tensor_model_parallel_world_size)
37 from vllm.model_executor.sampling_metadata import SamplingMetadata
38 from vllm.model_executor.weight_utils import (default_weight_loader,
39 hf_model_weights_iterator)
vllm/model_executor/models/internlm2.py
+1 -2
@@ -6,6 +6,7 @@ from torch import nn
6 from transformers import PretrainedConfig
7
8 from vllm.attention import Attention, AttentionMetadata
9 +from vllm.distributed import get_tensor_model_parallel_world_size
10 from vllm.model_executor.layers.activation import SiluAndMul
11 from vllm.model_executor.layers.layernorm import RMSNorm
12 from vllm.model_executor.layers.linear import (LinearMethodBase,
@@ -17,8 +18,6 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
18 from vllm.model_executor.layers.sampler import Sampler
19 from vllm.model_executor.layers.vocab_parallel_embedding import (
20 ParallelLMHead, VocabParallelEmbedding)
20 -from vllm.model_executor.parallel_utils.parallel_state import (
21 - get_tensor_model_parallel_world_size)
21 from vllm.model_executor.sampling_metadata import SamplingMetadata
22 from vllm.model_executor.weight_utils import (default_weight_loader,
23 hf_model_weights_iterator)
vllm/model_executor/models/jais.py
+2 -2
@@ -26,6 +26,8 @@ import torch
26 from torch import nn
27
28 from vllm.attention import Attention, AttentionMetadata
29 +from vllm.distributed import (get_tensor_model_parallel_rank,
30 + get_tensor_model_parallel_world_size)
31 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
32 LinearMethodBase,
33 QKVParallelLinear,
@@ -34,8 +36,6 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
36 from vllm.model_executor.layers.sampler import Sampler
37 from vllm.model_executor.layers.vocab_parallel_embedding import (
38 VocabParallelEmbedding)
37 -from vllm.model_executor.parallel_utils.parallel_state import (
38 - get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
39 from vllm.model_executor.sampling_metadata import SamplingMetadata
40 from vllm.model_executor.weight_utils import (default_weight_loader,
41 hf_model_weights_iterator)
vllm/model_executor/models/llama.py
+2 -2
@@ -29,6 +29,8 @@ from transformers import LlamaConfig
29
30 from vllm.attention import Attention, AttentionMetadata
31 from vllm.config import LoRAConfig
32 +from vllm.distributed import (get_tensor_model_parallel_rank,
33 + get_tensor_model_parallel_world_size)
34 from vllm.model_executor.layers.activation import SiluAndMul
35 from vllm.model_executor.layers.layernorm import RMSNorm
36 from vllm.model_executor.layers.linear import (LinearMethodBase,
@@ -40,8 +42,6 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
42 from vllm.model_executor.layers.sampler import Sampler
43 from vllm.model_executor.layers.vocab_parallel_embedding import (
44 DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
43 -from vllm.model_executor.parallel_utils.parallel_state import (
44 - get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
45 from vllm.model_executor.sampling_metadata import SamplingMetadata
46 from vllm.model_executor.weight_utils import (default_weight_loader,
47 hf_model_weights_iterator,
vllm/model_executor/models/minicpm.py
+3 -4
@@ -29,6 +29,9 @@ from torch import nn
29
30 from vllm.attention import Attention, AttentionMetadata
31 from vllm.config import LoRAConfig
32 +from vllm.distributed import (get_tensor_model_parallel_rank,
33 + get_tensor_model_parallel_world_size,
34 + tensor_model_parallel_all_reduce)
35 from vllm.model_executor.layers.activation import SiluAndMul
36 from vllm.model_executor.layers.fused_moe import fused_moe
37 from vllm.model_executor.layers.layernorm import RMSNorm
@@ -42,10 +45,6 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
45 from vllm.model_executor.layers.sampler import Sampler
46 from vllm.model_executor.layers.vocab_parallel_embedding import (
47 DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
45 -from vllm.model_executor.parallel_utils.communication_op import (
46 - tensor_model_parallel_all_reduce)
47 -from vllm.model_executor.parallel_utils.parallel_state import (
48 - get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
48 from vllm.model_executor.sampling_metadata import SamplingMetadata
49 from vllm.model_executor.utils import set_weight_attrs
50 from vllm.model_executor.weight_utils import (default_weight_loader,
vllm/model_executor/models/mixtral.py
+3 -4
@@ -29,6 +29,9 @@ from transformers import MixtralConfig
29
30 from vllm.attention import Attention, AttentionMetadata
31 from vllm.config import LoRAConfig
32 +from vllm.distributed import (get_tensor_model_parallel_rank,
33 + get_tensor_model_parallel_world_size,
34 + tensor_model_parallel_all_reduce)
35 from vllm.model_executor.layers.fused_moe import fused_moe
36 from vllm.model_executor.layers.layernorm import RMSNorm
37 from vllm.model_executor.layers.linear import (LinearMethodBase,
@@ -40,10 +43,6 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
43 from vllm.model_executor.layers.sampler import Sampler
44 from vllm.model_executor.layers.vocab_parallel_embedding import (
45 DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
43 -from vllm.model_executor.parallel_utils.communication_op import (
44 - tensor_model_parallel_all_reduce)
45 -from vllm.model_executor.parallel_utils.parallel_state import (
46 - get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
46 from vllm.model_executor.sampling_metadata import SamplingMetadata
47 from vllm.model_executor.utils import set_weight_attrs
48 from vllm.model_executor.weight_utils import (default_weight_loader,
vllm/model_executor/models/mixtral_quant.py
+3 -4
@@ -30,6 +30,9 @@ from torch import nn
30 from transformers import MixtralConfig
31
32 from vllm.attention import Attention, AttentionMetadata
33 +from vllm.distributed import (get_tensor_model_parallel_rank,
34 + get_tensor_model_parallel_world_size,
35 + tensor_model_parallel_all_reduce)
36 from vllm.model_executor.layers.layernorm import RMSNorm
37 from vllm.model_executor.layers.linear import (LinearMethodBase,
38 QKVParallelLinear,
@@ -40,10 +43,6 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
43 from vllm.model_executor.layers.sampler import Sampler
44 from vllm.model_executor.layers.vocab_parallel_embedding import (
45 ParallelLMHead, VocabParallelEmbedding)
43 -from vllm.model_executor.parallel_utils.communication_op import (
44 - tensor_model_parallel_all_reduce)
45 -from vllm.model_executor.parallel_utils.parallel_state import (
46 - get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
46 from vllm.model_executor.sampling_metadata import SamplingMetadata
47 from vllm.model_executor.weight_utils import (default_weight_loader,
48 hf_model_weights_iterator)
vllm/model_executor/models/mpt.py
+2 -2
@@ -7,6 +7,8 @@ import torch
7 import torch.nn as nn
8
9 from vllm.attention import Attention, AttentionMetadata
10 +from vllm.distributed import (get_tensor_model_parallel_rank,
11 + get_tensor_model_parallel_world_size)
12 from vllm.model_executor.layers.activation import get_act_fn
13 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
14 LinearMethodBase,
@@ -16,8 +18,6 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
18 from vllm.model_executor.layers.sampler import Sampler
19 from vllm.model_executor.layers.vocab_parallel_embedding import (
20 VocabParallelEmbedding)
19 -from vllm.model_executor.parallel_utils.parallel_state import (
20 - get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
21 from vllm.model_executor.sampling_metadata import SamplingMetadata
22 from vllm.model_executor.weight_utils import (default_weight_loader,
23 hf_model_weights_iterator)
vllm/model_executor/models/olmo.py
+1 -2
@@ -44,6 +44,7 @@ from hf_olmo import OLMoConfig
44 from torch import nn
45
46 from vllm.attention import Attention, AttentionMetadata
47 +from vllm.distributed import get_tensor_model_parallel_world_size
48 from vllm.model_executor.layers.activation import SiluAndMul
49 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
50 LinearMethodBase,
@@ -55,8 +56,6 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
56 from vllm.model_executor.layers.sampler import Sampler
57 from vllm.model_executor.layers.vocab_parallel_embedding import (
58 VocabParallelEmbedding)
58 -from vllm.model_executor.parallel_utils.parallel_state import (
59 - get_tensor_model_parallel_world_size)
59 from vllm.model_executor.sampling_metadata import SamplingMetadata
60 from vllm.model_executor.weight_utils import (default_weight_loader,
61 hf_model_weights_iterator)
vllm/model_executor/models/opt.py
+1 -2
@@ -24,6 +24,7 @@ from torch import nn
24 from transformers import OPTConfig
25
26 from vllm.attention import Attention, AttentionMetadata
27 +from vllm.distributed import get_tensor_model_parallel_world_size
28 from vllm.model_executor.layers.activation import get_act_fn
29 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
30 LinearMethodBase,
@@ -34,8 +35,6 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
35 from vllm.model_executor.layers.sampler import Sampler
36 from vllm.model_executor.layers.vocab_parallel_embedding import (
37 VocabParallelEmbedding)
37 -from vllm.model_executor.parallel_utils.parallel_state import (
38 - get_tensor_model_parallel_world_size)
38 from vllm.model_executor.sampling_metadata import SamplingMetadata
39 from vllm.model_executor.weight_utils import (default_weight_loader,
40 hf_model_weights_iterator)
vllm/model_executor/models/orion.py
+1 -2
@@ -11,6 +11,7 @@ from torch import nn
11 from transformers import PretrainedConfig
12
13 from vllm.attention import Attention, AttentionMetadata
14 +from vllm.distributed import get_tensor_model_parallel_world_size
15 from vllm.model_executor.layers.activation import SiluAndMul
16 from vllm.model_executor.layers.linear import (LinearMethodBase,
17 MergedColumnParallelLinear,
@@ -21,8 +22,6 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
22 from vllm.model_executor.layers.sampler import Sampler
23 from vllm.model_executor.layers.vocab_parallel_embedding import (
24 ParallelLMHead, VocabParallelEmbedding)
24 -from vllm.model_executor.parallel_utils.parallel_state import (
25 - get_tensor_model_parallel_world_size)
25 from vllm.model_executor.sampling_metadata import SamplingMetadata
26 from vllm.model_executor.weight_utils import (default_weight_loader,
27 hf_model_weights_iterator)
vllm/model_executor/models/phi.py
+1 -2
@@ -42,6 +42,7 @@ from torch import nn
42 from transformers import PretrainedConfig
43
44 from vllm.attention import Attention, AttentionMetadata
45 +from vllm.distributed import get_tensor_model_parallel_world_size
46 from vllm.model_executor.layers.activation import get_act_fn
47 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
48 LinearMethodBase,
@@ -52,8 +53,6 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
53 from vllm.model_executor.layers.sampler import Sampler
54 from vllm.model_executor.layers.vocab_parallel_embedding import (
55 ParallelLMHead, VocabParallelEmbedding)
55 -from vllm.model_executor.parallel_utils.parallel_state import (
56 - get_tensor_model_parallel_world_size)
56 from vllm.model_executor.sampling_metadata import SamplingMetadata
57 from vllm.model_executor.weight_utils import (default_weight_loader,
58 hf_model_weights_iterator)
vllm/model_executor/models/qwen.py
+1 -2
@@ -11,6 +11,7 @@ from torch import nn
11 from transformers import PretrainedConfig
12
13 from vllm.attention import Attention, AttentionMetadata
14 +from vllm.distributed import get_tensor_model_parallel_world_size
15 from vllm.model_executor.layers.activation import SiluAndMul
16 from vllm.model_executor.layers.layernorm import RMSNorm
17 from vllm.model_executor.layers.linear import (LinearMethodBase,
@@ -22,8 +23,6 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
23 from vllm.model_executor.layers.sampler import Sampler
24 from vllm.model_executor.layers.vocab_parallel_embedding import (
25 ParallelLMHead, VocabParallelEmbedding)
25 -from vllm.model_executor.parallel_utils.parallel_state import (
26 - get_tensor_model_parallel_world_size)
26 from vllm.model_executor.sampling_metadata import SamplingMetadata
27 from vllm.model_executor.weight_utils import (default_weight_loader,
28 hf_model_weights_iterator)
vllm/model_executor/models/qwen2.py
+1 -2
@@ -30,6 +30,7 @@ from transformers import Qwen2Config
30
31 from vllm.attention import Attention, AttentionMetadata
32 from vllm.config import LoRAConfig
33 +from vllm.distributed import get_tensor_model_parallel_world_size
34 from vllm.model_executor.layers.activation import SiluAndMul
35 from vllm.model_executor.layers.layernorm import RMSNorm
36 from vllm.model_executor.layers.linear import (LinearMethodBase,
@@ -41,8 +42,6 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
42 from vllm.model_executor.layers.sampler import Sampler
43 from vllm.model_executor.layers.vocab_parallel_embedding import (
44 ParallelLMHead, VocabParallelEmbedding)
44 -from vllm.model_executor.parallel_utils.parallel_state import (
45 - get_tensor_model_parallel_world_size)
45 from vllm.model_executor.sampling_metadata import SamplingMetadata
46 from vllm.model_executor.weight_utils import (default_weight_loader,
47 hf_model_weights_iterator)
vllm/model_executor/models/qwen2_moe.py
+3 -4
@@ -30,6 +30,9 @@ from torch import nn
30 from transformers import PretrainedConfig
31
32 from vllm.attention import Attention, AttentionMetadata
33 +from vllm.distributed import (get_tensor_model_parallel_rank,
34 + get_tensor_model_parallel_world_size,
35 + tensor_model_parallel_all_reduce)
36 from vllm.model_executor.layers.activation import SiluAndMul
37 from vllm.model_executor.layers.fused_moe import fused_moe
38 from vllm.model_executor.layers.layernorm import RMSNorm
@@ -43,10 +46,6 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
46 from vllm.model_executor.layers.sampler import Sampler
47 from vllm.model_executor.layers.vocab_parallel_embedding import (
48 ParallelLMHead, VocabParallelEmbedding)
46 -from vllm.model_executor.parallel_utils.communication_op import (
47 - tensor_model_parallel_all_reduce)
48 -from vllm.model_executor.parallel_utils.parallel_state import (
49 - get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
49 from vllm.model_executor.sampling_metadata import SamplingMetadata
50 from vllm.model_executor.weight_utils import (default_weight_loader,
51 hf_model_weights_iterator)
vllm/model_executor/models/stablelm.py
+1 -2
@@ -26,6 +26,7 @@ from torch import nn
26 from transformers import PretrainedConfig
27
28 from vllm.attention import Attention, AttentionMetadata
29 +from vllm.distributed import get_tensor_model_parallel_world_size
30 from vllm.model_executor.layers.activation import SiluAndMul
31 from vllm.model_executor.layers.linear import (LinearMethodBase,
32 MergedColumnParallelLinear,
@@ -36,8 +37,6 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
37 from vllm.model_executor.layers.sampler import Sampler
38 from vllm.model_executor.layers.vocab_parallel_embedding import (
39 ParallelLMHead, VocabParallelEmbedding)
39 -from vllm.model_executor.parallel_utils.parallel_state import (
40 - get_tensor_model_parallel_world_size)
40 from vllm.model_executor.sampling_metadata import SamplingMetadata
41 from vllm.model_executor.weight_utils import (default_weight_loader,
42 hf_model_weights_iterator)
vllm/model_executor/models/starcoder2.py
+1 -2
@@ -25,6 +25,7 @@ from torch import nn
25 from transformers import Starcoder2Config
26
27 from vllm.attention import Attention, AttentionMetadata
28 +from vllm.distributed import get_tensor_model_parallel_world_size
29 from vllm.model_executor.layers.activation import get_act_fn
30 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
31 LinearMethodBase,
@@ -35,8 +36,6 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
36 from vllm.model_executor.layers.sampler import Sampler
37 from vllm.model_executor.layers.vocab_parallel_embedding import (
38 DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
38 -from vllm.model_executor.parallel_utils.parallel_state import (
39 - get_tensor_model_parallel_world_size)
39 from vllm.model_executor.sampling_metadata import SamplingMetadata
40 from vllm.model_executor.weight_utils import (default_weight_loader,
41 hf_model_weights_iterator)
vllm/model_executor/models/xverse.py
+1 -2
@@ -28,6 +28,7 @@ from transformers import PretrainedConfig
28
29 from vllm.attention import Attention, AttentionMetadata
30 from vllm.config import LoRAConfig
31 +from vllm.distributed import get_tensor_model_parallel_world_size
32 from vllm.model_executor.layers.activation import SiluAndMul
33 from vllm.model_executor.layers.layernorm import RMSNorm
34 from vllm.model_executor.layers.linear import (LinearMethodBase,
@@ -39,8 +40,6 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
40 from vllm.model_executor.layers.sampler import Sampler
41 from vllm.model_executor.layers.vocab_parallel_embedding import (
42 ParallelLMHead, VocabParallelEmbedding)
42 -from vllm.model_executor.parallel_utils.parallel_state import (
43 - get_tensor_model_parallel_world_size)
43 from vllm.model_executor.sampling_metadata import SamplingMetadata
44 from vllm.model_executor.weight_utils import (default_weight_loader,
45 hf_model_weights_iterator)
vllm/model_executor/parallel_utils/README.md deleted
-1
@@ -1 +0,0 @@
1 -The files in this folder are ported from [Megatron-LM](https://github.com/NVIDIA/Megatron-LM/tree/main/megatron/core). We only keep the codes that are used in inference.
\ No newline at end of file
vllm/test_utils.py
+2 -2
@@ -1,7 +1,7 @@
1 import ray
2
3 -from vllm.model_executor.parallel_utils.parallel_state import (
4 - ensure_model_parallel_initialized, init_distributed_environment)
3 +from vllm.distributed import (ensure_model_parallel_initialized,
4 + init_distributed_environment)
5 from vllm.utils import get_open_port
6
7
vllm/worker/cpu_worker.py
+3 -4
@@ -7,13 +7,12 @@ import torch.distributed
7 from vllm.attention import get_attn_backend
8 from vllm.config import (CacheConfig, DeviceConfig, LoRAConfig, ModelConfig,
9 ParallelConfig, SchedulerConfig)
10 +from vllm.distributed import (broadcast_tensor_dict,
11 + ensure_model_parallel_initialized,
12 + init_distributed_environment)
13 from vllm.logger import init_logger
14 from vllm.model_executor import set_random_seed
15 from vllm.model_executor.model_loader import get_model
13 -from vllm.model_executor.parallel_utils.communication_op import (
14 - broadcast_tensor_dict)
15 -from vllm.model_executor.parallel_utils.parallel_state import (
16 - ensure_model_parallel_initialized, init_distributed_environment)
16 from vllm.sequence import SamplerOutput, SequenceGroupMetadata
17 from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE
18 from vllm.worker.model_runner import ModelRunner
vllm/worker/model_runner.py
+3 -5
@@ -9,17 +9,15 @@ import torch.nn as nn
9 from vllm.attention import AttentionMetadata, get_attn_backend
10 from vllm.config import (DeviceConfig, LoRAConfig, ModelConfig, ParallelConfig,
11 SchedulerConfig, VisionLanguageConfig)
12 +from vllm.distributed import broadcast_tensor_dict, with_pynccl_for_all_reduce
13 +from vllm.distributed.device_communicators import (custom_all_reduce,
14 + pynccl_utils)
15 from vllm.logger import init_logger
16 from vllm.lora.layers import LoRAMapping
17 from vllm.lora.request import LoRARequest
18 from vllm.lora.worker_manager import LRUCacheWorkerLoRAManager
19 from vllm.model_executor import SamplingMetadata
20 from vllm.model_executor.model_loader import get_model
18 -from vllm.model_executor.parallel_utils import custom_all_reduce, pynccl_utils
19 -from vllm.model_executor.parallel_utils.communication_op import (
20 - broadcast_tensor_dict)
21 -from vllm.model_executor.parallel_utils.parallel_state import (
22 - with_pynccl_for_all_reduce)
21 from vllm.sampling_params import SamplingParams, SamplingType
22 from vllm.sequence import (MultiModalData, SamplerOutput, SequenceData,
23 SequenceGroupMetadata)
vllm/worker/worker.py
+6 -6
@@ -8,14 +8,14 @@ import torch.distributed
8
9 from vllm.config import (CacheConfig, DeviceConfig, LoRAConfig, ModelConfig,
10 ParallelConfig, SchedulerConfig, VisionLanguageConfig)
11 +from vllm.distributed import (broadcast_tensor_dict,
12 + ensure_model_parallel_initialized,
13 + init_distributed_environment)
14 +from vllm.distributed.device_communicators import pynccl_utils
15 +from vllm.distributed.device_communicators.custom_all_reduce import (
16 + init_custom_ar)
17 from vllm.lora.request import LoRARequest
18 from vllm.model_executor import set_random_seed
13 -from vllm.model_executor.parallel_utils import pynccl_utils
14 -from vllm.model_executor.parallel_utils.communication_op import (
15 - broadcast_tensor_dict)
16 -from vllm.model_executor.parallel_utils.custom_all_reduce import init_custom_ar
17 -from vllm.model_executor.parallel_utils.parallel_state import (
18 - ensure_model_parallel_initialized, init_distributed_environment)
19 from vllm.sequence import SamplerOutput, SequenceGroupMetadata
20 from vllm.worker.cache_engine import CacheEngine
21 from vllm.worker.model_runner import ModelRunner