A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 18:15 UTC
[Bugfix] neuron: enable tensor parallelism (#7562)
Signed-off-by: omrishiv <327609+omrishiv@users.noreply.github.com>
omrishiv committed
Aug 26, 2024 at 15:13 UTC
760e9f71a839ddc2a05c47af1fea23eeefbc368e
3 files changed
+44
-11
vllm/engine/arg_utils.py
+5
-3
@@ -317,9 +317,10 @@ class EngineArgs:
317
parser.add_argument('--block-size',
318
type=int,
319
default=EngineArgs.block_size,
320
- choices=[8, 16, 32, 128, 256, 512, 1024, 2048],
320
+ choices=[8, 16, 32],
321
help='Token block size for contiguous chunks of '
322
- 'tokens.')
322
+ 'tokens. This is ignored on neuron devices and '
323
+ 'set to max-model-len')
324
325
parser.add_argument('--enable-prefix-caching',
326
action='store_true',
@@ -793,7 +794,8 @@ class EngineArgs:
794
limit_mm_per_prompt=self.limit_mm_per_prompt,
795
)
796
cache_config = CacheConfig(
796
- block_size=self.block_size,
797
+ block_size=self.block_size if self.device != "neuron" else
798
+ self.max_model_len, # neuron needs block_size = max_model_len
799
gpu_memory_utilization=self.gpu_memory_utilization,
800
swap_space=self.swap_space,
801
cache_dtype=self.kv_cache_dtype,
vllm/executor/neuron_executor.py
+12
-8
@@ -4,7 +4,8 @@ from vllm.executor.executor_base import ExecutorAsyncBase, ExecutorBase
4
from vllm.logger import init_logger
5
from vllm.lora.request import LoRARequest
6
from vllm.sequence import ExecuteModelRequest, SamplerOutput
7
-from vllm.utils import make_async
7
+from vllm.utils import (get_distributed_init_method, get_ip, get_open_port,
8
+ make_async)
9
10
logger = init_logger(__name__)
11
@@ -24,14 +25,17 @@ class NeuronExecutor(ExecutorBase):
25
26
def _init_worker(self):
27
from vllm.worker.neuron_worker import NeuronWorker
27
-
28
+ distributed_init_method = get_distributed_init_method(
29
+ get_ip(), get_open_port())
30
self.driver_worker = NeuronWorker(
29
- self.model_config,
30
- self.parallel_config,
31
- self.scheduler_config,
32
- self.device_config,
33
- self.cache_config,
34
- )
31
+ model_config=self.model_config,
32
+ parallel_config=self.parallel_config,
33
+ scheduler_config=self.scheduler_config,
34
+ device_config=self.device_config,
35
+ cache_config=self.cache_config,
36
+ local_rank=0,
37
+ rank=0,
38
+ distributed_init_method=distributed_init_method)
39
self.driver_worker.init_device()
40
self.driver_worker.load_model()
41
vllm/worker/neuron_worker.py
+27
@@ -6,6 +6,8 @@ import torch.distributed
6
7
from vllm.config import (CacheConfig, DeviceConfig, ModelConfig,
8
ParallelConfig, SchedulerConfig)
9
+from vllm.distributed import (ensure_model_parallel_initialized,
10
+ init_distributed_environment)
11
from vllm.model_executor import set_random_seed
12
from vllm.sequence import ExecuteModelRequest
13
from vllm.worker.neuron_model_runner import NeuronModelRunner
@@ -24,12 +26,18 @@ class NeuronWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
26
scheduler_config: SchedulerConfig,
27
device_config: DeviceConfig,
28
cache_config: CacheConfig,
29
+ local_rank: int,
30
+ rank: int,
31
+ distributed_init_method: str,
32
) -> None:
33
self.model_config = model_config
34
self.parallel_config = parallel_config
35
self.scheduler_config = scheduler_config
36
self.device_config = device_config
37
self.cache_config = cache_config
38
+ self.local_rank = local_rank
39
+ self.rank = rank
40
+ self.distributed_init_method = distributed_init_method
41
if self.model_config.trust_remote_code:
42
# note: lazy import to avoid importing torch before initializing
43
from vllm.utils import init_cached_hf_modules
@@ -40,6 +48,8 @@ class NeuronWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
48
self.is_driver_worker = True
49
50
def init_device(self) -> None:
51
+ self.init_distributed_environment()
52
+
53
# Set random seed.
54
set_random_seed(self.model_config.seed)
55
@@ -98,3 +108,20 @@ class NeuronWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
108
This is required for speculative decoding; it is not yet implemented.
109
"""
110
raise NotImplementedError
111
+
112
+ def init_distributed_environment(self):
113
+ """Neuron uses transformers-neuronx for tensor parallelism.
114
+
115
+ vLLM still needs the environment inited when TP/PP > 1
116
+ """
117
+ init_distributed_environment(
118
+ world_size=1,
119
+ rank=self.rank,
120
+ local_rank=self.local_rank,
121
+ distributed_init_method=self.distributed_init_method,
122
+ backend="gloo",
123
+ )
124
+ ensure_model_parallel_initialized(
125
+ 1,
126
+ 1,
127
+ )