@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 760e9f71a8
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 18:15 UTC

[Bugfix] neuron: enable tensor parallelism (#7562)

Signed-off-by: omrishiv <327609+omrishiv@users.noreply.github.com>

omrishiv committed Aug 26, 2024 at 15:13 UTC 760e9f71a839ddc2a05c47af1fea23eeefbc368e
3 files changed +44 -11
vllm/engine/arg_utils.py
+5 -3
@@ -317,9 +317,10 @@ class EngineArgs:
317 parser.add_argument('--block-size',
318 type=int,
319 default=EngineArgs.block_size,
320 - choices=[8, 16, 32, 128, 256, 512, 1024, 2048],
320 + choices=[8, 16, 32],
321 help='Token block size for contiguous chunks of '
322 - 'tokens.')
322 + 'tokens. This is ignored on neuron devices and '
323 + 'set to max-model-len')
324
325 parser.add_argument('--enable-prefix-caching',
326 action='store_true',
@@ -793,7 +794,8 @@ class EngineArgs:
794 limit_mm_per_prompt=self.limit_mm_per_prompt,
795 )
796 cache_config = CacheConfig(
796 - block_size=self.block_size,
797 + block_size=self.block_size if self.device != "neuron" else
798 + self.max_model_len, # neuron needs block_size = max_model_len
799 gpu_memory_utilization=self.gpu_memory_utilization,
800 swap_space=self.swap_space,
801 cache_dtype=self.kv_cache_dtype,
vllm/executor/neuron_executor.py
+12 -8
@@ -4,7 +4,8 @@ from vllm.executor.executor_base import ExecutorAsyncBase, ExecutorBase
4 from vllm.logger import init_logger
5 from vllm.lora.request import LoRARequest
6 from vllm.sequence import ExecuteModelRequest, SamplerOutput
7 -from vllm.utils import make_async
7 +from vllm.utils import (get_distributed_init_method, get_ip, get_open_port,
8 + make_async)
9
10 logger = init_logger(__name__)
11
@@ -24,14 +25,17 @@ class NeuronExecutor(ExecutorBase):
25
26 def _init_worker(self):
27 from vllm.worker.neuron_worker import NeuronWorker
27 -
28 + distributed_init_method = get_distributed_init_method(
29 + get_ip(), get_open_port())
30 self.driver_worker = NeuronWorker(
29 - self.model_config,
30 - self.parallel_config,
31 - self.scheduler_config,
32 - self.device_config,
33 - self.cache_config,
34 - )
31 + model_config=self.model_config,
32 + parallel_config=self.parallel_config,
33 + scheduler_config=self.scheduler_config,
34 + device_config=self.device_config,
35 + cache_config=self.cache_config,
36 + local_rank=0,
37 + rank=0,
38 + distributed_init_method=distributed_init_method)
39 self.driver_worker.init_device()
40 self.driver_worker.load_model()
41
vllm/worker/neuron_worker.py
+27
@@ -6,6 +6,8 @@ import torch.distributed
6
7 from vllm.config import (CacheConfig, DeviceConfig, ModelConfig,
8 ParallelConfig, SchedulerConfig)
9 +from vllm.distributed import (ensure_model_parallel_initialized,
10 + init_distributed_environment)
11 from vllm.model_executor import set_random_seed
12 from vllm.sequence import ExecuteModelRequest
13 from vllm.worker.neuron_model_runner import NeuronModelRunner
@@ -24,12 +26,18 @@ class NeuronWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
26 scheduler_config: SchedulerConfig,
27 device_config: DeviceConfig,
28 cache_config: CacheConfig,
29 + local_rank: int,
30 + rank: int,
31 + distributed_init_method: str,
32 ) -> None:
33 self.model_config = model_config
34 self.parallel_config = parallel_config
35 self.scheduler_config = scheduler_config
36 self.device_config = device_config
37 self.cache_config = cache_config
38 + self.local_rank = local_rank
39 + self.rank = rank
40 + self.distributed_init_method = distributed_init_method
41 if self.model_config.trust_remote_code:
42 # note: lazy import to avoid importing torch before initializing
43 from vllm.utils import init_cached_hf_modules
@@ -40,6 +48,8 @@ class NeuronWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
48 self.is_driver_worker = True
49
50 def init_device(self) -> None:
51 + self.init_distributed_environment()
52 +
53 # Set random seed.
54 set_random_seed(self.model_config.seed)
55
@@ -98,3 +108,20 @@ class NeuronWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
108 This is required for speculative decoding; it is not yet implemented.
109 """
110 raise NotImplementedError
111 +
112 + def init_distributed_environment(self):
113 + """Neuron uses transformers-neuronx for tensor parallelism.
114 +
115 + vLLM still needs the environment inited when TP/PP > 1
116 + """
117 + init_distributed_environment(
118 + world_size=1,
119 + rank=self.rank,
120 + local_rank=self.local_rank,
121 + distributed_init_method=self.distributed_init_method,
122 + backend="gloo",
123 + )
124 + ensure_model_parallel_initialized(
125 + 1,
126 + 1,
127 + )