A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
[Hardware][Neuron] Refactor neuron support (#3471)
Zhuohan Li committed
Mar 21, 2024 at 18:22 UTC
e90fc21f2eda7e53f692398ee2c0cb5a0ac19693
33 files changed
+615
-549
examples/offline_inference_neuron.py
+3
-2
@@ -12,7 +12,7 @@ sampling_params = SamplingParams(temperature=0.8, top_p=0.95)
12
13
# Create an LLM.
14
llm = LLM(
15
- model="openlm-research/open_llama_3b",
15
+ model="TinyLlama/TinyLlama-1.1B-Chat-v1.0",
16
max_num_seqs=8,
17
# The max_model_len and block_size arguments are required to be same as
18
# max sequence length when targeting neuron device.
@@ -24,7 +24,8 @@ llm = LLM(
24
# The device can be automatically detected when AWS Neuron SDK is installed.
25
# The device argument can be either unspecified for automated detection,
26
# or explicitly assigned.
27
- device="neuron")
27
+ device="neuron",
28
+ tensor_parallel_size=2)
29
# Generate texts from the prompts. The output is a list of RequestOutput objects
30
# that contain the prompt, generated text, and other information.
31
outputs = llm.generate(prompts, sampling_params)
tests/lora/test_worker.py
+1
-1
@@ -33,7 +33,7 @@ def test_worker_apply_lora(sql_lora_files):
33
max_loras=32),
34
distributed_init_method=f"file://{tempfile.mkstemp()[1]}",
35
)
36
- worker.init_model()
36
+ worker.init_device()
37
worker.load_model()
38
39
worker.model_runner.set_active_loras([], LoRAMapping([], []))
tests/spec_decode/test_spec_decode_worker.py
+9
-9
@@ -71,7 +71,7 @@ def test_correctly_calls_target_model(k: int, batch_size: int):
71
72
worker = SpecDecodeWorker(draft_worker, target_worker, rejection_sampler,
73
metrics_collector)
74
- worker.init_model()
74
+ worker.init_device()
75
76
vocab_size = 32_000
77
@@ -151,7 +151,7 @@ def test_correctly_calls_rejection_sampler(k: int, batch_size: int):
151
152
worker = SpecDecodeWorker(draft_worker, target_worker, rejection_sampler,
153
metrics_collector)
154
- worker.init_model()
154
+ worker.init_device()
155
156
proposal_token_ids = torch.randint(low=0,
157
high=vocab_size,
@@ -230,7 +230,7 @@ def test_correctly_formats_output(k: int, batch_size: int):
230
231
worker = SpecDecodeWorker(draft_worker, target_worker, rejection_sampler,
232
metrics_collector)
233
- worker.init_model()
233
+ worker.init_device()
234
235
proposal_token_ids = torch.randint(low=0,
236
high=vocab_size,
@@ -342,7 +342,7 @@ def test_collects_metrics(k: int, batch_size: int, returns_metrics: bool):
342
343
worker = SpecDecodeWorker(draft_worker, target_worker, rejection_sampler,
344
metrics_collector)
345
- worker.init_model()
345
+ worker.init_device()
346
347
proposal_token_ids = torch.randint(low=0,
348
high=vocab_size,
@@ -486,8 +486,8 @@ def test_empty_input_batch(k: int, batch_size: int):
486
487
488
@torch.inference_mode()
489
-def test_init_model():
490
- """Verify SpecDecodeWorker invokes proposer/scorer worker init_model, as
489
+def test_init_device():
490
+ """Verify SpecDecodeWorker invokes proposer/scorer worker init_device, as
491
well as other GPU initialization.
492
"""
493
draft_worker = mock_worker(cls=MultiStepWorker)
@@ -499,11 +499,11 @@ def test_init_model():
499
worker = SpecDecodeWorker(draft_worker, target_worker, rejection_sampler,
500
metrics_collector)
501
502
- worker.init_model()
502
+ worker.init_device()
503
504
- draft_worker.init_model.assert_called_once()
504
+ draft_worker.init_device.assert_called_once()
505
506
- target_worker.init_model.assert_called_once()
506
+ target_worker.init_device.assert_called_once()
507
508
metrics_collector.init_gpu_tensors.assert_called_once()
509
rejection_sampler.init_gpu_tensors.assert_called_once()
tests/spec_decode/utils.py
+1
-1
@@ -123,7 +123,7 @@ def create_worker(cls: type,
123
is_driver_worker=is_driver_worker,
124
)
125
126
- worker.init_model()
126
+ worker.init_device()
127
worker.load_model()
128
129
cache_config.num_gpu_blocks = num_gpu_blocks
tests/worker/test_swap.py
+1
-1
@@ -30,7 +30,7 @@ def test_swap() -> None:
30
)
31
32
# Initialize the worker.
33
- worker.init_model()
33
+ worker.init_device()
34
worker.load_model()
35
worker.init_cache_engine(cache_config)
36
worker.warm_up_model()
vllm/config.py
+2
-15
@@ -474,15 +474,7 @@ class ParallelConfig:
474
placement_group: Optional["PlacementGroup"] = None,
475
) -> None:
476
self.pipeline_parallel_size = pipeline_parallel_size
477
- if is_neuron():
478
- # For Neuron device support, here we assign TP=1 to avoid sharding
479
- # within vLLM directly. Transformer-neuronx would take
480
- # neuron_tp_degree attribute, and distribute the workload
481
- # to multiple NeuronCores.
482
- self.tensor_parallel_size = 1
483
- self.neuron_tp_degree = tensor_parallel_size
484
- else:
485
- self.tensor_parallel_size = tensor_parallel_size
477
+ self.tensor_parallel_size = tensor_parallel_size
478
self.worker_use_ray = worker_use_ray
479
self.max_parallel_loading_workers = max_parallel_loading_workers
480
self.disable_custom_all_reduce = disable_custom_all_reduce
@@ -491,8 +483,7 @@ class ParallelConfig:
483
self.placement_group = placement_group
484
485
self.world_size = pipeline_parallel_size * self.tensor_parallel_size
494
- # Ray worker is not supported for Neuron backend.
495
- if self.world_size > 1 and not is_neuron():
486
+ if self.world_size > 1:
487
self.worker_use_ray = True
488
self._verify_args()
489
@@ -591,10 +582,6 @@ class DeviceConfig:
582
# Set device with device type
583
self.device = torch.device(self.device_type)
584
594
- @property
595
- def is_neuron(self):
596
- return self.device_type == "neuron"
597
-
585
586
@dataclass
587
class LoRAConfig:
vllm/engine/async_llm_engine.py
+6
-1
@@ -325,7 +325,12 @@ class AsyncLLMEngine:
325
# Create the engine configs.
326
engine_configs = engine_args.create_engine_configs()
327
parallel_config = engine_configs[2]
328
- if parallel_config.worker_use_ray or engine_args.engine_use_ray:
328
+ device_config = engine_configs[4]
329
+
330
+ if device_config.device_type == "neuron":
331
+ raise NotImplementedError("Neuron is not supported for "
332
+ "async engine yet.")
333
+ elif parallel_config.worker_use_ray or engine_args.engine_use_ray:
334
initialize_ray_cluster(parallel_config)
335
from vllm.executor.ray_gpu_executor import RayGPUExecutorAsync
336
executor_class = RayGPUExecutorAsync
vllm/engine/llm_engine.py
+5
-1
@@ -125,9 +125,13 @@ class LLMEngine:
125
# Create the engine configs.
126
engine_configs = engine_args.create_engine_configs()
127
parallel_config = engine_configs[2]
128
+ device_config = engine_configs[4]
129
130
# Initialize the cluster and specify the executor class.
130
- if parallel_config.worker_use_ray:
131
+ if device_config.device_type == "neuron":
132
+ from vllm.executor.neuron_executor import NeuronExecutor
133
+ executor_class = NeuronExecutor
134
+ elif parallel_config.worker_use_ray:
135
initialize_ray_cluster(parallel_config)
136
from vllm.executor.ray_gpu_executor import RayGPUExecutor
137
executor_class = RayGPUExecutor
vllm/executor/gpu_executor.py
+2
-16
@@ -1,4 +1,3 @@
1
-import importlib
1
from typing import Dict, List, Optional
2
3
from vllm.lora.request import LoRARequest
@@ -13,12 +12,6 @@ from vllm.utils import (get_ip, get_open_port, get_distributed_init_method,
12
13
logger = init_logger(__name__)
14
16
-# A map between the device type (in device config) to its worker module.
17
-DEVICE_TO_WORKER_MODULE_MAP = {
18
- "cuda": "vllm.worker.worker",
19
- "neuron": "vllm.worker.neuron_worker",
20
-}
21
-
15
16
class GPUExecutor(ExecutorBase):
17
@@ -44,17 +37,10 @@ class GPUExecutor(ExecutorBase):
37
# Profile the memory usage and initialize the cache.
38
self._init_cache()
39
47
- def _dispatch_worker(self):
48
- worker_module = DEVICE_TO_WORKER_MODULE_MAP[
49
- self.device_config.device_type]
50
- imported_worker = importlib.import_module(worker_module)
51
- Worker = imported_worker.Worker
52
- return Worker
53
-
40
def _init_worker(self):
41
# Lazy import the Worker to avoid importing torch.cuda/xformers
42
# before CUDA_VISIBLE_DEVICES is set in the Worker
57
- Worker = self._dispatch_worker()
43
+ from vllm.worker.worker import Worker
44
45
assert self.parallel_config.world_size == 1, (
46
"GPUExecutor only supports single GPU.")
@@ -73,7 +59,7 @@ class GPUExecutor(ExecutorBase):
59
kv_cache_dtype=self.cache_config.cache_dtype,
60
is_driver_worker=True,
61
)
76
- self.driver_worker.init_model()
62
+ self.driver_worker.init_device()
63
self.driver_worker.load_model()
64
65
def _init_cache(self) -> None:
vllm/executor/neuron_executor.py
new
+80
@@ -0,0 +1,80 @@
1
+from typing import Dict, List, Optional
2
+
3
+from vllm.lora.request import LoRARequest
4
+from vllm.config import (CacheConfig, DeviceConfig, ModelConfig,
5
+ ParallelConfig, SchedulerConfig, LoRAConfig)
6
+from vllm.executor.executor_base import ExecutorBase
7
+from vllm.logger import init_logger
8
+from vllm.sequence import SamplerOutput, SequenceGroupMetadata
9
+
10
+logger = init_logger(__name__)
11
+
12
+
13
+class NeuronExecutor(ExecutorBase):
14
+
15
+ def __init__(
16
+ self,
17
+ model_config: ModelConfig,
18
+ cache_config: CacheConfig,
19
+ parallel_config: ParallelConfig,
20
+ scheduler_config: SchedulerConfig,
21
+ device_config: DeviceConfig,
22
+ lora_config: Optional[LoRAConfig],
23
+ ) -> None:
24
+ self.model_config = model_config
25
+ self.cache_config = cache_config
26
+ assert lora_config is None, "LoRA is not supported for Neuron backend."
27
+ self.parallel_config = parallel_config
28
+ self.scheduler_config = scheduler_config
29
+ self.device_config = device_config
30
+
31
+ # Set the number of GPU blocks to be the same as the maximum number of
32
+ # sequences that can be processed in a single batch. This is equivalent
33
+ # to schedule without PagedAttention.
34
+ self.cache_config.num_gpu_blocks = self.scheduler_config.max_num_seqs
35
+ self.cache_config.num_cpu_blocks = 0
36
+
37
+ # Instantiate the worker and load the model to the device.
38
+ self._init_worker()
39
+
40
+ def _init_worker(self):
41
+ from vllm.worker.neuron_worker import NeuronWorker
42
+
43
+ self.driver_worker = NeuronWorker(
44
+ self.model_config,
45
+ self.parallel_config,
46
+ self.scheduler_config,
47
+ self.device_config,
48
+ )
49
+ self.driver_worker.init_device()
50
+ self.driver_worker.load_model()
51
+
52
+ def execute_model(self,
53
+ seq_group_metadata_list: List[SequenceGroupMetadata],
54
+ blocks_to_swap_in: Dict[int, int],
55
+ blocks_to_swap_out: Dict[int, int],
56
+ blocks_to_copy: Dict[int, List[int]]) -> SamplerOutput:
57
+ assert (blocks_to_swap_in == {} and blocks_to_swap_out == {}
58
+ and blocks_to_copy == {}), (
59
+ "Cache operations are not supported for Neuron backend.")
60
+
61
+ output = self.driver_worker.execute_model(
62
+ seq_group_metadata_list=seq_group_metadata_list)
63
+ return output
64
+
65
+ def add_lora(self, lora_request: LoRARequest) -> bool:
66
+ raise NotImplementedError(
67
+ "LoRA is not implemented for neuron backend.")
68
+
69
+ def remove_lora(self, lora_id: int) -> bool:
70
+ raise NotImplementedError(
71
+ "LoRA is not implemented for neuron backend.")
72
+
73
+ def list_loras(self) -> List[int]:
74
+ raise NotImplementedError(
75
+ "LoRA is not implemented for neuron backend.")
76
+
77
+ def check_health(self) -> None:
78
+ # NeuronExecutor will always be healthy as long as
79
+ # it's running.
80
+ return
vllm/executor/ray_gpu_executor.py
+2
-16
@@ -3,7 +3,6 @@ import copy
3
from collections import defaultdict
4
import os
5
import pickle
6
-import importlib
6
from typing import TYPE_CHECKING, Any, Dict, List, Optional
7
8
from vllm.config import (CacheConfig, DeviceConfig, ModelConfig,
@@ -25,12 +24,6 @@ if TYPE_CHECKING:
24
25
logger = init_logger(__name__)
26
28
-# A map between the device type (in device config) to its worker module.
29
-DEVICE_TO_WORKER_MODULE_MAP = {
30
- "cuda": "vllm.worker.worker",
31
- "neuron": "vllm.worker.neuron_worker",
32
-}
33
-
27
# If the env var is set, it uses the Ray's compiled DAG API
28
# which optimizes the control plane overhead.
29
# Run vLLM with VLLM_USE_RAY_COMPILED_DAG=1 to enable it.
@@ -73,13 +66,6 @@ class RayGPUExecutor(ExecutorBase):
66
if USE_RAY_COMPILED_DAG:
67
self.forward_dag = self._compiled_ray_dag()
68
76
- def _dispatch_worker(self):
77
- worker_module = DEVICE_TO_WORKER_MODULE_MAP[
78
- self.device_config.device_type]
79
- imported_worker = importlib.import_module(worker_module)
80
- Worker = imported_worker.Worker
81
- return Worker
82
-
69
def _init_workers_ray(self, placement_group: "PlacementGroup",
70
**ray_remote_kwargs):
71
if self.parallel_config.tensor_parallel_size == 1:
@@ -155,7 +141,7 @@ class RayGPUExecutor(ExecutorBase):
141
142
# Lazy import the Worker to avoid importing torch.cuda/xformers
143
# before CUDA_VISIBLE_DEVICES is set in the Worker
158
- Worker = self._dispatch_worker()
144
+ from vllm.worker.worker import Worker
145
146
model_config = copy.deepcopy(self.model_config)
147
parallel_config = copy.deepcopy(self.parallel_config)
@@ -201,7 +187,7 @@ class RayGPUExecutor(ExecutorBase):
187
188
# FIXME(woosuk): We are not properly initializing cupy NCCL when
189
# we have multiple nodes.
204
- self._run_workers("init_model",
190
+ self._run_workers("init_device",
191
cupy_port=get_open_port()
192
if not model_config.enforce_eager else None)
193
self._run_workers(
vllm/lora/layers.py
+2
-2
@@ -799,8 +799,8 @@ class LogitsProcessorWithLoRA(BaseLayerWithLoRA):
799
self.device = device
800
801
@property
802
- def logits_as_hidden_states(self):
803
- return self.base_layer.logits_as_hidden_states
802
+ def logits_as_input(self):
803
+ return self.base_layer.logits_as_input
804
805
@property
806
def vocab_size(self):
vllm/lora/lora.py
+2
-2
@@ -1,7 +1,7 @@
1
from typing import List, Optional
2
3
import torch
4
-from vllm.utils import in_wsl
4
+from vllm.utils import is_pin_memory_available
5
6
7
class LoRALayerWeights:
@@ -64,7 +64,7 @@ class LoRALayerWeights:
64
dtype: torch.dtype,
65
device: torch.device,
66
embeddings_tensor_dim: Optional[int] = None) -> "LoRALayerWeights":
67
- pin_memory = str(device) == "cpu" and not in_wsl()
67
+ pin_memory = str(device) == "cpu" and is_pin_memory_available()
68
lora_a = torch.zeros([input_dim, rank],
69
dtype=dtype,
70
device=device,
vllm/lora/models.py
+2
-2
@@ -11,7 +11,7 @@ import torch
11
from torch import nn
12
13
from vllm.config import LoRAConfig
14
-from vllm.utils import LRUCache, in_wsl
14
+from vllm.utils import LRUCache, is_pin_memory_available
15
16
from vllm.lora.layers import (BaseLayerWithLoRA, LoRAMapping, from_layer,
17
from_layer_logits_processor)
@@ -143,7 +143,7 @@ class LoRAModel:
143
embedding_padding_modules: Optional[List[str]] = None,
144
) -> "LoRAModel":
145
"""Create a LoRAModel from a dictionary of tensors."""
146
- pin_memory = str(device) == "cpu" and not in_wsl()
146
+ pin_memory = str(device) == "cpu" and is_pin_memory_available()
147
loras: Dict[str, LoRALayerWeights] = {}
148
for tensor_name, tensor in tensors.items():
149
module_name, is_lora_a = parse_fine_tuned_lora_name(tensor_name)
vllm/model_executor/__init__.py
+1
-2
@@ -1,10 +1,9 @@
1
from vllm.model_executor.input_metadata import InputMetadata
2
from vllm.model_executor.sampling_metadata import SamplingMetadata
3
-from vllm.model_executor.utils import set_random_seed, get_model
3
+from vllm.model_executor.utils import set_random_seed
4
5
__all__ = [
6
"InputMetadata",
7
- "get_model",
7
"SamplingMetadata",
8
"set_random_seed",
9
]
vllm/model_executor/input_metadata.py
+4
-3
@@ -1,8 +1,9 @@
1
from dataclasses import dataclass, fields
2
-from typing import Optional, List, Any, Dict
2
+from typing import TYPE_CHECKING, Optional, List, Any, Dict
3
4
import torch
5
-from xformers.ops.fmha.attn_bias import AttentionBias
5
+if TYPE_CHECKING:
6
+ from xformers.ops.fmha.attn_bias import AttentionBias
7
8
9
@dataclass
@@ -82,7 +83,7 @@ class InputMetadata:
83
# when alibi slopes is used. It is because of the limitation
84
# from xformer API.
85
# will not appear in the __repr__ and __init__
85
- self.attn_bias: Optional[List[AttentionBias]] = None
86
+ self.attn_bias: Optional[List["AttentionBias"]] = None
87
88
# Cuda graph is only used for decoding now.
89
if self.use_cuda_graph:
vllm/model_executor/layers/logits_processor.py
+5
-6
@@ -4,8 +4,6 @@ from typing import Optional
4
import torch
5
import torch.nn as nn
6
7
-from vllm.utils import is_neuron
8
-
7
from vllm.model_executor.parallel_utils.communication_op import (
8
tensor_model_parallel_gather)
9
from vllm.model_executor.sampling_metadata import SamplingMetadata
@@ -23,7 +21,8 @@ class LogitsProcessor(nn.Module):
21
def __init__(self,
22
vocab_size: int,
23
org_vocab_size: Optional[int] = None,
26
- scale: Optional[float] = 1.0) -> None:
24
+ scale: Optional[float] = 1.0,
25
+ logits_as_input: bool = False) -> None:
26
"""
27
Args:
28
scale: A scaling factor to apply to the logits.
@@ -31,8 +30,8 @@ class LogitsProcessor(nn.Module):
30
super().__init__()
31
self.scale = scale
32
self.vocab_size = vocab_size
34
- # Transformers-neuronx generate outputs as logits directly.
35
- self.logits_as_hidden_states = is_neuron()
33
+ # Whether the input is logits (default is hidden states).
34
+ self.logits_as_input = logits_as_input
35
# original vocabulary size (without LoRA).
36
self.org_vocab_size = org_vocab_size or vocab_size
37
@@ -43,7 +42,7 @@ class LogitsProcessor(nn.Module):
42
sampling_metadata: SamplingMetadata,
43
embedding_bias: Optional[torch.Tensor] = None,
44
) -> torch.Tensor:
46
- if self.logits_as_hidden_states:
45
+ if self.logits_as_input:
46
logits = hidden_states
47
else:
48
hidden_states = _prune_hidden_states(hidden_states,
vllm/model_executor/layers/sampler.py
+1
-1
@@ -4,13 +4,13 @@ from typing import Dict, List, Optional, Tuple
4
import torch
5
import torch.nn as nn
6
7
+from vllm.model_executor.layers.ops.sample import sample as sample_triton
8
from vllm.model_executor.sampling_metadata import (SamplingMetadata,
9
SamplingTensors)
10
from vllm.sampling_params import SamplingParams, SamplingType
11
from vllm.sequence import (Logprob, PromptLogprobs, SampleLogprobs,
12
SamplerOutput, SequenceData, SequenceGroupOutput,
13
SequenceOutput)
13
-from vllm.model_executor.layers.ops.sample import (sample as sample_triton)
14
15
16
class Sampler(nn.Module):
vllm/model_executor/models/__init__.py
+1
-14
@@ -4,7 +4,7 @@ from typing import List, Optional, Type
4
import torch.nn as nn
5
6
from vllm.logger import init_logger
7
-from vllm.utils import is_hip, is_neuron
7
+from vllm.utils import is_hip
8
9
logger = init_logger(__name__)
10
@@ -63,12 +63,6 @@ _ROCM_PARTIALLY_SUPPORTED_MODELS = {
63
"Sliding window attention is not yet supported in ROCm's flash attention",
64
}
65
66
-# Models supported by Neuron.
67
-_NEURON_SUPPORTED_MODELS = {
68
- "LlamaForCausalLM": "neuron.llama",
69
- "MistralForCausalLM": "neuron.mistral"
70
-}
71
-
66
67
class ModelRegistry:
68
@@ -85,15 +79,8 @@ class ModelRegistry:
79
logger.warning(
80
f"Model architecture {model_arch} is partially supported "
81
"by ROCm: " + _ROCM_PARTIALLY_SUPPORTED_MODELS[model_arch])
88
- elif is_neuron():
89
- if model_arch not in _NEURON_SUPPORTED_MODELS:
90
- raise ValueError(
91
- f"Model architecture {model_arch} is not supported by "
92
- "Neuron for now.")
82
83
module_name, model_cls_name = _MODELS[model_arch]
95
- if is_neuron():
96
- module_name = _NEURON_SUPPORTED_MODELS[model_arch]
84
module = importlib.import_module(
85
f"vllm.model_executor.models.{module_name}")
86
return getattr(module, model_cls_name, None)
vllm/model_executor/models/neuron/llama.py
deleted
-86
@@ -1,86 +0,0 @@
1
-"""Inference-only LLaMA model compatible with HuggingFace weights."""
2
-import os
3
-from typing import List, Optional, Tuple
4
-
5
-import torch
6
-from torch import nn
7
-from transformers import LlamaConfig
8
-
9
-from vllm.model_executor.input_metadata import InputMetadata
10
-from vllm.model_executor.layers.logits_processor import LogitsProcessor
11
-from vllm.model_executor.layers.sampler import Sampler
12
-from vllm.model_executor.sampling_metadata import SamplingMetadata
13
-from vllm.sequence import SamplerOutput
14
-
15
-KVCache = Tuple[torch.Tensor, torch.Tensor]
16
-
17
-
18
-class LlamaForCausalLM(nn.Module):
19
-
20
- def __init__(
21
- self,
22
- config: LlamaConfig,
23
- linear_method=None,
24
- ) -> None:
25
- super().__init__()
26
- self.config = config
27
- self.linear_method = linear_method
28
- self.model = None
29
- self.logits_processor = LogitsProcessor(config.vocab_size)
30
- self.sampler = Sampler()
31
-
32
- def forward(
33
- self,
34
- input_ids: torch.Tensor,
35
- positions: torch.Tensor,
36
- kv_caches: List[KVCache],
37
- input_metadata: InputMetadata,
38
- ) -> torch.Tensor:
39
- with torch.inference_mode():
40
- block_size = self.model.context_buckets[-1]
41
- if input_metadata.is_prompt:
42
- seq_ids = input_metadata.slot_mapping[:, 0] // block_size
43
- else:
44
- seq_ids = input_metadata.block_tables
45
- logits = self.model(input_ids,
46
- cache_ids=positions,
47
- start_ids=seq_ids.flatten())
48
- return logits
49
-
50
- def compute_logits(self, hidden_states: torch.Tensor,
51
- sampling_metadata: SamplingMetadata) -> torch.Tensor:
52
- logits = self.logits_processor(self.model.chkpt_model.lm_head,
53
- hidden_states, sampling_metadata)
54
- return logits
55
-
56
- def sample(
57
- self,
58
- logits: torch.Tensor,
59
- sampling_metadata: SamplingMetadata,
60
- ) -> Optional[SamplerOutput]:
61
- next_tokens = self.sampler(logits, sampling_metadata)
62
- return next_tokens
63
-
64
- def load_weights(self,
65
- model_name_or_path: str,
66
- cache_dir: Optional[str] = None,
67
- load_format: str = "auto",
68
- revision: Optional[str] = None,
69
- **kwargs):
70
- from transformers_neuronx.llama.model import LlamaForSampling
71
-
72
- split_model_dir = f"{model_name_or_path}-split"
73
- if os.path.isdir(os.path.join(model_name_or_path,
74
- "pytorch_model.bin")):
75
- split_model_dir = model_name_or_path
76
- elif not os.path.exists(f"{model_name_or_path}-split"):
77
- from transformers.models.llama import LlamaForCausalLM
78
- from transformers_neuronx.module import save_pretrained_split
79
-
80
- hf_model = LlamaForCausalLM.from_pretrained(model_name_or_path,
81
- low_cpu_mem_usage=True)
82
- save_pretrained_split(hf_model, f"{model_name_or_path}-split")
83
-
84
- self.model = LlamaForSampling.from_pretrained(split_model_dir,
85
- **kwargs)
86
- self.model.to_neuron()
vllm/model_executor/models/neuron/mistral.py
deleted
-89
@@ -1,89 +0,0 @@
1
-"""Inference-only Mistral model compatible with HuggingFace weights."""
2
-from typing import List, Optional, Tuple
3
-
4
-import torch
5
-from torch import nn
6
-from transformers import MistralConfig
7
-
8
-from vllm.model_executor.input_metadata import InputMetadata
9
-from vllm.model_executor.layers.logits_processor import LogitsProcessor
10
-from vllm.model_executor.layers.sampler import Sampler
11
-from vllm.model_executor.sampling_metadata import SamplingMetadata
12
-from vllm.sequence import SamplerOutput
13
-import os
14
-
15
-KVCache = Tuple[torch.Tensor, torch.Tensor]
16
-
17
-
18
-class MistralForCausalLM(nn.Module):
19
-
20
- def __init__(
21
- self,
22
- config: MistralConfig,
23
- linear_method=None,
24
- ) -> None:
25
- super().__init__()
26
- self.config = config
27
- self.linear_method = linear_method
28
- self.model = None
29
- self.lm_head = None
30
- self.logits_processor = LogitsProcessor(config.vocab_size)
31
- self.sampler = Sampler()
32
-
33
- def forward(
34
- self,
35
- input_ids: torch.Tensor,
36
- positions: torch.Tensor,
37
- kv_caches: List[KVCache],
38
- input_metadata: InputMetadata,
39
- ) -> SamplerOutput:
40
- with torch.inference_mode():
41
- seq_ids = []
42
- block_size = self.model.context_buckets[-1]
43
- if input_metadata.is_prompt:
44
- seq_ids = input_metadata.slot_mapping[:, 0] // block_size
45
- else:
46
- seq_ids = input_metadata.block_tables
47
-
48
- logits = self.model(input_ids,
49
- cache_ids=positions,
50
- start_ids=seq_ids)
51
- return logits
52
-
53
- def compute_logits(self, hidden_states: torch.Tensor,
54
- sampling_metadata: SamplingMetadata) -> torch.Tensor:
55
- logits = self.logits_processor(self.model.chkpt_model.lm_head,
56
- hidden_states, sampling_metadata)
57
- return logits
58
-
59
- def sample(
60
- self,
61
- logits: torch.Tensor,
62
- sampling_metadata: SamplingMetadata,
63
- ) -> Optional[SamplerOutput]:
64
- next_tokens = self.sampler(logits, sampling_metadata)
65
- return next_tokens
66
-
67
- def load_weights(self,
68
- model_name_or_path: str,
69
- cache_dir: Optional[str] = None,
70
- load_format: str = "auto",
71
- revision: Optional[str] = None,
72
- **kwargs):
73
- from transformers_neuronx.mistral.model import MistralForSampling
74
-
75
- split_model_dir = f"{model_name_or_path}-split"
76
- if os.path.isdir(os.path.join(model_name_or_path,
77
- "pytorch_model.bin")):
78
- split_model_dir = model_name_or_path
79
- elif not os.path.exists(f"{model_name_or_path}-split"):
80
- from transformers import MistralForCausalLM
81
- from transformers_neuronx.module import save_pretrained_split
82
-
83
- hf_model = MistralForCausalLM.from_pretrained(
84
- model_name_or_path, low_cpu_mem_usage=True)
85
- save_pretrained_split(hf_model, f"{model_name_or_path}-split")
86
-
87
- self.model = MistralForSampling.from_pretrained(
88
- split_model_dir, **kwargs)
89
- self.model.to_neuron()
vllm/model_executor/neuron_model_loader.py
+89
-22
@@ -1,12 +1,18 @@
1
-"""Utilities for selecting and loading models."""
2
-from typing import Type
1
+"""Utilities for selecting and loading neuron models."""
2
+import importlib
3
+import os
4
+from typing import Optional, Type
5
6
import torch
7
import torch.nn as nn
8
+import transformers
9
from transformers import PretrainedConfig
10
8
-from vllm.config import ModelConfig, DeviceConfig
9
-from vllm.model_executor.models import ModelRegistry
11
+from vllm.config import ModelConfig, ParallelConfig, SchedulerConfig
12
+from vllm.model_executor.layers.logits_processor import LogitsProcessor
13
+from vllm.model_executor.layers.sampler import Sampler
14
+from vllm.model_executor.sampling_metadata import SamplingMetadata
15
+from vllm.sequence import SamplerOutput
16
17
TORCH_DTYPE_TO_NEURON_AMP = {
18
"auto": "f32",
@@ -20,31 +26,95 @@ TORCH_DTYPE_TO_NEURON_AMP = {
26
torch.float32: "f32",
27
}
28
29
+# Models supported by Neuron.
30
+_NEURON_SUPPORTED_MODELS = {
31
+ "LlamaForCausalLM": ("transformers_neuronx.llama.model",
32
+ "LlamaForSampling", "LlamaForCausalLM"),
33
+ "MistralForCausalLM": ("transformers_neuronx.mistral.model",
34
+ "MistralForSampling", "MistralForCausalLM")
35
+}
36
+
37
+
38
+class NeuronCasualLM(nn.Module):
39
+
40
+ def __init__(
41
+ self,
42
+ config: PretrainedConfig,
43
+ ) -> None:
44
+ super().__init__()
45
+ self.config = config
46
+ self.model = None
47
+ self.logits_processor = LogitsProcessor(config.vocab_size,
48
+ logits_as_input=True)
49
+ self.sampler = Sampler()
50
+
51
+ def forward(
52
+ self,
53
+ input_ids: torch.Tensor,
54
+ positions: torch.Tensor,
55
+ input_block_ids: torch.Tensor,
56
+ ) -> torch.Tensor:
57
+ logits = self.model(input_ids,
58
+ cache_ids=positions,
59
+ start_ids=input_block_ids)
60
+ return logits
61
+
62
+ def compute_logits(self, hidden_states: torch.Tensor,
63
+ sampling_metadata: SamplingMetadata) -> torch.Tensor:
64
+ logits = self.logits_processor(None, hidden_states, sampling_metadata)
65
+ return logits
66
+
67
+ def sample(
68
+ self,
69
+ logits: torch.Tensor,
70
+ sampling_metadata: SamplingMetadata,
71
+ ) -> Optional[SamplerOutput]:
72
+ next_tokens = self.sampler(logits, sampling_metadata)
73
+ return next_tokens
74
+
75
+ def load_weights(self, model_name_or_path: str, **kwargs):
76
+ arch = _get_model_architecture(self.config)
77
+ neuronx_module_path, neuronx_model_cls, hf_model_cls = (
78
+ _NEURON_SUPPORTED_MODELS[arch])
79
+ neuronx_module = importlib.import_module(neuronx_module_path)
80
+ neuronx_model_cls = getattr(neuronx_module, neuronx_model_cls)
81
+
82
+ split_model_dir = f"{model_name_or_path}-split"
83
+ if os.path.isdir(os.path.join(model_name_or_path,
84
+ "pytorch_model.bin")):
85
+ split_model_dir = model_name_or_path
86
+ elif not os.path.exists(f"{model_name_or_path}-split"):
87
+ hf_model_cls = getattr(transformers, hf_model_cls)
88
+ from transformers_neuronx.module import save_pretrained_split
89
+
90
+ hf_model = hf_model_cls.from_pretrained(model_name_or_path,
91
+ low_cpu_mem_usage=True)
92
+ save_pretrained_split(hf_model, f"{model_name_or_path}-split")
93
+
94
+ self.model = neuronx_model_cls.from_pretrained(split_model_dir,
95
+ **kwargs)
96
+ self.model.to_neuron()
97
+
98
99
def _get_model_architecture(config: PretrainedConfig) -> Type[nn.Module]:
100
architectures = getattr(config, "architectures", [])
101
for arch in architectures:
27
- model_cls = ModelRegistry.load_model_cls(arch)
28
- if model_cls is not None:
29
- return model_cls
102
+ if arch in _NEURON_SUPPORTED_MODELS:
103
+ return arch
104
raise ValueError(
31
- f"Model architectures {architectures} are not supported for now. "
32
- f"Supported architectures: {ModelRegistry.get_supported_archs()}")
105
+ f"Model architectures {architectures} are not supported on Neuron "
106
+ f"for now. Supported architectures: "
107
+ f"{list(_NEURON_SUPPORTED_MODELS.keys())}")
108
109
35
-def get_model(model_config: ModelConfig, device_config: DeviceConfig,
36
- **kwargs) -> nn.Module:
110
+def get_neuron_model(model_config: ModelConfig,
111
+ parallel_config: ParallelConfig,
112
+ scheduler_config: SchedulerConfig) -> nn.Module:
113
from transformers_neuronx.config import (NeuronConfig,
114
ContinuousBatchingConfig)
115
40
- parallel_config = kwargs.get("parallel_config")
41
- scheduler_config = kwargs.get("scheduler_config")
42
-
43
- model_class = _get_model_architecture(model_config.hf_config)
44
- linear_method = None
45
-
116
# Create a model instance.
47
- model = model_class(model_config.hf_config, linear_method)
117
+ model = NeuronCasualLM(model_config.hf_config)
118
119
continuous_batching_config = ContinuousBatchingConfig(
120
batch_size_for_shared_caches=scheduler_config.max_num_seqs)
@@ -54,10 +124,7 @@ def get_model(model_config: ModelConfig, device_config: DeviceConfig,
124
# Load the weights from the cached or downloaded files.
125
model.load_weights(
126
model_config.model,
57
- model_config.download_dir,
58
- model_config.load_format,
59
- model_config.revision,
60
- tp_degree=parallel_config.neuron_tp_degree,
127
+ tp_degree=parallel_config.tensor_parallel_size,
128
amp=TORCH_DTYPE_TO_NEURON_AMP[model_config.dtype],
129
neuron_config=neuron_config,
130
context_length_estimate=[scheduler_config.max_model_len],
vllm/model_executor/sampling_metadata.py
+4
-4
@@ -4,11 +4,11 @@ from typing import Dict, List, Optional, Tuple
4
import torch
5
import random
6
7
-from vllm.sampling_params import SamplingParams, SamplingType
8
-from vllm.sequence import SequenceData
9
-from vllm.utils import in_wsl, is_neuron
7
from vllm.model_executor.layers.ops.sample import (
8
get_num_triton_sampler_splits)
9
+from vllm.sampling_params import SamplingParams, SamplingType
10
+from vllm.sequence import SequenceData
11
+from vllm.utils import is_pin_memory_available
12
13
_SAMPLING_EPS = 1e-5
14
_SEED_0_REPLACEMENT = 3403598558
@@ -213,7 +213,7 @@ class SamplingTensors:
213
dtype: torch.dtype) -> "SamplingTensors":
214
# Note that the performance will be very bad without
215
# pinned memory.
216
- pin_memory = not in_wsl() and not is_neuron()
216
+ pin_memory = is_pin_memory_available()
217
prompt_max_len = max(len(tokens) for tokens in prompt_tokens)
218
prompt_padded_tokens = [
219
tokens + [vocab_size] * (prompt_max_len - len(tokens))
vllm/model_executor/utils.py
-17
@@ -1,18 +1,10 @@
1
"""Utils for model executor."""
2
import random
3
-import importlib
3
from typing import Any, Dict, Optional
4
5
import numpy as np
6
import torch
7
9
-from vllm.config import DeviceConfig, ModelConfig
10
-
11
-DEVICE_TO_MODEL_LOADER_MAP = {
12
- "cuda": "model_loader",
13
- "neuron": "neuron_model_loader",
14
-}
15
-
8
9
def set_random_seed(seed: int) -> None:
10
random.seed(seed)
@@ -41,12 +33,3 @@ def set_weight_attrs(
33
assert not hasattr(
34
weight, key), (f"Overwriting existing tensor attribute: {key}")
35
setattr(weight, key, value)
44
-
45
-
46
-def get_model(model_config: ModelConfig, device_config: DeviceConfig,
47
- **kwargs) -> torch.nn.Module:
48
- model_loader_module = DEVICE_TO_MODEL_LOADER_MAP[device_config.device_type]
49
- imported_model_loader = importlib.import_module(
50
- f"vllm.model_executor.{model_loader_module}")
51
- get_model_fn = imported_model_loader.get_model
52
- return get_model_fn(model_config, device_config, **kwargs)
vllm/spec_decode/metrics.py
+2
-2
@@ -2,7 +2,7 @@ import torch
2
from dataclasses import dataclass
3
from vllm.model_executor.layers.rejection_sampler import RejectionSampler
4
from typing import Optional
5
-from vllm.utils import in_wsl
5
+from vllm.utils import is_pin_memory_available
6
import time
7
from typing import Callable
8
@@ -63,7 +63,7 @@ class AsyncMetricsCollector:
63
64
self._in_flight_copy: Optional[torch.cuda.Event] = None
65
66
- pin_memory = not in_wsl()
66
+ pin_memory = is_pin_memory_available()
67
self._aggregate_num_accepted_tokens = torch.tensor(
68
0, dtype=torch.long, device="cpu", pin_memory=pin_memory)
69
self._aggregate_num_emitted_tokens = torch.tensor(
vllm/spec_decode/multi_step_worker.py
+2
-2
@@ -27,8 +27,8 @@ class MultiStepWorker(Worker):
27
28
self._proposer: Optional[DraftModelTop1Proposer] = None
29
30
- def init_model(self):
31
- super().init_model()
30
+ def init_device(self):
31
+ super().init_device()
32
33
self._proposer = DraftModelTop1Proposer(
34
self,
vllm/spec_decode/spec_decode_worker.py
+3
-3
@@ -79,13 +79,13 @@ class SpecDecodeWorker:
79
80
self.scorer: SpeculativeScorer = None
81
82
- def init_model(self) -> None:
82
+ def init_device(self) -> None:
83
"""Initialize both scorer and proposer models.
84
"""
85
# The scorer worker model is initialized first in case the proposer
86
# model has a smaller TP degree than the target worker.
87
- self.scorer_worker.init_model()
88
- self.proposer_worker.init_model()
87
+ self.scorer_worker.init_device()
88
+ self.proposer_worker.init_device()
89
90
self._metrics.init_gpu_tensors(self.rank)
91
self.rejection_sampler.init_gpu_tensors(self.rank)
vllm/utils.py
+62
-1
@@ -338,7 +338,27 @@ def create_kv_caches_with_random(
338
return key_caches, value_caches
339
340
341
-class measure_cuda_memory:
341
+@lru_cache
342
+def print_warning_once(msg: str) -> None:
343
+ logger.warning(msg)
344
+
345
+
346
+@lru_cache(maxsize=None)
347
+def is_pin_memory_available() -> bool:
348
+
349
+ if in_wsl():
350
+ # Pinning memory in WSL is not supported.
351
+ # https://docs.nvidia.com/cuda/wsl-user-guide/index.html#known-limitations-for-linux-cuda-applications
352
+ print_warning_once("Using 'pin_memory=False' as WSL is detected. "
353
+ "This may slow down the performance.")
354
+ return False
355
+ elif is_neuron():
356
+ print_warning_once("Pin memory is not supported on Neuron.")
357
+ return False
358
+ return True
359
+
360
+
361
+class CudaMemoryProfiler:
362
363
def __init__(self, device=None):
364
self.device = device
@@ -360,3 +380,44 @@ class measure_cuda_memory:
380
381
# Force garbage collection
382
gc.collect()
383
+
384
+
385
+def pad_to_max_length(x: List[int], max_len: int, pad: int) -> List[int]:
386
+ assert len(x) <= max_len
387
+ return x + [pad] * (max_len - len(x))
388
+
389
+
390
+def make_tensor_with_pad(
391
+ x: List[List[int]],
392
+ max_len: int,
393
+ pad: int,
394
+ dtype: torch.dtype,
395
+ device: Optional[Union[str, torch.device]],
396
+) -> torch.Tensor:
397
+ """Make a padded tensor of a 2D inputs.
398
+
399
+ The padding is applied to the end of each inner list until it reaches
400
+ `max_len`.
401
+ """
402
+ padded_x = [pad_to_max_length(x_i, max_len, pad) for x_i in x]
403
+ return torch.tensor(padded_x, dtype=dtype, device=device)
404
+
405
+
406
+def async_tensor_h2d(
407
+ data: list,
408
+ dtype: torch.dtype,
409
+ target_device: Union[str, torch.device],
410
+ pin_memory: bool,
411
+) -> torch.Tensor:
412
+ """Asynchronously create a tensor and copy it from host to device."""
413
+ t = torch.tensor(data, dtype=dtype, pin_memory=pin_memory, device="cpu")
414
+ return t.to(device=target_device, non_blocking=True)
415
+
416
+
417
+def maybe_expand_dim(tensor: torch.Tensor,
418
+ target_dims: int,
419
+ size: int = 1) -> torch.Tensor:
420
+ """Expand the tensor to the target_dims."""
421
+ if tensor.ndim < target_dims:
422
+ tensor = tensor.view(-1, *([size] * (target_dims - tensor.ndim)))
423
+ return tensor
vllm/worker/cache_engine.py
+2
-11
@@ -5,7 +5,7 @@ import torch
5
6
from vllm.config import CacheConfig, ModelConfig, ParallelConfig
7
from vllm.logger import init_logger
8
-from vllm.utils import in_wsl, is_neuron, STR_DTYPE_TO_TORCH_DTYPE
8
+from vllm.utils import is_pin_memory_available, STR_DTYPE_TO_TORCH_DTYPE
9
10
logger = init_logger(__name__)
11
@@ -38,10 +38,6 @@ class CacheEngine:
38
self.num_gpu_blocks = cache_config.num_gpu_blocks
39
self.num_cpu_blocks = cache_config.num_cpu_blocks
40
41
- # Skip initializing KV cache for Neuron backend.
42
- if is_neuron():
43
- return
44
-
41
if cache_config.cache_dtype == "auto":
42
self.dtype = model_config.dtype
43
else:
@@ -90,12 +86,7 @@ class CacheEngine:
86
cpu_cache: List[KVCache] = []
87
key_block_shape = self.get_key_block_shape()
88
value_block_shape = self.get_value_block_shape()
93
- pin_memory = not in_wsl()
94
- if not pin_memory:
95
- # Pinning memory in WSL is not supported.
96
- # https://docs.nvidia.com/cuda/wsl-user-guide/index.html#known-limitations-for-linux-cuda-applications
97
- logger.warning("Using 'pin_memory=False' as WSL is detected. "
98
- "This may slow down the performance.")
89
+ pin_memory = is_pin_memory_available()
90
for _ in range(self.num_layers):
91
key_blocks = torch.empty(
92
size=(self.num_cpu_blocks, *key_block_shape),
vllm/worker/model_runner.py
+20
-62
@@ -1,6 +1,6 @@
1
import contextlib
2
import time
3
-from typing import Dict, List, Optional, Tuple, Set, Union
3
+from typing import Dict, List, Optional, Tuple, Set
4
5
import numpy as np
6
import torch
@@ -9,7 +9,8 @@ import torch.nn as nn
9
from vllm.config import (DeviceConfig, ModelConfig, LoRAConfig, ParallelConfig,
10
SchedulerConfig)
11
from vllm.logger import init_logger
12
-from vllm.model_executor import get_model, InputMetadata, SamplingMetadata
12
+from vllm.model_executor import InputMetadata, SamplingMetadata
13
+from vllm.model_executor.model_loader import get_model
14
from vllm.model_executor.parallel_utils import cupy_utils
15
from vllm.model_executor.parallel_utils.communication_op import (
16
broadcast_tensor_dict)
@@ -21,7 +22,9 @@ from vllm.sequence import SamplerOutput, SequenceData, SequenceGroupMetadata
22
from vllm.lora.worker_manager import LRUCacheWorkerLoRAManager
23
from vllm.lora.layers import LoRAMapping
24
from vllm.lora.request import LoRARequest
24
-from vllm.utils import in_wsl, measure_cuda_memory
25
+from vllm.utils import (async_tensor_h2d, CudaMemoryProfiler,
26
+ is_pin_memory_available, make_tensor_with_pad,
27
+ maybe_expand_dim)
28
29
logger = init_logger(__name__)
30
@@ -79,16 +82,11 @@ class ModelRunner:
82
# The shape of the cached block table will be
83
# (max batch size to capture, max context len to capture / block size).
84
self.graph_block_tables = None # Set after initial profiling.
82
- # cache in_wsl result
83
- self.in_wsl = in_wsl()
85
+ self.pin_memory = is_pin_memory_available()
86
self.kv_cache_dtype = kv_cache_dtype
87
86
- # Set enforce_eager to True for Neuron backend, to avoid capturing graph
87
- if self.device_config.is_neuron:
88
- self.model_config.enforce_eager = True
89
-
88
def load_model(self) -> None:
91
- with measure_cuda_memory() as m:
89
+ with CudaMemoryProfiler() as m:
90
self.model = get_model(self.model_config,
91
self.device_config,
92
lora_config=self.lora_config,
@@ -238,7 +236,7 @@ class ModelRunner:
236
device=self.device)
237
# Prepare prefix block tables
238
max_prompt_block_table_len = max(len(t) for t in prefix_block_tables)
241
- block_tables = _make_tensor_with_pad(
239
+ block_tables = make_tensor_with_pad(
240
prefix_block_tables,
241
max_len=max_prompt_block_table_len,
242
pad=0,
@@ -395,7 +393,7 @@ class ModelRunner:
393
else:
394
max_block_table_len = max(
395
len(block_table) for block_table in block_tables)
398
- block_tables = _make_tensor_with_pad(
396
+ block_tables = make_tensor_with_pad(
397
block_tables,
398
max_len=max_block_table_len,
399
pad=0,
@@ -436,7 +434,6 @@ class ModelRunner:
434
categorized_sample_indices = {t: [] for t in SamplingType}
435
categorized_sample_indices_start_idx = 0
436
categorized_sampled_token_indices_start_idx = 0
439
- pin_memory = not self.in_wsl and not self.device_config.is_neuron
437
438
for i, seq_group_metadata in enumerate(seq_group_metadata_list):
439
seq_ids = list(seq_group_metadata.seq_data.keys())
@@ -469,7 +466,7 @@ class ModelRunner:
466
467
if sampling_params.seed is not None:
468
seq_group_metadata.state.generator = torch.Generator(
472
- device="cuda").manual_seed(sampling_params.seed)
469
+ device=self.device).manual_seed(sampling_params.seed)
470
else:
471
num_seqs = len(seq_ids)
472
selected_token_indices.extend(
@@ -494,17 +491,17 @@ class ModelRunner:
491
if sampling_params.seed is not None:
492
generators.append(seq_group_metadata.state.generator)
493
497
- selected_token_indices = _async_h2d(selected_token_indices,
498
- dtype=torch.long,
499
- target_device=self.device,
500
- pin_memory=not self.in_wsl)
494
+ selected_token_indices = async_tensor_h2d(selected_token_indices,
495
+ dtype=torch.long,
496
+ target_device=self.device,
497
+ pin_memory=self.pin_memory)
498
499
categorized_sample_indices = {
503
- t: _maybe_expand_dim(
504
- _async_h2d(seq_ids,
505
- dtype=torch.int,
506
- target_device=self.device,
507
- pin_memory=pin_memory), 2, 2)
500
+ t: maybe_expand_dim(
501
+ async_tensor_h2d(seq_ids,
502
+ dtype=torch.int,
503
+ target_device=self.device,
504
+ pin_memory=self.pin_memory), 2, 2)
505
for t, seq_ids in categorized_sample_indices.items()
506
}
507
@@ -910,27 +907,6 @@ def _maybe_cupy_nccl():
907
yield
908
909
913
-def _pad_to_max(x: List[int], max_len: int, pad: int) -> List[int]:
914
- assert len(x) <= max_len
915
- return x + [pad] * (max_len - len(x))
916
-
917
-
918
-def _make_tensor_with_pad(
919
- x: List[List[int]],
920
- max_len: int,
921
- pad: int,
922
- dtype: torch.dtype,
923
- device: Optional[Union[str, torch.device]],
924
-) -> torch.Tensor:
925
- """Make a padded tensor of a 2D inputs.
926
-
927
- The padding is applied to the end of each inner list until it reaches
928
- `max_len`.
929
- """
930
- padded_x = [_pad_to_max(x_i, max_len, pad) for x_i in x]
931
- return torch.tensor(padded_x, dtype=dtype, device=device)
932
-
933
-
910
def _get_graph_batch_size(batch_size: int) -> int:
911
"""Returns the padded batch size given actual batch size.
912
@@ -944,21 +920,3 @@ def _get_graph_batch_size(batch_size: int) -> int:
920
else:
921
return ((batch_size + _BATCH_SIZE_ALIGNMENT - 1) //
922
_BATCH_SIZE_ALIGNMENT * _BATCH_SIZE_ALIGNMENT)
947
-
948
-
949
-def _async_h2d(
950
- data: list,
951
- dtype: torch.dtype,
952
- target_device: Union[str, torch.device],
953
- pin_memory: bool,
954
-) -> torch.Tensor:
955
- t = torch.tensor(data, dtype=dtype, pin_memory=pin_memory, device="cpu")
956
- return t.to(device=target_device, non_blocking=True)
957
-
958
-
959
-def _maybe_expand_dim(tensor: torch.Tensor,
960
- target_dims: int,
961
- size: int = 1) -> torch.Tensor:
962
- if tensor.ndim < target_dims:
963
- tensor = tensor.view(-1, *([size] * (target_dims - tensor.ndim)))
964
- return tensor
vllm/worker/neuron_model_runner.py
new
+287
@@ -0,0 +1,287 @@
1
+from typing import Dict, List, Optional, Tuple
2
+
3
+import torch
4
+
5
+from vllm.config import (DeviceConfig, ModelConfig, ParallelConfig,
6
+ SchedulerConfig)
7
+from vllm.logger import init_logger
8
+from vllm.model_executor import SamplingMetadata
9
+from vllm.model_executor.neuron_model_loader import get_neuron_model
10
+from vllm.sampling_params import SamplingParams, SamplingType
11
+from vllm.sequence import SamplerOutput, SequenceData, SequenceGroupMetadata
12
+from vllm.utils import (async_tensor_h2d, is_pin_memory_available,
13
+ make_tensor_with_pad, maybe_expand_dim)
14
+
15
+logger = init_logger(__name__)
16
+
17
+KVCache = Tuple[torch.Tensor, torch.Tensor]
18
+
19
+
20
+class NeuronModelRunner:
21
+
22
+ def __init__(
23
+ self,
24
+ model_config: ModelConfig,
25
+ parallel_config: ParallelConfig,
26
+ scheduler_config: SchedulerConfig,
27
+ device_config: DeviceConfig,
28
+ ):
29
+ self.model_config = model_config
30
+ self.parallel_config = parallel_config
31
+ self.scheduler_config = scheduler_config
32
+
33
+ if model_config is not None and model_config.get_sliding_window():
34
+ logger.warning("Sliding window is not supported on Neuron. "
35
+ "The model will run without sliding window.")
36
+ self.device_config = (device_config
37
+ if device_config is not None else DeviceConfig())
38
+ self.device = self.device_config.device
39
+ self.model = None
40
+ self.pin_memory = is_pin_memory_available()
41
+
42
+ def load_model(self) -> None:
43
+ self.model = get_neuron_model(self.model_config,
44
+ parallel_config=self.parallel_config,
45
+ scheduler_config=self.scheduler_config)
46
+
47
+ def _prepare_prompt(
48
+ self,
49
+ seq_group_metadata_list: List[SequenceGroupMetadata],
50
+ ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, List[int]]:
51
+ assert len(seq_group_metadata_list) > 0
52
+ input_tokens: List[List[int]] = []
53
+ input_positions: List[List[int]] = []
54
+ input_block_ids: List[int] = []
55
+
56
+ prompt_lens: List[int] = []
57
+ for seq_group_metadata in seq_group_metadata_list:
58
+ assert seq_group_metadata.is_prompt
59
+ seq_ids = list(seq_group_metadata.seq_data.keys())
60
+ assert len(seq_ids) == 1
61
+ seq_id = seq_ids[0]
62
+
63
+ seq_data = seq_group_metadata.seq_data[seq_id]
64
+ prompt_tokens = seq_data.get_token_ids()
65
+ prompt_len = len(prompt_tokens)
66
+ prompt_lens.append(prompt_len)
67
+
68
+ input_tokens.append(prompt_tokens)
69
+ input_positions.append(list(range(prompt_len)))
70
+
71
+ assert seq_group_metadata.block_tables is not None
72
+ block_table = seq_group_metadata.block_tables[seq_id]
73
+ assert len(block_table) == 1
74
+ input_block_ids.append(block_table[0])
75
+
76
+ max_prompt_len = max(prompt_lens)
77
+ assert max_prompt_len > 0
78
+ input_tokens = make_tensor_with_pad(input_tokens,
79
+ max_prompt_len,
80
+ pad=0,
81
+ dtype=torch.long,
82
+ device=self.device)
83
+ input_positions = make_tensor_with_pad(input_positions,
84
+ max_prompt_len,
85
+ pad=0,
86
+ dtype=torch.long,
87
+ device=self.device)
88
+ input_block_ids = torch.tensor(input_block_ids,
89
+ dtype=torch.long,
90
+ device=self.device)
91
+
92
+ return input_tokens, input_positions, input_block_ids, prompt_lens
93
+
94
+ def _prepare_decode(
95
+ self,
96
+ seq_group_metadata_list: List[SequenceGroupMetadata],
97
+ ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
98
+ assert len(seq_group_metadata_list) > 0
99
+ input_tokens: List[List[int]] = []
100
+ input_positions: List[List[int]] = []
101
+ input_block_ids: List[int] = []
102
+ context_lens: List[int] = []
103
+
104
+ for seq_group_metadata in seq_group_metadata_list:
105
+ assert not seq_group_metadata.is_prompt
106
+
107
+ seq_ids = list(seq_group_metadata.seq_data.keys())
108
+
109
+ for seq_id in seq_ids:
110
+ seq_data = seq_group_metadata.seq_data[seq_id]
111
+ generation_token = seq_data.get_last_token_id()
112
+ input_tokens.append([generation_token])
113
+
114
+ seq_len = seq_data.get_len()
115
+ position = seq_len - 1
116
+ input_positions.append([position])
117
+ context_lens.append(seq_len)
118
+
119
+ assert seq_group_metadata.block_tables is not None
120
+ block_table = seq_group_metadata.block_tables[seq_id]
121
+ assert len(block_table) == 1
122
+ input_block_ids.append(block_table[0])
123
+
124
+ input_tokens = make_tensor_with_pad(input_tokens,
125
+ max_len=1,
126
+ pad=0,
127
+ dtype=torch.long,
128
+ device=self.device)
129
+ input_positions = make_tensor_with_pad(input_positions,
130
+ max_len=1,
131
+ pad=0,
132
+ dtype=torch.long,
133
+ device=self.device)
134
+ context_lens = torch.tensor(context_lens,
135
+ dtype=torch.int,
136
+ device=self.device)
137
+ input_block_ids = torch.tensor(input_block_ids,
138
+ dtype=torch.long,
139
+ device=self.device)
140
+
141
+ return input_tokens, input_positions, input_block_ids
142
+
143
+ def _prepare_sample(
144
+ self,
145
+ seq_group_metadata_list: List[SequenceGroupMetadata],
146
+ prompt_lens: List[int],
147
+ ) -> SamplingMetadata:
148
+ seq_groups: List[Tuple[List[int], SamplingParams]] = []
149
+ selected_token_indices: List[int] = []
150
+ generators: List[torch.Generator] = []
151
+ selected_token_start_idx = 0
152
+ categorized_sample_indices = {t: [] for t in SamplingType}
153
+ categorized_sample_indices_start_idx = 0
154
+ categorized_sampled_token_indices_start_idx = 0
155
+
156
+ for i, seq_group_metadata in enumerate(seq_group_metadata_list):
157
+ seq_ids = list(seq_group_metadata.seq_data.keys())
158
+ sampling_params = seq_group_metadata.sampling_params
159
+ seq_groups.append((seq_ids, sampling_params))
160
+
161
+ if seq_group_metadata.is_prompt:
162
+ assert len(seq_ids) == 1
163
+ assert prompt_lens is not None
164
+ prompt_len = prompt_lens[i]
165
+ if sampling_params.prompt_logprobs is not None:
166
+ # NOTE: prompt token positions do not need sample, skip
167
+ categorized_sample_indices_start_idx += prompt_len - 1
168
+
169
+ categorized_sample_indices[
170
+ sampling_params.sampling_type].append([
171
+ categorized_sample_indices_start_idx,
172
+ categorized_sampled_token_indices_start_idx
173
+ ])
174
+ categorized_sample_indices_start_idx += 1
175
+ categorized_sampled_token_indices_start_idx += 1
176
+
177
+ if sampling_params.prompt_logprobs is not None:
178
+ selected_token_indices.extend(
179
+ range(selected_token_start_idx,
180
+ selected_token_start_idx + prompt_len - 1))
181
+ selected_token_indices.append(selected_token_start_idx +
182
+ prompt_len - 1)
183
+ selected_token_start_idx += prompt_len
184
+
185
+ if sampling_params.seed is not None:
186
+ seq_group_metadata.state.generator = torch.Generator(
187
+ device=self.device).manual_seed(sampling_params.seed)
188
+ else:
189
+ num_seqs = len(seq_ids)
190
+ selected_token_indices.extend(
191
+ range(selected_token_start_idx,
192
+ selected_token_start_idx + num_seqs))
193
+ selected_token_start_idx += num_seqs
194
+
195
+ categorized_sample_indices[
196
+ sampling_params.sampling_type].extend(
197
+ zip(
198
+ range(
199
+ categorized_sample_indices_start_idx,
200
+ categorized_sample_indices_start_idx +
201
+ num_seqs),
202
+ range(
203
+ categorized_sampled_token_indices_start_idx,
204
+ categorized_sampled_token_indices_start_idx +
205
+ num_seqs)))
206
+ categorized_sample_indices_start_idx += num_seqs
207
+ categorized_sampled_token_indices_start_idx += num_seqs
208
+
209
+ if sampling_params.seed is not None:
210
+ generators.append(seq_group_metadata.state.generator)
211
+
212
+ selected_token_indices = async_tensor_h2d(selected_token_indices,
213
+ dtype=torch.long,
214
+ target_device=self.device,
215
+ pin_memory=self.pin_memory)
216
+
217
+ categorized_sample_indices = {
218
+ t: maybe_expand_dim(
219
+ async_tensor_h2d(seq_ids,
220
+ dtype=torch.int,
221
+ target_device=self.device,
222
+ pin_memory=self.pin_memory), 2, 2)
223
+ for t, seq_ids in categorized_sample_indices.items()
224
+ }
225
+
226
+ seq_data: Dict[int, SequenceData] = {}
227
+ for seq_group_metadata in seq_group_metadata_list:
228
+ seq_data.update(seq_group_metadata.seq_data)
229
+
230
+ sampling_metadata = SamplingMetadata(
231
+ seq_groups=seq_groups,
232
+ seq_data=seq_data,
233
+ prompt_lens=prompt_lens,
234
+ selected_token_indices=selected_token_indices,
235
+ categorized_sample_indices=categorized_sample_indices,
236
+ generators=generators,
237
+ )
238
+ return sampling_metadata
239
+
240
+ def prepare_input_tensors(
241
+ self,
242
+ seq_group_metadata_list: Optional[List[SequenceGroupMetadata]],
243
+ ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, SamplingMetadata]:
244
+ # NOTE: We assume that all sequences in the group are all prompts or
245
+ # all decodes.
246
+ is_prompt = seq_group_metadata_list[0].is_prompt
247
+ # Prepare input tensors.
248
+ if is_prompt:
249
+ (input_tokens, input_positions, input_block_ids,
250
+ prompt_lens) = self._prepare_prompt(seq_group_metadata_list)
251
+ else:
252
+ (input_tokens, input_positions,
253
+ input_block_ids) = self._prepare_decode(seq_group_metadata_list)
254
+ prompt_lens = []
255
+ sampling_metadata = self._prepare_sample(seq_group_metadata_list,
256
+ prompt_lens)
257
+
258
+ return (input_tokens, input_positions, input_block_ids,
259
+ sampling_metadata)
260
+
261
+ @torch.inference_mode()
262
+ def execute_model(
263
+ self,
264
+ seq_group_metadata_list: Optional[List[SequenceGroupMetadata]],
265
+ ) -> Optional[SamplerOutput]:
266
+ (input_tokens, input_positions, input_block_ids, sampling_metadata
267
+ ) = self.prepare_input_tensors(seq_group_metadata_list)
268
+
269
+ hidden_states = self.model(
270
+ input_ids=input_tokens,
271
+ positions=input_positions,
272
+ input_block_ids=input_block_ids,
273
+ )
274
+
275
+ # Compute the logits.
276
+ logits = self.model.compute_logits(hidden_states, sampling_metadata)
277
+
278
+ # Sample the next token.
279
+ output = self.model.sample(
280
+ logits=logits,
281
+ sampling_metadata=sampling_metadata,
282
+ )
283
+ return output
284
+
285
+ @property
286
+ def vocab_size(self) -> int:
287
+ return self.model_config.get_vocab_size()
vllm/worker/neuron_worker.py
+12
-153
@@ -1,22 +1,17 @@
1
"""A Neuron worker class."""
2
-from typing import Dict, List, Optional, Tuple
2
+from typing import List, Optional
3
4
import torch
5
import torch.distributed
6
7
-from vllm.config import (CacheConfig, DeviceConfig, ModelConfig,
8
- ParallelConfig, SchedulerConfig, LoRAConfig)
7
+from vllm.config import (DeviceConfig, ModelConfig, ParallelConfig,
8
+ SchedulerConfig)
9
from vllm.model_executor import set_random_seed
10
-from vllm.model_executor.parallel_utils.communication_op import (
11
- broadcast_tensor_dict)
12
-from vllm.model_executor.parallel_utils.parallel_state import (
13
- ensure_model_parallel_initialized)
10
from vllm.sequence import SamplerOutput, SequenceGroupMetadata
15
-from vllm.worker.cache_engine import CacheEngine
16
-from vllm.worker.model_runner import ModelRunner
11
+from vllm.worker.neuron_model_runner import NeuronModelRunner
12
13
19
-class Worker:
14
+class NeuronWorker:
15
"""A worker class that executes the model on a group of neuron cores.
16
"""
17
@@ -26,168 +21,32 @@ class Worker:
21
parallel_config: ParallelConfig,
22
scheduler_config: SchedulerConfig,
23
device_config: DeviceConfig,
29
- local_rank: int,
30
- rank: int,
31
- distributed_init_method: str,
32
- lora_config: Optional[LoRAConfig] = None,
33
- kv_cache_dtype: Optional[str] = "auto",
34
- is_driver_worker: bool = False,
24
) -> None:
25
self.model_config = model_config
26
self.parallel_config = parallel_config
27
self.scheduler_config = scheduler_config
28
self.device_config = device_config
40
- self.local_rank = local_rank
41
- self.rank = rank
42
- self.distributed_init_method = distributed_init_method
43
- self.lora_config = lora_config
44
- self.is_driver_worker = is_driver_worker
45
- if self.is_driver_worker:
46
- assert self.rank == 0, "The driver worker must have rank 0."
29
48
- self.model_runner = ModelRunner(model_config,
49
- parallel_config,
50
- scheduler_config,
51
- device_config,
52
- lora_config=self.lora_config,
53
- is_driver_worker=is_driver_worker)
54
- # Uninitialized cache engine. Will be initialized by
55
- # self.init_cache_engine().
56
- self.cache_config = None
57
- self.cache_engine = None
58
- self.cache_events = None
59
- self.gpu_cache = None
30
+ self.model_runner = NeuronModelRunner(model_config, parallel_config,
31
+ scheduler_config, device_config)
32
61
- def init_model(self) -> None:
62
- # Initialize the distributed environment.
63
- _init_distributed_environment(self.parallel_config,
64
- self.rank,
65
- self.distributed_init_method,
66
- distributed_backend="gloo")
67
-
68
- # Initialize the model.
33
+ def init_device(self) -> None:
34
+ # Set random seed.
35
set_random_seed(self.model_config.seed)
36
37
def load_model(self):
38
self.model_runner.load_model()
39
74
- @torch.inference_mode()
75
- def profile_num_available_blocks(
76
- self,
77
- block_size: int = 128,
78
- gpu_memory_utilization: float = 0.9,
79
- cpu_swap_space: int = 0,
80
- cache_dtype: str = "float16",
81
- ) -> Tuple[int, int]:
82
- """Simply returns max_num_seqs as num_gpu_blocks, 0 as
83
- num_cpu_blocks."""
84
- num_gpu_blocks = self.scheduler_config.max_num_seqs
85
- num_cpu_blocks = 0
86
- return num_gpu_blocks, num_cpu_blocks
87
-
88
- def init_cache_engine(self, cache_config: CacheConfig) -> None:
89
- self.cache_config = cache_config
90
- self.cache_engine = CacheEngine(self.cache_config, self.model_config,
91
- self.parallel_config)
92
- self.model_runner.set_block_size(self.cache_engine.block_size)
93
-
94
- def warm_up_model(self) -> None:
95
- # Warm up is maintained in transformers-neuronx
96
- pass
97
-
98
- def cache_swap(
99
- self,
100
- blocks_to_swap_in: Dict[int, int],
101
- blocks_to_swap_out: Dict[int, int],
102
- blocks_to_copy: Dict[int, List[int]],
103
- ) -> None:
104
- # Issue cache operations.
105
- issued_cache_op = False
106
- if blocks_to_swap_in:
107
- self.cache_engine.swap_in(blocks_to_swap_in)
108
- issued_cache_op = True
109
- if blocks_to_swap_out:
110
- self.cache_engine.swap_out(blocks_to_swap_out)
111
- issued_cache_op = True
112
- if blocks_to_copy:
113
- self.cache_engine.copy(blocks_to_copy)
114
- issued_cache_op = True
115
-
116
- cache_events = self.cache_events if issued_cache_op else None
117
-
118
- # Wait for cache operations to finish.
119
- if cache_events is not None:
120
- raise NotImplementedError(
121
- "cache operations are not implemented for neuron backend.")
122
-
40
@torch.inference_mode()
41
def execute_model(
42
self,
126
- seq_group_metadata_list: Optional[List[SequenceGroupMetadata]] = None,
127
- blocks_to_swap_in: Optional[Dict[int, int]] = None,
128
- blocks_to_swap_out: Optional[Dict[int, int]] = None,
129
- blocks_to_copy: Optional[Dict[int, List[int]]] = None,
43
+ seq_group_metadata_list: List[SequenceGroupMetadata],
44
) -> Optional[SamplerOutput]:
131
- if self.is_driver_worker:
132
- assert seq_group_metadata_list is not None
133
- num_seq_groups = len(seq_group_metadata_list)
134
- assert blocks_to_swap_in is not None
135
- assert blocks_to_swap_out is not None
136
- assert blocks_to_copy is not None
137
- data = {
138
- "num_seq_groups": num_seq_groups,
139
- "blocks_to_swap_in": blocks_to_swap_in,
140
- "blocks_to_swap_out": blocks_to_swap_out,
141
- "blocks_to_copy": blocks_to_copy,
142
- }
143
- broadcast_tensor_dict(data, src=0)
144
- else:
145
- data = broadcast_tensor_dict(src=0)
146
- num_seq_groups = data["num_seq_groups"]
147
- blocks_to_swap_in = data["blocks_to_swap_in"]
148
- blocks_to_swap_out = data["blocks_to_swap_out"]
149
- blocks_to_copy = data["blocks_to_copy"]
150
-
151
- self.cache_swap(blocks_to_swap_in, blocks_to_swap_out, blocks_to_copy)
45
+ num_seq_groups = len(seq_group_metadata_list)
46
47
# If there is no input, we don't need to execute the model.
48
if num_seq_groups == 0:
49
return {}
50
157
- output = self.model_runner.execute_model(seq_group_metadata_list,
158
- self.gpu_cache)
51
+ output = self.model_runner.execute_model(seq_group_metadata_list)
52
return output
160
-
161
-
162
-def _init_distributed_environment(
163
- parallel_config: ParallelConfig,
164
- rank: int,
165
- distributed_init_method: Optional[str] = None,
166
- distributed_backend: Optional[str] = None,
167
-) -> None:
168
- """Initialize the distributed environment."""
169
- if torch.distributed.is_initialized():
170
- torch_world_size = torch.distributed.get_world_size()
171
- if torch_world_size != parallel_config.world_size:
172
- raise RuntimeError(
173
- "torch.distributed is already initialized but the torch world "
174
- "size does not match parallel_config.world_size "
175
- f"({torch_world_size} vs. {parallel_config.world_size}).")
176
- elif not distributed_init_method:
177
- raise ValueError(
178
- "distributed_init_method must be set if torch.distributed "
179
- "is not already initialized")
180
- else:
181
- distributed_backend = (distributed_backend
182
- if distributed_backend else "nccl")
183
- torch.distributed.init_process_group(
184
- backend=distributed_backend,
185
- world_size=parallel_config.world_size,
186
- rank=rank,
187
- init_method=distributed_init_method,
188
- )
189
-
190
- # A small all_reduce for warmup.
191
- torch.distributed.all_reduce(torch.zeros(1))
192
- ensure_model_parallel_initialized(parallel_config.tensor_parallel_size,
193
- parallel_config.pipeline_parallel_size)
vllm/worker/worker.py
+2
-2
@@ -67,7 +67,7 @@ class Worker:
67
self.cache_engine = None
68
self.gpu_cache = None
69
70
- def init_model(self, cupy_port: Optional[int] = None) -> None:
70
+ def init_device(self, cupy_port: Optional[int] = None) -> None:
71
if self.device_config.device.type == "cuda":
72
# torch.distributed.all_reduce does not free the input tensor until
73
# the synchronization point. This causes the memory usage to grow
@@ -91,7 +91,7 @@ class Worker:
91
# Initialize the distributed environment.
92
init_distributed_environment(self.parallel_config, self.rank,
93
cupy_port, self.distributed_init_method)
94
- # Initialize the model.
94
+ # Set random seed.
95
set_random_seed(self.model_config.seed)
96
97
def load_model(self):