A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
[Neuron] Support inference with transformers-neuronx (#2569)
Liangfu Chen committed
Feb 28, 2024 at 09:34 UTC
3b7178cfa4a317922d4aef9dd3b2647b8d950e7d
18 files changed
+516
-42
examples/offline_inference_neuron.py
new
+33
@@ -0,0 +1,33 @@
1
+from vllm import LLM, SamplingParams
2
+
3
+# Sample prompts.
4
+prompts = [
5
+ "Hello, my name is",
6
+ "The president of the United States is",
7
+ "The capital of France is",
8
+ "The future of AI is",
9
+]
10
+# Create a sampling params object.
11
+sampling_params = SamplingParams(temperature=0.8, top_p=0.95)
12
+
13
+# Create an LLM.
14
+llm = LLM(
15
+ model="openlm-research/open_llama_3b",
16
+ max_num_seqs=8,
17
+ # The max_model_len and block_size arguments are required to be same as max sequence length,
18
+ # when targeting neuron device. Currently, this is a known limitation in continuous batching
19
+ # support in transformers-neuronx.
20
+ # TODO(liangfu): Support paged-attention in transformers-neuronx.
21
+ max_model_len=128,
22
+ block_size=128,
23
+ # The device can be automatically detected when AWS Neuron SDK is installed.
24
+ # The device argument can be either unspecified for automated detection, or explicitly assigned.
25
+ device="neuron")
26
+# Generate texts from the prompts. The output is a list of RequestOutput objects
27
+# that contain the prompt, generated text, and other information.
28
+outputs = llm.generate(prompts, sampling_params)
29
+# Print the outputs.
30
+for output in outputs:
31
+ prompt = output.prompt
32
+ generated_text = output.outputs[0].text
33
+ print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
tests/lora/conftest.py
+5
-3
@@ -131,9 +131,11 @@ def llama_2_7b_engine_extra_embeddings() -> nn.Module:
131
cleanup()
132
get_model_old = get_model
133
134
- def get_model_patched(model_config, device_config, lora_config=None):
135
- return get_model_old(model_config, device_config,
136
- LoRAConfig(max_loras=4, max_lora_rank=8))
134
+ def get_model_patched(model_config, device_config, **kwargs):
135
+ return get_model_old(model_config,
136
+ device_config,
137
+ lora_config=LoRAConfig(max_loras=4,
138
+ max_lora_rank=8))
139
140
with patch("vllm.worker.model_runner.get_model", get_model_patched):
141
engine = vllm.LLM("meta-llama/Llama-2-7b-hf", enable_lora=False)
vllm/config.py
+35
-6
@@ -8,7 +8,7 @@ from transformers import PretrainedConfig
8
9
from vllm.logger import init_logger
10
from vllm.transformers_utils.config import get_config
11
-from vllm.utils import get_cpu_memory, is_hip, get_nvcc_cuda_version
11
+from vllm.utils import get_cpu_memory, is_hip, is_neuron, get_nvcc_cuda_version
12
13
logger = init_logger(__name__)
14
@@ -380,13 +380,21 @@ class ParallelConfig:
380
disable_custom_all_reduce: bool = False,
381
) -> None:
382
self.pipeline_parallel_size = pipeline_parallel_size
383
- self.tensor_parallel_size = tensor_parallel_size
383
+ if is_neuron():
384
+ # For Neuron device support, here we assign TP=1 to avoid sharding within vLLM directly.
385
+ # Transformer-neuronx would take neuron_tp_degree attribute, and distribute the workload
386
+ # to multiple NeuronCores.
387
+ self.tensor_parallel_size = 1
388
+ self.neuron_tp_degree = tensor_parallel_size
389
+ else:
390
+ self.tensor_parallel_size = tensor_parallel_size
391
self.worker_use_ray = worker_use_ray
392
self.max_parallel_loading_workers = max_parallel_loading_workers
393
self.disable_custom_all_reduce = disable_custom_all_reduce
394
388
- self.world_size = pipeline_parallel_size * tensor_parallel_size
389
- if self.world_size > 1:
395
+ self.world_size = pipeline_parallel_size * self.tensor_parallel_size
396
+ # Ray worker is not supported for Neuron backend.
397
+ if self.world_size > 1 and not is_neuron():
398
self.worker_use_ray = True
399
self._verify_args()
400
@@ -465,8 +473,29 @@ class SchedulerConfig:
473
474
class DeviceConfig:
475
468
- def __init__(self, device: str = "cuda") -> None:
469
- self.device = torch.device(device)
476
+ def __init__(self, device: str = "auto") -> None:
477
+ if device == "auto":
478
+ # Automated device type detection
479
+ if torch.cuda.is_available():
480
+ self.device_type = "cuda"
481
+ elif is_neuron():
482
+ self.device_type = "neuron"
483
+ else:
484
+ raise RuntimeError("No supported device detected.")
485
+ else:
486
+ # Device type is assigned explicitly
487
+ self.device_type = device
488
+
489
+ # Some device types require processing inputs on CPU
490
+ if self.device_type in ["neuron"]:
491
+ self.device = torch.device("cpu")
492
+ else:
493
+ # Set device with device type
494
+ self.device = torch.device(self.device_type)
495
+
496
+ @property
497
+ def is_neuron(self):
498
+ return self.device_type == "neuron"
499
500
501
@dataclass
vllm/engine/arg_utils.py
+7
-9
@@ -44,7 +44,7 @@ class EngineArgs:
44
lora_extra_vocab_size: int = 256
45
lora_dtype = 'auto'
46
max_cpu_loras: Optional[int] = None
47
- device: str = 'cuda'
47
+ device: str = 'auto'
48
49
def __post_init__(self):
50
if self.tokenizer is None:
@@ -171,7 +171,7 @@ class EngineArgs:
171
parser.add_argument('--block-size',
172
type=int,
173
default=EngineArgs.block_size,
174
- choices=[8, 16, 32],
174
+ choices=[8, 16, 32, 128],
175
help='token block size')
176
parser.add_argument('--seed',
177
type=int,
@@ -264,13 +264,11 @@ class EngineArgs:
264
help=('Maximum number of LoRAs to store in CPU memory. '
265
'Must be >= than max_num_seqs. '
266
'Defaults to max_num_seqs.'))
267
- parser.add_argument(
268
- "--device",
269
- type=str,
270
- default=EngineArgs.device,
271
- choices=["cuda"],
272
- help=('Device type for vLLM execution. '
273
- 'Currently, only CUDA-compatible devices are supported.'))
267
+ parser.add_argument("--device",
268
+ type=str,
269
+ default=EngineArgs.device,
270
+ choices=["auto", "cuda", "neuron"],
271
+ help='Device type for vLLM execution.')
272
return parser
273
274
@classmethod
vllm/engine/llm_engine.py
+18
-3
@@ -3,6 +3,7 @@ from collections import defaultdict
3
import os
4
import time
5
import pickle
6
+import importlib
7
from typing import (TYPE_CHECKING, Any, Dict, Iterable, List, Optional, Tuple,
8
Union)
9
@@ -20,7 +21,8 @@ from vllm.sequence import (SamplerOutput, Sequence, SequenceGroup,
21
SequenceGroupOutput, SequenceOutput, SequenceStatus)
22
from vllm.transformers_utils.tokenizer import (detokenize_incrementally,
23
TokenizerGroup)
23
-from vllm.utils import Counter, set_cuda_visible_devices, get_ip, get_open_port, get_distributed_init_method
24
+from vllm.utils import (Counter, set_cuda_visible_devices, get_ip,
25
+ get_open_port, get_distributed_init_method)
26
27
if ray:
28
from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy
@@ -31,6 +33,12 @@ if TYPE_CHECKING:
33
logger = init_logger(__name__)
34
_LOCAL_LOGGING_INTERVAL_SEC = 5
35
36
+# A map between the device type (in device config) to its worker module.
37
+DEVICE_TO_WORKER_MODULE_MAP = {
38
+ "cuda": "vllm.worker.worker",
39
+ "neuron": "vllm.worker.neuron_worker",
40
+}
41
+
42
# If the env var is set, it uses the Ray's compiled DAG API
43
# which optimizes the control plane overhead.
44
# Run VLLM with VLLM_USE_RAY_COMPILED_DAG=1 to enable it.
@@ -138,10 +146,17 @@ class LLMEngine:
146
def get_tokenizer_for_seq(self, sequence: Sequence):
147
return self.tokenizer.get_lora_tokenizer(sequence.lora_request)
148
149
+ def _dispatch_worker(self):
150
+ worker_module = DEVICE_TO_WORKER_MODULE_MAP[
151
+ self.device_config.device_type]
152
+ imported_worker = importlib.import_module(worker_module)
153
+ Worker = imported_worker.Worker
154
+ return Worker
155
+
156
def _init_workers(self):
157
# Lazy import the Worker to avoid importing torch.cuda/xformers
158
# before CUDA_VISIBLE_DEVICES is set in the Worker
144
- from vllm.worker.worker import Worker
159
+ Worker = self._dispatch_worker()
160
161
assert self.parallel_config.world_size == 1, (
162
"Ray is required if parallel_config.world_size > 1.")
@@ -243,7 +258,7 @@ class LLMEngine:
258
259
# Lazy import the Worker to avoid importing torch.cuda/xformers
260
# before CUDA_VISIBLE_DEVICES is set in the Worker
246
- from vllm.worker.worker import Worker
261
+ Worker = self._dispatch_worker()
262
263
# Initialize torch distributed process group for the workers.
264
model_config = copy.deepcopy(self.model_config)
vllm/lora/layers.py
+4
@@ -795,6 +795,10 @@ class SamplerWithLoRA(BaseLayerWithLoRA):
795
self.dtype = dtype
796
self.device = device
797
798
+ @property
799
+ def logits_as_hidden_states(self):
800
+ return self.base_layer.logits_as_hidden_states
801
+
802
@property
803
def vocab_size(self):
804
return self.base_layer.vocab_size
vllm/model_executor/__init__.py
+1
-2
@@ -1,7 +1,6 @@
1
from vllm.model_executor.input_metadata import InputMetadata
2
-from vllm.model_executor.model_loader import get_model
2
from vllm.model_executor.sampling_metadata import SamplingMetadata
4
-from vllm.model_executor.utils import set_random_seed
3
+from vllm.model_executor.utils import set_random_seed, get_model
4
5
__all__ = [
6
"InputMetadata",
vllm/model_executor/layers/sampler.py
+13
-5
@@ -10,6 +10,7 @@ from vllm.model_executor.sampling_metadata import SamplingMetadata, SamplingTens
10
from vllm.sampling_params import SamplingParams, SamplingType
11
from vllm.sequence import (PromptLogprobs, SampleLogprobs, SamplerOutput,
12
SequenceData, SequenceGroupOutput, SequenceOutput)
13
+from vllm.utils import is_neuron
14
15
16
class Sampler(nn.Module):
@@ -32,6 +33,8 @@ class Sampler(nn.Module):
33
org_vocab_size: Optional[int] = None) -> None:
34
super().__init__()
35
self.vocab_size = vocab_size
36
+ # Transformers-neuronx generate outputs as logits directly.
37
+ self.logits_as_hidden_states = is_neuron()
38
# original vocabulary size (without LoRA).
39
self.org_vocab_size = org_vocab_size or vocab_size
40
@@ -55,10 +58,14 @@ class Sampler(nn.Module):
58
embedding_bias: Optional[torch.Tensor] = None,
59
) -> Optional[SamplerOutput]:
60
# Get the hidden states that we use for sampling.
58
- hidden_states = _prune_hidden_states(hidden_states, sampling_metadata)
61
+ if self.logits_as_hidden_states:
62
+ logits = hidden_states
63
+ else:
64
+ hidden_states = _prune_hidden_states(hidden_states,
65
+ sampling_metadata)
66
60
- # Get the logits for the next tokens.
61
- logits = self._get_logits(hidden_states, embedding, embedding_bias)
67
+ # Get the logits for the next tokens.
68
+ logits = self._get_logits(hidden_states, embedding, embedding_bias)
69
70
# Only perform sampling in the driver worker.
71
# Note: `_get_logits` is still distributed across TP workers because
@@ -395,7 +402,8 @@ def _sample(
402
sample_metadata[sampling_type] = (seq_group_ids, seq_groups,
403
is_prompts, sample_indices)
404
if sampling_type == SamplingType.GREEDY:
398
- greedy_samples = torch.argmax(logprobs[sample_indices], dim=-1)
405
+ greedy_samples = torch.argmax(logprobs[sample_indices.long()],
406
+ dim=-1)
407
elif sampling_type in (SamplingType.RANDOM, SamplingType.RANDOM_SEED):
408
max_best_of = 1
409
for seq_group, is_prompt in zip(seq_groups, is_prompts):
@@ -407,7 +415,7 @@ def _sample(
415
"generators": sampling_metadata.generators,
416
}
417
multinomial_samples[sampling_type] = _multinomial(
410
- probs[sample_indices], max_best_of, **seeded_args)
418
+ probs[sample_indices.long()], max_best_of, **seeded_args)
419
elif sampling_type == SamplingType.BEAM:
420
beam_search_logprobs = logprobs[sample_indices]
421
else:
vllm/model_executor/model_loader.py
+5
-5
@@ -1,11 +1,11 @@
1
"""Utilities for selecting and loading models."""
2
import contextlib
3
-from typing import Optional, Type
3
+from typing import Type
4
5
import torch
6
import torch.nn as nn
7
8
-from vllm.config import DeviceConfig, ModelConfig, LoRAConfig
8
+from vllm.config import DeviceConfig, ModelConfig
9
from vllm.model_executor.models import ModelRegistry
10
from vllm.model_executor.weight_utils import (get_quant_config,
11
initialize_dummy_weights)
@@ -37,9 +37,9 @@ def _get_model_architecture(model_config: ModelConfig) -> Type[nn.Module]:
37
f"Supported architectures: {ModelRegistry.get_supported_archs()}")
38
39
40
-def get_model(model_config: ModelConfig,
41
- device_config: DeviceConfig,
42
- lora_config: Optional[LoRAConfig] = None) -> nn.Module:
40
+def get_model(model_config: ModelConfig, device_config: DeviceConfig,
41
+ **kwargs) -> nn.Module:
42
+ lora_config = kwargs.get("lora_config", None)
43
model_class = _get_model_architecture(model_config)
44
45
# Get the (maybe quantized) linear method.
vllm/model_executor/models/__init__.py
+11
-1
@@ -4,7 +4,7 @@ from typing import List, Optional, Type
4
import torch.nn as nn
5
6
from vllm.logger import init_logger
7
-from vllm.utils import is_hip
7
+from vllm.utils import is_hip, is_neuron
8
9
logger = init_logger(__name__)
10
@@ -61,6 +61,9 @@ _ROCM_PARTIALLY_SUPPORTED_MODELS = {
61
"Sliding window attention is not yet supported in ROCm's flash attention",
62
}
63
64
+# Models not supported by Neuron.
65
+_NEURON_SUPPORTED_MODELS = {"LlamaForCausalLM": "neuron.llama"}
66
+
67
68
class ModelRegistry:
69
@@ -77,8 +80,15 @@ class ModelRegistry:
80
logger.warning(
81
f"Model architecture {model_arch} is partially supported "
82
"by ROCm: " + _ROCM_PARTIALLY_SUPPORTED_MODELS[model_arch])
83
+ elif is_neuron():
84
+ if model_arch not in _NEURON_SUPPORTED_MODELS:
85
+ raise ValueError(
86
+ f"Model architecture {model_arch} is not supported by "
87
+ "Neuron for now.")
88
89
module_name, model_cls_name = _MODELS[model_arch]
90
+ if is_neuron():
91
+ module_name = _NEURON_SUPPORTED_MODELS[model_arch]
92
module = importlib.import_module(
93
f"vllm.model_executor.models.{module_name}")
94
return getattr(module, model_cls_name, None)
vllm/model_executor/models/neuron/llama.py
new
+79
@@ -0,0 +1,79 @@
1
+"""Inference-only LLaMA model compatible with HuggingFace weights."""
2
+import os
3
+from typing import List, Optional, Tuple
4
+
5
+import torch
6
+from torch import nn
7
+from transformers import LlamaConfig
8
+
9
+from vllm.model_executor.input_metadata import InputMetadata
10
+from vllm.model_executor.layers.sampler import Sampler
11
+from vllm.model_executor.sampling_metadata import SamplingMetadata
12
+from vllm.sequence import SamplerOutput
13
+
14
+KVCache = Tuple[torch.Tensor, torch.Tensor]
15
+
16
+
17
+class LlamaForCausalLM(nn.Module):
18
+
19
+ def __init__(
20
+ self,
21
+ config: LlamaConfig,
22
+ linear_method=None,
23
+ ) -> None:
24
+ super().__init__()
25
+ self.config = config
26
+ self.linear_method = linear_method
27
+ self.model = None
28
+ self.sampler = Sampler(config.vocab_size)
29
+
30
+ def forward(
31
+ self,
32
+ input_ids: torch.Tensor,
33
+ positions: torch.Tensor,
34
+ kv_caches: List[KVCache],
35
+ input_metadata: InputMetadata,
36
+ ) -> torch.Tensor:
37
+ with torch.inference_mode():
38
+ block_size = self.model.context_buckets[-1]
39
+ if input_metadata.is_prompt:
40
+ seq_ids = input_metadata.slot_mapping[:, 0] // block_size
41
+ else:
42
+ seq_ids = input_metadata.block_tables
43
+ logits = self.model(input_ids,
44
+ cache_ids=positions,
45
+ start_ids=seq_ids.flatten())
46
+ return logits
47
+
48
+ def sample(
49
+ self,
50
+ hidden_states: torch.Tensor,
51
+ sampling_metadata: SamplingMetadata,
52
+ ) -> Optional[SamplerOutput]:
53
+ next_tokens = self.sampler(self.model.chkpt_model.lm_head,
54
+ hidden_states, sampling_metadata)
55
+ return next_tokens
56
+
57
+ def load_weights(self,
58
+ model_name_or_path: str,
59
+ cache_dir: Optional[str] = None,
60
+ load_format: str = "auto",
61
+ revision: Optional[str] = None,
62
+ **kwargs):
63
+ from transformers_neuronx.llama.model import LlamaForSampling
64
+
65
+ split_model_dir = f"{model_name_or_path}-split"
66
+ if os.path.isdir(os.path.join(model_name_or_path,
67
+ "pytorch_model.bin")):
68
+ split_model_dir = model_name_or_path
69
+ elif not os.path.exists(f"{model_name_or_path}-split"):
70
+ from transformers.models.llama import LlamaForCausalLM
71
+ from transformers_neuronx.module import save_pretrained_split
72
+
73
+ hf_model = LlamaForCausalLM.from_pretrained(model_name_or_path,
74
+ low_cpu_mem_usage=True)
75
+ save_pretrained_split(hf_model, f"{model_name_or_path}-split")
76
+
77
+ self.model = LlamaForSampling.from_pretrained(split_model_dir,
78
+ **kwargs)
79
+ self.model.to_neuron()
vllm/model_executor/neuron_model_loader.py
new
+66
@@ -0,0 +1,66 @@
1
+"""Utilities for selecting and loading models."""
2
+from typing import Type
3
+
4
+import torch
5
+import torch.nn as nn
6
+from transformers import PretrainedConfig
7
+
8
+from vllm.config import ModelConfig, DeviceConfig
9
+from vllm.model_executor.models import ModelRegistry
10
+
11
+TORCH_DTYPE_TO_NEURON_AMP = {
12
+ "auto": "f32",
13
+ "half": "f16",
14
+ "float16": "f16",
15
+ "bfloat16": "bf16",
16
+ "float": "f32",
17
+ "float32": "f32",
18
+ torch.float16: "f16",
19
+ torch.bfloat16: "bf16",
20
+ torch.float32: "f32",
21
+}
22
+
23
+
24
+def _get_model_architecture(config: PretrainedConfig) -> Type[nn.Module]:
25
+ architectures = getattr(config, "architectures", [])
26
+ for arch in architectures:
27
+ model_cls = ModelRegistry.load_model_cls(arch)
28
+ if model_cls is not None:
29
+ return model_cls
30
+ raise ValueError(
31
+ f"Model architectures {architectures} are not supported for now. "
32
+ f"Supported architectures: {ModelRegistry.get_supported_archs()}")
33
+
34
+
35
+def get_model(model_config: ModelConfig, device_config: DeviceConfig,
36
+ **kwargs) -> nn.Module:
37
+ from transformers_neuronx.config import NeuronConfig, ContinuousBatchingConfig
38
+
39
+ parallel_config = kwargs.get("parallel_config")
40
+ scheduler_config = kwargs.get("scheduler_config")
41
+
42
+ model_class = _get_model_architecture(model_config.hf_config)
43
+ linear_method = None
44
+
45
+ # Create a model instance.
46
+ model = model_class(model_config.hf_config, linear_method)
47
+
48
+ continuous_batching_config = ContinuousBatchingConfig(
49
+ batch_size_for_shared_caches=scheduler_config.max_num_seqs)
50
+ neuron_config = NeuronConfig(
51
+ continuous_batching=continuous_batching_config)
52
+
53
+ # Load the weights from the cached or downloaded files.
54
+ model.load_weights(
55
+ model_config.model,
56
+ model_config.download_dir,
57
+ model_config.load_format,
58
+ model_config.revision,
59
+ tp_degree=parallel_config.neuron_tp_degree,
60
+ amp=TORCH_DTYPE_TO_NEURON_AMP[model_config.dtype],
61
+ neuron_config=neuron_config,
62
+ context_length_estimate=[scheduler_config.max_model_len],
63
+ n_positions=[scheduler_config.max_model_len],
64
+ batch_size=scheduler_config.max_num_seqs)
65
+
66
+ return model.eval()
vllm/model_executor/sampling_metadata.py
+2
-2
@@ -5,7 +5,7 @@ import torch
5
6
from vllm.sampling_params import SamplingParams, SamplingType
7
from vllm.sequence import SequenceData
8
-from vllm.utils import in_wsl
8
+from vllm.utils import in_wsl, is_neuron
9
10
_SAMPLING_EPS = 1e-5
11
@@ -155,7 +155,7 @@ class SamplingTensors:
155
dtype: torch.dtype) -> "SamplingTensors":
156
# Note that the performance will be very bad without
157
# pinned memory.
158
- pin_memory = not in_wsl()
158
+ pin_memory = not in_wsl() and not is_neuron()
159
prompt_max_len = max(len(tokens) for tokens in prompt_tokens)
160
prompt_padded_tokens = [
161
tokens + [vocab_size] * (prompt_max_len - len(tokens))
vllm/model_executor/utils.py
+17
@@ -1,10 +1,18 @@
1
"""Utils for model executor."""
2
import random
3
+import importlib
4
from typing import Any, Dict, Optional
5
6
import numpy as np
7
import torch
8
9
+from vllm.config import DeviceConfig, ModelConfig
10
+
11
+DEVICE_TO_MODEL_LOADER_MAP = {
12
+ "cuda": "model_loader",
13
+ "neuron": "neuron_model_loader",
14
+}
15
+
16
17
def set_random_seed(seed: int) -> None:
18
random.seed(seed)
@@ -33,3 +41,12 @@ def set_weight_attrs(
41
assert not hasattr(
42
weight, key), (f"Overwriting existing tensor attribute: {key}")
43
setattr(weight, key, value)
44
+
45
+
46
+def get_model(model_config: ModelConfig, device_config: DeviceConfig,
47
+ **kwargs) -> torch.nn.Module:
48
+ model_loader_module = DEVICE_TO_MODEL_LOADER_MAP[device_config.device_type]
49
+ imported_model_loader = importlib.import_module(
50
+ f"vllm.model_executor.{model_loader_module}")
51
+ get_model_fn = imported_model_loader.get_model
52
+ return get_model_fn(model_config, device_config, **kwargs)
vllm/utils.py
+8
@@ -118,6 +118,14 @@ def is_hip() -> bool:
118
return torch.version.hip is not None
119
120
121
+def is_neuron() -> bool:
122
+ try:
123
+ import transformers_neuronx
124
+ except ImportError:
125
+ transformers_neuronx = None
126
+ return transformers_neuronx is not None
127
+
128
+
129
def get_max_shared_memory_bytes(gpu: int = 0) -> int:
130
"""Returns the maximum shared memory per thread block in bytes."""
131
# NOTE: This import statement should be executed lazily since
vllm/worker/cache_engine.py
+9
-2
@@ -3,10 +3,9 @@ from typing import Dict, List, Tuple
3
4
import torch
5
6
-from vllm._C import cache_ops
6
from vllm.config import CacheConfig, ModelConfig, ParallelConfig
7
from vllm.logger import init_logger
9
-from vllm.utils import in_wsl, STR_DTYPE_TO_TORCH_DTYPE
8
+from vllm.utils import in_wsl, is_neuron, STR_DTYPE_TO_TORCH_DTYPE
9
10
logger = init_logger(__name__)
11
@@ -39,6 +38,10 @@ class CacheEngine:
38
self.num_gpu_blocks = cache_config.num_gpu_blocks
39
self.num_cpu_blocks = cache_config.num_cpu_blocks
40
41
+ # Skip initializing CUDA stream and buffer for Neuron backend.
42
+ if is_neuron():
43
+ return
44
+
45
if cache_config.cache_dtype == "auto":
46
self.dtype = model_config.dtype
47
else:
@@ -121,6 +124,8 @@ class CacheEngine:
124
dst: List[KVCache],
125
src_to_dst: Dict[int, int],
126
) -> None:
127
+ from vllm._C import cache_ops
128
+
129
with torch.cuda.stream(self.cache_stream):
130
for i in range(self.num_layers):
131
src_key_cache, src_value_cache = src[i]
@@ -140,6 +145,8 @@ class CacheEngine:
145
self._swap(self.gpu_cache, self.cpu_cache, src_to_dst)
146
147
def copy(self, src_to_dsts: Dict[int, List[int]]) -> None:
148
+ from vllm._C import cache_ops
149
+
150
key_caches = [key_cache for key_cache, _ in self.gpu_cache]
151
value_caches = [value_cache for _, value_cache in self.gpu_cache]
152
# NOTE(woosuk): This operation implicitly synchronizes the CPU and GPU.
vllm/worker/model_runner.py
+12
-4
@@ -80,9 +80,16 @@ class ModelRunner:
80
self.in_wsl = in_wsl()
81
self.kv_cache_dtype = kv_cache_dtype
82
83
+ # Set enforce_eager to True for Neuron backend, to avoid capturing graph
84
+ if self.device_config.is_neuron:
85
+ self.model_config.enforce_eager = True
86
+
87
def load_model(self) -> None:
84
- self.model = get_model(self.model_config, self.device_config,
85
- self.lora_config)
88
+ self.model = get_model(self.model_config,
89
+ self.device_config,
90
+ lora_config=self.lora_config,
91
+ parallel_config=self.parallel_config,
92
+ scheduler_config=self.scheduler_config)
93
94
vocab_size = self.model.config.vocab_size
95
@@ -393,6 +400,7 @@ class ModelRunner:
400
selected_token_start_idx = 0
401
categorized_sample_indices = {t: [] for t in SamplingType}
402
categorized_sample_indices_start_idx = 0
403
+ pin_memory = not self.in_wsl and not self.device_config.is_neuron
404
405
max_subquery_len = max(subquery_lens) if subquery_lens else 1
406
for i, seq_group_metadata in enumerate(seq_group_metadata_list):
@@ -443,12 +451,12 @@ class ModelRunner:
451
selected_token_indices = _async_h2d(selected_token_indices,
452
dtype=torch.long,
453
target_device=self.device,
446
- pin_memory=not self.in_wsl)
454
+ pin_memory=pin_memory)
455
categorized_sample_indices = {
456
t: _async_h2d(seq_ids,
457
dtype=torch.int,
458
target_device=self.device,
451
- pin_memory=not self.in_wsl)
459
+ pin_memory=pin_memory)
460
for t, seq_ids in categorized_sample_indices.items()
461
}
462
vllm/worker/neuron_worker.py
new
+191
@@ -0,0 +1,191 @@
1
+"""A Neuron worker class."""
2
+from typing import Dict, List, Optional, Tuple
3
+
4
+import torch
5
+import torch.distributed
6
+
7
+from vllm.config import (CacheConfig, DeviceConfig, ModelConfig,
8
+ ParallelConfig, SchedulerConfig, LoRAConfig)
9
+from vllm.model_executor import set_random_seed
10
+from vllm.model_executor.parallel_utils.communication_op import (
11
+ broadcast_tensor_dict)
12
+from vllm.model_executor.parallel_utils.parallel_state import (
13
+ ensure_model_parallel_initialized)
14
+from vllm.sequence import SamplerOutput, SequenceGroupMetadata
15
+from vllm.worker.cache_engine import CacheEngine
16
+from vllm.worker.model_runner import ModelRunner
17
+
18
+
19
+class Worker:
20
+ """A worker class that executes the model on a group of neuron cores.
21
+ """
22
+
23
+ def __init__(
24
+ self,
25
+ model_config: ModelConfig,
26
+ parallel_config: ParallelConfig,
27
+ scheduler_config: SchedulerConfig,
28
+ device_config: DeviceConfig,
29
+ local_rank: int,
30
+ rank: int,
31
+ distributed_init_method: str,
32
+ lora_config: Optional[LoRAConfig] = None,
33
+ kv_cache_dtype: Optional[str] = "auto",
34
+ is_driver_worker: bool = False,
35
+ ) -> None:
36
+ self.model_config = model_config
37
+ self.parallel_config = parallel_config
38
+ self.scheduler_config = scheduler_config
39
+ self.device_config = device_config
40
+ self.local_rank = local_rank
41
+ self.rank = rank
42
+ self.distributed_init_method = distributed_init_method
43
+ self.lora_config = lora_config
44
+ self.is_driver_worker = is_driver_worker
45
+ if self.is_driver_worker:
46
+ assert self.rank == 0, "The driver worker must have rank 0."
47
+
48
+ self.model_runner = ModelRunner(model_config,
49
+ parallel_config,
50
+ scheduler_config,
51
+ device_config,
52
+ lora_config=self.lora_config,
53
+ is_driver_worker=is_driver_worker)
54
+ # Uninitialized cache engine. Will be initialized by
55
+ # self.init_cache_engine().
56
+ self.cache_config = None
57
+ self.cache_engine = None
58
+ self.cache_events = None
59
+ self.gpu_cache = None
60
+
61
+ def init_model(self) -> None:
62
+ # Initialize the distributed environment.
63
+ _init_distributed_environment(self.parallel_config,
64
+ self.rank,
65
+ self.distributed_init_method,
66
+ distributed_backend="gloo")
67
+
68
+ # Initialize the model.
69
+ set_random_seed(self.model_config.seed)
70
+
71
+ def load_model(self):
72
+ self.model_runner.load_model()
73
+
74
+ @torch.inference_mode()
75
+ def profile_num_available_blocks(
76
+ self,
77
+ block_size: int = 128,
78
+ gpu_memory_utilization: float = 0.9,
79
+ cpu_swap_space: int = 0,
80
+ cache_dtype: str = "float16",
81
+ ) -> Tuple[int, int]:
82
+ """Simply returns max_num_seqs as num_gpu_blocks, 0 as num_cpu_blocks."""
83
+ num_gpu_blocks = self.scheduler_config.max_num_seqs
84
+ num_cpu_blocks = 0
85
+ return num_gpu_blocks, num_cpu_blocks
86
+
87
+ def init_cache_engine(self, cache_config: CacheConfig) -> None:
88
+ self.cache_config = cache_config
89
+ self.cache_engine = CacheEngine(self.cache_config, self.model_config,
90
+ self.parallel_config)
91
+ self.model_runner.set_block_size(self.cache_engine.block_size)
92
+
93
+ def warm_up_model(self) -> None:
94
+ # Warm up is maintained in transformers-neuronx
95
+ pass
96
+
97
+ def cache_swap(
98
+ self,
99
+ blocks_to_swap_in: Dict[int, int],
100
+ blocks_to_swap_out: Dict[int, int],
101
+ blocks_to_copy: Dict[int, List[int]],
102
+ ) -> None:
103
+ # Issue cache operations.
104
+ issued_cache_op = False
105
+ if blocks_to_swap_in:
106
+ self.cache_engine.swap_in(blocks_to_swap_in)
107
+ issued_cache_op = True
108
+ if blocks_to_swap_out:
109
+ self.cache_engine.swap_out(blocks_to_swap_out)
110
+ issued_cache_op = True
111
+ if blocks_to_copy:
112
+ self.cache_engine.copy(blocks_to_copy)
113
+ issued_cache_op = True
114
+
115
+ cache_events = self.cache_events if issued_cache_op else None
116
+
117
+ # Wait for cache operations to finish.
118
+ if cache_events is not None:
119
+ raise NotImplementedError(
120
+ "cache operations are not implemented for neuron backend.")
121
+
122
+ @torch.inference_mode()
123
+ def execute_model(
124
+ self,
125
+ seq_group_metadata_list: Optional[List[SequenceGroupMetadata]] = None,
126
+ blocks_to_swap_in: Optional[Dict[int, int]] = None,
127
+ blocks_to_swap_out: Optional[Dict[int, int]] = None,
128
+ blocks_to_copy: Optional[Dict[int, List[int]]] = None,
129
+ ) -> Optional[SamplerOutput]:
130
+ if self.is_driver_worker:
131
+ assert seq_group_metadata_list is not None
132
+ num_seq_groups = len(seq_group_metadata_list)
133
+ assert blocks_to_swap_in is not None
134
+ assert blocks_to_swap_out is not None
135
+ assert blocks_to_copy is not None
136
+ data = {
137
+ "num_seq_groups": num_seq_groups,
138
+ "blocks_to_swap_in": blocks_to_swap_in,
139
+ "blocks_to_swap_out": blocks_to_swap_out,
140
+ "blocks_to_copy": blocks_to_copy,
141
+ }
142
+ broadcast_tensor_dict(data, src=0)
143
+ else:
144
+ data = broadcast_tensor_dict(src=0)
145
+ num_seq_groups = data["num_seq_groups"]
146
+ blocks_to_swap_in = data["blocks_to_swap_in"]
147
+ blocks_to_swap_out = data["blocks_to_swap_out"]
148
+ blocks_to_copy = data["blocks_to_copy"]
149
+
150
+ self.cache_swap(blocks_to_swap_in, blocks_to_swap_out, blocks_to_copy)
151
+
152
+ # If there is no input, we don't need to execute the model.
153
+ if num_seq_groups == 0:
154
+ return {}
155
+
156
+ output = self.model_runner.execute_model(seq_group_metadata_list,
157
+ self.gpu_cache)
158
+ return output
159
+
160
+
161
+def _init_distributed_environment(
162
+ parallel_config: ParallelConfig,
163
+ rank: int,
164
+ distributed_init_method: Optional[str] = None,
165
+ distributed_backend: Optional[str] = None,
166
+) -> None:
167
+ """Initialize the distributed environment."""
168
+ if torch.distributed.is_initialized():
169
+ torch_world_size = torch.distributed.get_world_size()
170
+ if torch_world_size != parallel_config.world_size:
171
+ raise RuntimeError(
172
+ "torch.distributed is already initialized but the torch world "
173
+ "size does not match parallel_config.world_size "
174
+ f"({torch_world_size} vs. {parallel_config.world_size}).")
175
+ elif not distributed_init_method:
176
+ raise ValueError(
177
+ "distributed_init_method must be set if torch.distributed "
178
+ "is not already initialized")
179
+ else:
180
+ distributed_backend = distributed_backend if distributed_backend else "nccl"
181
+ torch.distributed.init_process_group(
182
+ backend=distributed_backend,
183
+ world_size=parallel_config.world_size,
184
+ rank=rank,
185
+ init_method=distributed_init_method,
186
+ )
187
+
188
+ # A small all_reduce for warmup.
189
+ torch.distributed.all_reduce(torch.zeros(1))
190
+ ensure_model_parallel_initialized(parallel_config.tensor_parallel_size,
191
+ parallel_config.pipeline_parallel_size)