@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 3b7178cfa4
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

[Neuron] Support inference with transformers-neuronx (#2569)

Liangfu Chen committed Feb 28, 2024 at 09:34 UTC 3b7178cfa4a317922d4aef9dd3b2647b8d950e7d
18 files changed +516 -42
examples/offline_inference_neuron.py new
+33
@@ -0,0 +1,33 @@
1 +from vllm import LLM, SamplingParams
2 +
3 +# Sample prompts.
4 +prompts = [
5 + "Hello, my name is",
6 + "The president of the United States is",
7 + "The capital of France is",
8 + "The future of AI is",
9 +]
10 +# Create a sampling params object.
11 +sampling_params = SamplingParams(temperature=0.8, top_p=0.95)
12 +
13 +# Create an LLM.
14 +llm = LLM(
15 + model="openlm-research/open_llama_3b",
16 + max_num_seqs=8,
17 + # The max_model_len and block_size arguments are required to be same as max sequence length,
18 + # when targeting neuron device. Currently, this is a known limitation in continuous batching
19 + # support in transformers-neuronx.
20 + # TODO(liangfu): Support paged-attention in transformers-neuronx.
21 + max_model_len=128,
22 + block_size=128,
23 + # The device can be automatically detected when AWS Neuron SDK is installed.
24 + # The device argument can be either unspecified for automated detection, or explicitly assigned.
25 + device="neuron")
26 +# Generate texts from the prompts. The output is a list of RequestOutput objects
27 +# that contain the prompt, generated text, and other information.
28 +outputs = llm.generate(prompts, sampling_params)
29 +# Print the outputs.
30 +for output in outputs:
31 + prompt = output.prompt
32 + generated_text = output.outputs[0].text
33 + print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
tests/lora/conftest.py
+5 -3
@@ -131,9 +131,11 @@ def llama_2_7b_engine_extra_embeddings() -> nn.Module:
131 cleanup()
132 get_model_old = get_model
133
134 - def get_model_patched(model_config, device_config, lora_config=None):
135 - return get_model_old(model_config, device_config,
136 - LoRAConfig(max_loras=4, max_lora_rank=8))
134 + def get_model_patched(model_config, device_config, **kwargs):
135 + return get_model_old(model_config,
136 + device_config,
137 + lora_config=LoRAConfig(max_loras=4,
138 + max_lora_rank=8))
139
140 with patch("vllm.worker.model_runner.get_model", get_model_patched):
141 engine = vllm.LLM("meta-llama/Llama-2-7b-hf", enable_lora=False)
vllm/config.py
+35 -6
@@ -8,7 +8,7 @@ from transformers import PretrainedConfig
8
9 from vllm.logger import init_logger
10 from vllm.transformers_utils.config import get_config
11 -from vllm.utils import get_cpu_memory, is_hip, get_nvcc_cuda_version
11 +from vllm.utils import get_cpu_memory, is_hip, is_neuron, get_nvcc_cuda_version
12
13 logger = init_logger(__name__)
14
@@ -380,13 +380,21 @@ class ParallelConfig:
380 disable_custom_all_reduce: bool = False,
381 ) -> None:
382 self.pipeline_parallel_size = pipeline_parallel_size
383 - self.tensor_parallel_size = tensor_parallel_size
383 + if is_neuron():
384 + # For Neuron device support, here we assign TP=1 to avoid sharding within vLLM directly.
385 + # Transformer-neuronx would take neuron_tp_degree attribute, and distribute the workload
386 + # to multiple NeuronCores.
387 + self.tensor_parallel_size = 1
388 + self.neuron_tp_degree = tensor_parallel_size
389 + else:
390 + self.tensor_parallel_size = tensor_parallel_size
391 self.worker_use_ray = worker_use_ray
392 self.max_parallel_loading_workers = max_parallel_loading_workers
393 self.disable_custom_all_reduce = disable_custom_all_reduce
394
388 - self.world_size = pipeline_parallel_size * tensor_parallel_size
389 - if self.world_size > 1:
395 + self.world_size = pipeline_parallel_size * self.tensor_parallel_size
396 + # Ray worker is not supported for Neuron backend.
397 + if self.world_size > 1 and not is_neuron():
398 self.worker_use_ray = True
399 self._verify_args()
400
@@ -465,8 +473,29 @@ class SchedulerConfig:
473
474 class DeviceConfig:
475
468 - def __init__(self, device: str = "cuda") -> None:
469 - self.device = torch.device(device)
476 + def __init__(self, device: str = "auto") -> None:
477 + if device == "auto":
478 + # Automated device type detection
479 + if torch.cuda.is_available():
480 + self.device_type = "cuda"
481 + elif is_neuron():
482 + self.device_type = "neuron"
483 + else:
484 + raise RuntimeError("No supported device detected.")
485 + else:
486 + # Device type is assigned explicitly
487 + self.device_type = device
488 +
489 + # Some device types require processing inputs on CPU
490 + if self.device_type in ["neuron"]:
491 + self.device = torch.device("cpu")
492 + else:
493 + # Set device with device type
494 + self.device = torch.device(self.device_type)
495 +
496 + @property
497 + def is_neuron(self):
498 + return self.device_type == "neuron"
499
500
501 @dataclass
vllm/engine/arg_utils.py
+7 -9
@@ -44,7 +44,7 @@ class EngineArgs:
44 lora_extra_vocab_size: int = 256
45 lora_dtype = 'auto'
46 max_cpu_loras: Optional[int] = None
47 - device: str = 'cuda'
47 + device: str = 'auto'
48
49 def __post_init__(self):
50 if self.tokenizer is None:
@@ -171,7 +171,7 @@ class EngineArgs:
171 parser.add_argument('--block-size',
172 type=int,
173 default=EngineArgs.block_size,
174 - choices=[8, 16, 32],
174 + choices=[8, 16, 32, 128],
175 help='token block size')
176 parser.add_argument('--seed',
177 type=int,
@@ -264,13 +264,11 @@ class EngineArgs:
264 help=('Maximum number of LoRAs to store in CPU memory. '
265 'Must be >= than max_num_seqs. '
266 'Defaults to max_num_seqs.'))
267 - parser.add_argument(
268 - "--device",
269 - type=str,
270 - default=EngineArgs.device,
271 - choices=["cuda"],
272 - help=('Device type for vLLM execution. '
273 - 'Currently, only CUDA-compatible devices are supported.'))
267 + parser.add_argument("--device",
268 + type=str,
269 + default=EngineArgs.device,
270 + choices=["auto", "cuda", "neuron"],
271 + help='Device type for vLLM execution.')
272 return parser
273
274 @classmethod
vllm/engine/llm_engine.py
+18 -3
@@ -3,6 +3,7 @@ from collections import defaultdict
3 import os
4 import time
5 import pickle
6 +import importlib
7 from typing import (TYPE_CHECKING, Any, Dict, Iterable, List, Optional, Tuple,
8 Union)
9
@@ -20,7 +21,8 @@ from vllm.sequence import (SamplerOutput, Sequence, SequenceGroup,
21 SequenceGroupOutput, SequenceOutput, SequenceStatus)
22 from vllm.transformers_utils.tokenizer import (detokenize_incrementally,
23 TokenizerGroup)
23 -from vllm.utils import Counter, set_cuda_visible_devices, get_ip, get_open_port, get_distributed_init_method
24 +from vllm.utils import (Counter, set_cuda_visible_devices, get_ip,
25 + get_open_port, get_distributed_init_method)
26
27 if ray:
28 from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy
@@ -31,6 +33,12 @@ if TYPE_CHECKING:
33 logger = init_logger(__name__)
34 _LOCAL_LOGGING_INTERVAL_SEC = 5
35
36 +# A map between the device type (in device config) to its worker module.
37 +DEVICE_TO_WORKER_MODULE_MAP = {
38 + "cuda": "vllm.worker.worker",
39 + "neuron": "vllm.worker.neuron_worker",
40 +}
41 +
42 # If the env var is set, it uses the Ray's compiled DAG API
43 # which optimizes the control plane overhead.
44 # Run VLLM with VLLM_USE_RAY_COMPILED_DAG=1 to enable it.
@@ -138,10 +146,17 @@ class LLMEngine:
146 def get_tokenizer_for_seq(self, sequence: Sequence):
147 return self.tokenizer.get_lora_tokenizer(sequence.lora_request)
148
149 + def _dispatch_worker(self):
150 + worker_module = DEVICE_TO_WORKER_MODULE_MAP[
151 + self.device_config.device_type]
152 + imported_worker = importlib.import_module(worker_module)
153 + Worker = imported_worker.Worker
154 + return Worker
155 +
156 def _init_workers(self):
157 # Lazy import the Worker to avoid importing torch.cuda/xformers
158 # before CUDA_VISIBLE_DEVICES is set in the Worker
144 - from vllm.worker.worker import Worker
159 + Worker = self._dispatch_worker()
160
161 assert self.parallel_config.world_size == 1, (
162 "Ray is required if parallel_config.world_size > 1.")
@@ -243,7 +258,7 @@ class LLMEngine:
258
259 # Lazy import the Worker to avoid importing torch.cuda/xformers
260 # before CUDA_VISIBLE_DEVICES is set in the Worker
246 - from vllm.worker.worker import Worker
261 + Worker = self._dispatch_worker()
262
263 # Initialize torch distributed process group for the workers.
264 model_config = copy.deepcopy(self.model_config)
vllm/lora/layers.py
+4
@@ -795,6 +795,10 @@ class SamplerWithLoRA(BaseLayerWithLoRA):
795 self.dtype = dtype
796 self.device = device
797
798 + @property
799 + def logits_as_hidden_states(self):
800 + return self.base_layer.logits_as_hidden_states
801 +
802 @property
803 def vocab_size(self):
804 return self.base_layer.vocab_size
vllm/model_executor/__init__.py
+1 -2
@@ -1,7 +1,6 @@
1 from vllm.model_executor.input_metadata import InputMetadata
2 -from vllm.model_executor.model_loader import get_model
2 from vllm.model_executor.sampling_metadata import SamplingMetadata
4 -from vllm.model_executor.utils import set_random_seed
3 +from vllm.model_executor.utils import set_random_seed, get_model
4
5 __all__ = [
6 "InputMetadata",
vllm/model_executor/layers/sampler.py
+13 -5
@@ -10,6 +10,7 @@ from vllm.model_executor.sampling_metadata import SamplingMetadata, SamplingTens
10 from vllm.sampling_params import SamplingParams, SamplingType
11 from vllm.sequence import (PromptLogprobs, SampleLogprobs, SamplerOutput,
12 SequenceData, SequenceGroupOutput, SequenceOutput)
13 +from vllm.utils import is_neuron
14
15
16 class Sampler(nn.Module):
@@ -32,6 +33,8 @@ class Sampler(nn.Module):
33 org_vocab_size: Optional[int] = None) -> None:
34 super().__init__()
35 self.vocab_size = vocab_size
36 + # Transformers-neuronx generate outputs as logits directly.
37 + self.logits_as_hidden_states = is_neuron()
38 # original vocabulary size (without LoRA).
39 self.org_vocab_size = org_vocab_size or vocab_size
40
@@ -55,10 +58,14 @@ class Sampler(nn.Module):
58 embedding_bias: Optional[torch.Tensor] = None,
59 ) -> Optional[SamplerOutput]:
60 # Get the hidden states that we use for sampling.
58 - hidden_states = _prune_hidden_states(hidden_states, sampling_metadata)
61 + if self.logits_as_hidden_states:
62 + logits = hidden_states
63 + else:
64 + hidden_states = _prune_hidden_states(hidden_states,
65 + sampling_metadata)
66
60 - # Get the logits for the next tokens.
61 - logits = self._get_logits(hidden_states, embedding, embedding_bias)
67 + # Get the logits for the next tokens.
68 + logits = self._get_logits(hidden_states, embedding, embedding_bias)
69
70 # Only perform sampling in the driver worker.
71 # Note: `_get_logits` is still distributed across TP workers because
@@ -395,7 +402,8 @@ def _sample(
402 sample_metadata[sampling_type] = (seq_group_ids, seq_groups,
403 is_prompts, sample_indices)
404 if sampling_type == SamplingType.GREEDY:
398 - greedy_samples = torch.argmax(logprobs[sample_indices], dim=-1)
405 + greedy_samples = torch.argmax(logprobs[sample_indices.long()],
406 + dim=-1)
407 elif sampling_type in (SamplingType.RANDOM, SamplingType.RANDOM_SEED):
408 max_best_of = 1
409 for seq_group, is_prompt in zip(seq_groups, is_prompts):
@@ -407,7 +415,7 @@ def _sample(
415 "generators": sampling_metadata.generators,
416 }
417 multinomial_samples[sampling_type] = _multinomial(
410 - probs[sample_indices], max_best_of, **seeded_args)
418 + probs[sample_indices.long()], max_best_of, **seeded_args)
419 elif sampling_type == SamplingType.BEAM:
420 beam_search_logprobs = logprobs[sample_indices]
421 else:
vllm/model_executor/model_loader.py
+5 -5
@@ -1,11 +1,11 @@
1 """Utilities for selecting and loading models."""
2 import contextlib
3 -from typing import Optional, Type
3 +from typing import Type
4
5 import torch
6 import torch.nn as nn
7
8 -from vllm.config import DeviceConfig, ModelConfig, LoRAConfig
8 +from vllm.config import DeviceConfig, ModelConfig
9 from vllm.model_executor.models import ModelRegistry
10 from vllm.model_executor.weight_utils import (get_quant_config,
11 initialize_dummy_weights)
@@ -37,9 +37,9 @@ def _get_model_architecture(model_config: ModelConfig) -> Type[nn.Module]:
37 f"Supported architectures: {ModelRegistry.get_supported_archs()}")
38
39
40 -def get_model(model_config: ModelConfig,
41 - device_config: DeviceConfig,
42 - lora_config: Optional[LoRAConfig] = None) -> nn.Module:
40 +def get_model(model_config: ModelConfig, device_config: DeviceConfig,
41 + **kwargs) -> nn.Module:
42 + lora_config = kwargs.get("lora_config", None)
43 model_class = _get_model_architecture(model_config)
44
45 # Get the (maybe quantized) linear method.
vllm/model_executor/models/__init__.py
+11 -1
@@ -4,7 +4,7 @@ from typing import List, Optional, Type
4 import torch.nn as nn
5
6 from vllm.logger import init_logger
7 -from vllm.utils import is_hip
7 +from vllm.utils import is_hip, is_neuron
8
9 logger = init_logger(__name__)
10
@@ -61,6 +61,9 @@ _ROCM_PARTIALLY_SUPPORTED_MODELS = {
61 "Sliding window attention is not yet supported in ROCm's flash attention",
62 }
63
64 +# Models not supported by Neuron.
65 +_NEURON_SUPPORTED_MODELS = {"LlamaForCausalLM": "neuron.llama"}
66 +
67
68 class ModelRegistry:
69
@@ -77,8 +80,15 @@ class ModelRegistry:
80 logger.warning(
81 f"Model architecture {model_arch} is partially supported "
82 "by ROCm: " + _ROCM_PARTIALLY_SUPPORTED_MODELS[model_arch])
83 + elif is_neuron():
84 + if model_arch not in _NEURON_SUPPORTED_MODELS:
85 + raise ValueError(
86 + f"Model architecture {model_arch} is not supported by "
87 + "Neuron for now.")
88
89 module_name, model_cls_name = _MODELS[model_arch]
90 + if is_neuron():
91 + module_name = _NEURON_SUPPORTED_MODELS[model_arch]
92 module = importlib.import_module(
93 f"vllm.model_executor.models.{module_name}")
94 return getattr(module, model_cls_name, None)
vllm/model_executor/models/neuron/llama.py new
+79
@@ -0,0 +1,79 @@
1 +"""Inference-only LLaMA model compatible with HuggingFace weights."""
2 +import os
3 +from typing import List, Optional, Tuple
4 +
5 +import torch
6 +from torch import nn
7 +from transformers import LlamaConfig
8 +
9 +from vllm.model_executor.input_metadata import InputMetadata
10 +from vllm.model_executor.layers.sampler import Sampler
11 +from vllm.model_executor.sampling_metadata import SamplingMetadata
12 +from vllm.sequence import SamplerOutput
13 +
14 +KVCache = Tuple[torch.Tensor, torch.Tensor]
15 +
16 +
17 +class LlamaForCausalLM(nn.Module):
18 +
19 + def __init__(
20 + self,
21 + config: LlamaConfig,
22 + linear_method=None,
23 + ) -> None:
24 + super().__init__()
25 + self.config = config
26 + self.linear_method = linear_method
27 + self.model = None
28 + self.sampler = Sampler(config.vocab_size)
29 +
30 + def forward(
31 + self,
32 + input_ids: torch.Tensor,
33 + positions: torch.Tensor,
34 + kv_caches: List[KVCache],
35 + input_metadata: InputMetadata,
36 + ) -> torch.Tensor:
37 + with torch.inference_mode():
38 + block_size = self.model.context_buckets[-1]
39 + if input_metadata.is_prompt:
40 + seq_ids = input_metadata.slot_mapping[:, 0] // block_size
41 + else:
42 + seq_ids = input_metadata.block_tables
43 + logits = self.model(input_ids,
44 + cache_ids=positions,
45 + start_ids=seq_ids.flatten())
46 + return logits
47 +
48 + def sample(
49 + self,
50 + hidden_states: torch.Tensor,
51 + sampling_metadata: SamplingMetadata,
52 + ) -> Optional[SamplerOutput]:
53 + next_tokens = self.sampler(self.model.chkpt_model.lm_head,
54 + hidden_states, sampling_metadata)
55 + return next_tokens
56 +
57 + def load_weights(self,
58 + model_name_or_path: str,
59 + cache_dir: Optional[str] = None,
60 + load_format: str = "auto",
61 + revision: Optional[str] = None,
62 + **kwargs):
63 + from transformers_neuronx.llama.model import LlamaForSampling
64 +
65 + split_model_dir = f"{model_name_or_path}-split"
66 + if os.path.isdir(os.path.join(model_name_or_path,
67 + "pytorch_model.bin")):
68 + split_model_dir = model_name_or_path
69 + elif not os.path.exists(f"{model_name_or_path}-split"):
70 + from transformers.models.llama import LlamaForCausalLM
71 + from transformers_neuronx.module import save_pretrained_split
72 +
73 + hf_model = LlamaForCausalLM.from_pretrained(model_name_or_path,
74 + low_cpu_mem_usage=True)
75 + save_pretrained_split(hf_model, f"{model_name_or_path}-split")
76 +
77 + self.model = LlamaForSampling.from_pretrained(split_model_dir,
78 + **kwargs)
79 + self.model.to_neuron()
vllm/model_executor/neuron_model_loader.py new
+66
@@ -0,0 +1,66 @@
1 +"""Utilities for selecting and loading models."""
2 +from typing import Type
3 +
4 +import torch
5 +import torch.nn as nn
6 +from transformers import PretrainedConfig
7 +
8 +from vllm.config import ModelConfig, DeviceConfig
9 +from vllm.model_executor.models import ModelRegistry
10 +
11 +TORCH_DTYPE_TO_NEURON_AMP = {
12 + "auto": "f32",
13 + "half": "f16",
14 + "float16": "f16",
15 + "bfloat16": "bf16",
16 + "float": "f32",
17 + "float32": "f32",
18 + torch.float16: "f16",
19 + torch.bfloat16: "bf16",
20 + torch.float32: "f32",
21 +}
22 +
23 +
24 +def _get_model_architecture(config: PretrainedConfig) -> Type[nn.Module]:
25 + architectures = getattr(config, "architectures", [])
26 + for arch in architectures:
27 + model_cls = ModelRegistry.load_model_cls(arch)
28 + if model_cls is not None:
29 + return model_cls
30 + raise ValueError(
31 + f"Model architectures {architectures} are not supported for now. "
32 + f"Supported architectures: {ModelRegistry.get_supported_archs()}")
33 +
34 +
35 +def get_model(model_config: ModelConfig, device_config: DeviceConfig,
36 + **kwargs) -> nn.Module:
37 + from transformers_neuronx.config import NeuronConfig, ContinuousBatchingConfig
38 +
39 + parallel_config = kwargs.get("parallel_config")
40 + scheduler_config = kwargs.get("scheduler_config")
41 +
42 + model_class = _get_model_architecture(model_config.hf_config)
43 + linear_method = None
44 +
45 + # Create a model instance.
46 + model = model_class(model_config.hf_config, linear_method)
47 +
48 + continuous_batching_config = ContinuousBatchingConfig(
49 + batch_size_for_shared_caches=scheduler_config.max_num_seqs)
50 + neuron_config = NeuronConfig(
51 + continuous_batching=continuous_batching_config)
52 +
53 + # Load the weights from the cached or downloaded files.
54 + model.load_weights(
55 + model_config.model,
56 + model_config.download_dir,
57 + model_config.load_format,
58 + model_config.revision,
59 + tp_degree=parallel_config.neuron_tp_degree,
60 + amp=TORCH_DTYPE_TO_NEURON_AMP[model_config.dtype],
61 + neuron_config=neuron_config,
62 + context_length_estimate=[scheduler_config.max_model_len],
63 + n_positions=[scheduler_config.max_model_len],
64 + batch_size=scheduler_config.max_num_seqs)
65 +
66 + return model.eval()
vllm/model_executor/sampling_metadata.py
+2 -2
@@ -5,7 +5,7 @@ import torch
5
6 from vllm.sampling_params import SamplingParams, SamplingType
7 from vllm.sequence import SequenceData
8 -from vllm.utils import in_wsl
8 +from vllm.utils import in_wsl, is_neuron
9
10 _SAMPLING_EPS = 1e-5
11
@@ -155,7 +155,7 @@ class SamplingTensors:
155 dtype: torch.dtype) -> "SamplingTensors":
156 # Note that the performance will be very bad without
157 # pinned memory.
158 - pin_memory = not in_wsl()
158 + pin_memory = not in_wsl() and not is_neuron()
159 prompt_max_len = max(len(tokens) for tokens in prompt_tokens)
160 prompt_padded_tokens = [
161 tokens + [vocab_size] * (prompt_max_len - len(tokens))
vllm/model_executor/utils.py
+17
@@ -1,10 +1,18 @@
1 """Utils for model executor."""
2 import random
3 +import importlib
4 from typing import Any, Dict, Optional
5
6 import numpy as np
7 import torch
8
9 +from vllm.config import DeviceConfig, ModelConfig
10 +
11 +DEVICE_TO_MODEL_LOADER_MAP = {
12 + "cuda": "model_loader",
13 + "neuron": "neuron_model_loader",
14 +}
15 +
16
17 def set_random_seed(seed: int) -> None:
18 random.seed(seed)
@@ -33,3 +41,12 @@ def set_weight_attrs(
41 assert not hasattr(
42 weight, key), (f"Overwriting existing tensor attribute: {key}")
43 setattr(weight, key, value)
44 +
45 +
46 +def get_model(model_config: ModelConfig, device_config: DeviceConfig,
47 + **kwargs) -> torch.nn.Module:
48 + model_loader_module = DEVICE_TO_MODEL_LOADER_MAP[device_config.device_type]
49 + imported_model_loader = importlib.import_module(
50 + f"vllm.model_executor.{model_loader_module}")
51 + get_model_fn = imported_model_loader.get_model
52 + return get_model_fn(model_config, device_config, **kwargs)
vllm/utils.py
+8
@@ -118,6 +118,14 @@ def is_hip() -> bool:
118 return torch.version.hip is not None
119
120
121 +def is_neuron() -> bool:
122 + try:
123 + import transformers_neuronx
124 + except ImportError:
125 + transformers_neuronx = None
126 + return transformers_neuronx is not None
127 +
128 +
129 def get_max_shared_memory_bytes(gpu: int = 0) -> int:
130 """Returns the maximum shared memory per thread block in bytes."""
131 # NOTE: This import statement should be executed lazily since
vllm/worker/cache_engine.py
+9 -2
@@ -3,10 +3,9 @@ from typing import Dict, List, Tuple
3
4 import torch
5
6 -from vllm._C import cache_ops
6 from vllm.config import CacheConfig, ModelConfig, ParallelConfig
7 from vllm.logger import init_logger
9 -from vllm.utils import in_wsl, STR_DTYPE_TO_TORCH_DTYPE
8 +from vllm.utils import in_wsl, is_neuron, STR_DTYPE_TO_TORCH_DTYPE
9
10 logger = init_logger(__name__)
11
@@ -39,6 +38,10 @@ class CacheEngine:
38 self.num_gpu_blocks = cache_config.num_gpu_blocks
39 self.num_cpu_blocks = cache_config.num_cpu_blocks
40
41 + # Skip initializing CUDA stream and buffer for Neuron backend.
42 + if is_neuron():
43 + return
44 +
45 if cache_config.cache_dtype == "auto":
46 self.dtype = model_config.dtype
47 else:
@@ -121,6 +124,8 @@ class CacheEngine:
124 dst: List[KVCache],
125 src_to_dst: Dict[int, int],
126 ) -> None:
127 + from vllm._C import cache_ops
128 +
129 with torch.cuda.stream(self.cache_stream):
130 for i in range(self.num_layers):
131 src_key_cache, src_value_cache = src[i]
@@ -140,6 +145,8 @@ class CacheEngine:
145 self._swap(self.gpu_cache, self.cpu_cache, src_to_dst)
146
147 def copy(self, src_to_dsts: Dict[int, List[int]]) -> None:
148 + from vllm._C import cache_ops
149 +
150 key_caches = [key_cache for key_cache, _ in self.gpu_cache]
151 value_caches = [value_cache for _, value_cache in self.gpu_cache]
152 # NOTE(woosuk): This operation implicitly synchronizes the CPU and GPU.
vllm/worker/model_runner.py
+12 -4
@@ -80,9 +80,16 @@ class ModelRunner:
80 self.in_wsl = in_wsl()
81 self.kv_cache_dtype = kv_cache_dtype
82
83 + # Set enforce_eager to True for Neuron backend, to avoid capturing graph
84 + if self.device_config.is_neuron:
85 + self.model_config.enforce_eager = True
86 +
87 def load_model(self) -> None:
84 - self.model = get_model(self.model_config, self.device_config,
85 - self.lora_config)
88 + self.model = get_model(self.model_config,
89 + self.device_config,
90 + lora_config=self.lora_config,
91 + parallel_config=self.parallel_config,
92 + scheduler_config=self.scheduler_config)
93
94 vocab_size = self.model.config.vocab_size
95
@@ -393,6 +400,7 @@ class ModelRunner:
400 selected_token_start_idx = 0
401 categorized_sample_indices = {t: [] for t in SamplingType}
402 categorized_sample_indices_start_idx = 0
403 + pin_memory = not self.in_wsl and not self.device_config.is_neuron
404
405 max_subquery_len = max(subquery_lens) if subquery_lens else 1
406 for i, seq_group_metadata in enumerate(seq_group_metadata_list):
@@ -443,12 +451,12 @@ class ModelRunner:
451 selected_token_indices = _async_h2d(selected_token_indices,
452 dtype=torch.long,
453 target_device=self.device,
446 - pin_memory=not self.in_wsl)
454 + pin_memory=pin_memory)
455 categorized_sample_indices = {
456 t: _async_h2d(seq_ids,
457 dtype=torch.int,
458 target_device=self.device,
451 - pin_memory=not self.in_wsl)
459 + pin_memory=pin_memory)
460 for t, seq_ids in categorized_sample_indices.items()
461 }
462
vllm/worker/neuron_worker.py new
+191
@@ -0,0 +1,191 @@
1 +"""A Neuron worker class."""
2 +from typing import Dict, List, Optional, Tuple
3 +
4 +import torch
5 +import torch.distributed
6 +
7 +from vllm.config import (CacheConfig, DeviceConfig, ModelConfig,
8 + ParallelConfig, SchedulerConfig, LoRAConfig)
9 +from vllm.model_executor import set_random_seed
10 +from vllm.model_executor.parallel_utils.communication_op import (
11 + broadcast_tensor_dict)
12 +from vllm.model_executor.parallel_utils.parallel_state import (
13 + ensure_model_parallel_initialized)
14 +from vllm.sequence import SamplerOutput, SequenceGroupMetadata
15 +from vllm.worker.cache_engine import CacheEngine
16 +from vllm.worker.model_runner import ModelRunner
17 +
18 +
19 +class Worker:
20 + """A worker class that executes the model on a group of neuron cores.
21 + """
22 +
23 + def __init__(
24 + self,
25 + model_config: ModelConfig,
26 + parallel_config: ParallelConfig,
27 + scheduler_config: SchedulerConfig,
28 + device_config: DeviceConfig,
29 + local_rank: int,
30 + rank: int,
31 + distributed_init_method: str,
32 + lora_config: Optional[LoRAConfig] = None,
33 + kv_cache_dtype: Optional[str] = "auto",
34 + is_driver_worker: bool = False,
35 + ) -> None:
36 + self.model_config = model_config
37 + self.parallel_config = parallel_config
38 + self.scheduler_config = scheduler_config
39 + self.device_config = device_config
40 + self.local_rank = local_rank
41 + self.rank = rank
42 + self.distributed_init_method = distributed_init_method
43 + self.lora_config = lora_config
44 + self.is_driver_worker = is_driver_worker
45 + if self.is_driver_worker:
46 + assert self.rank == 0, "The driver worker must have rank 0."
47 +
48 + self.model_runner = ModelRunner(model_config,
49 + parallel_config,
50 + scheduler_config,
51 + device_config,
52 + lora_config=self.lora_config,
53 + is_driver_worker=is_driver_worker)
54 + # Uninitialized cache engine. Will be initialized by
55 + # self.init_cache_engine().
56 + self.cache_config = None
57 + self.cache_engine = None
58 + self.cache_events = None
59 + self.gpu_cache = None
60 +
61 + def init_model(self) -> None:
62 + # Initialize the distributed environment.
63 + _init_distributed_environment(self.parallel_config,
64 + self.rank,
65 + self.distributed_init_method,
66 + distributed_backend="gloo")
67 +
68 + # Initialize the model.
69 + set_random_seed(self.model_config.seed)
70 +
71 + def load_model(self):
72 + self.model_runner.load_model()
73 +
74 + @torch.inference_mode()
75 + def profile_num_available_blocks(
76 + self,
77 + block_size: int = 128,
78 + gpu_memory_utilization: float = 0.9,
79 + cpu_swap_space: int = 0,
80 + cache_dtype: str = "float16",
81 + ) -> Tuple[int, int]:
82 + """Simply returns max_num_seqs as num_gpu_blocks, 0 as num_cpu_blocks."""
83 + num_gpu_blocks = self.scheduler_config.max_num_seqs
84 + num_cpu_blocks = 0
85 + return num_gpu_blocks, num_cpu_blocks
86 +
87 + def init_cache_engine(self, cache_config: CacheConfig) -> None:
88 + self.cache_config = cache_config
89 + self.cache_engine = CacheEngine(self.cache_config, self.model_config,
90 + self.parallel_config)
91 + self.model_runner.set_block_size(self.cache_engine.block_size)
92 +
93 + def warm_up_model(self) -> None:
94 + # Warm up is maintained in transformers-neuronx
95 + pass
96 +
97 + def cache_swap(
98 + self,
99 + blocks_to_swap_in: Dict[int, int],
100 + blocks_to_swap_out: Dict[int, int],
101 + blocks_to_copy: Dict[int, List[int]],
102 + ) -> None:
103 + # Issue cache operations.
104 + issued_cache_op = False
105 + if blocks_to_swap_in:
106 + self.cache_engine.swap_in(blocks_to_swap_in)
107 + issued_cache_op = True
108 + if blocks_to_swap_out:
109 + self.cache_engine.swap_out(blocks_to_swap_out)
110 + issued_cache_op = True
111 + if blocks_to_copy:
112 + self.cache_engine.copy(blocks_to_copy)
113 + issued_cache_op = True
114 +
115 + cache_events = self.cache_events if issued_cache_op else None
116 +
117 + # Wait for cache operations to finish.
118 + if cache_events is not None:
119 + raise NotImplementedError(
120 + "cache operations are not implemented for neuron backend.")
121 +
122 + @torch.inference_mode()
123 + def execute_model(
124 + self,
125 + seq_group_metadata_list: Optional[List[SequenceGroupMetadata]] = None,
126 + blocks_to_swap_in: Optional[Dict[int, int]] = None,
127 + blocks_to_swap_out: Optional[Dict[int, int]] = None,
128 + blocks_to_copy: Optional[Dict[int, List[int]]] = None,
129 + ) -> Optional[SamplerOutput]:
130 + if self.is_driver_worker:
131 + assert seq_group_metadata_list is not None
132 + num_seq_groups = len(seq_group_metadata_list)
133 + assert blocks_to_swap_in is not None
134 + assert blocks_to_swap_out is not None
135 + assert blocks_to_copy is not None
136 + data = {
137 + "num_seq_groups": num_seq_groups,
138 + "blocks_to_swap_in": blocks_to_swap_in,
139 + "blocks_to_swap_out": blocks_to_swap_out,
140 + "blocks_to_copy": blocks_to_copy,
141 + }
142 + broadcast_tensor_dict(data, src=0)
143 + else:
144 + data = broadcast_tensor_dict(src=0)
145 + num_seq_groups = data["num_seq_groups"]
146 + blocks_to_swap_in = data["blocks_to_swap_in"]
147 + blocks_to_swap_out = data["blocks_to_swap_out"]
148 + blocks_to_copy = data["blocks_to_copy"]
149 +
150 + self.cache_swap(blocks_to_swap_in, blocks_to_swap_out, blocks_to_copy)
151 +
152 + # If there is no input, we don't need to execute the model.
153 + if num_seq_groups == 0:
154 + return {}
155 +
156 + output = self.model_runner.execute_model(seq_group_metadata_list,
157 + self.gpu_cache)
158 + return output
159 +
160 +
161 +def _init_distributed_environment(
162 + parallel_config: ParallelConfig,
163 + rank: int,
164 + distributed_init_method: Optional[str] = None,
165 + distributed_backend: Optional[str] = None,
166 +) -> None:
167 + """Initialize the distributed environment."""
168 + if torch.distributed.is_initialized():
169 + torch_world_size = torch.distributed.get_world_size()
170 + if torch_world_size != parallel_config.world_size:
171 + raise RuntimeError(
172 + "torch.distributed is already initialized but the torch world "
173 + "size does not match parallel_config.world_size "
174 + f"({torch_world_size} vs. {parallel_config.world_size}).")
175 + elif not distributed_init_method:
176 + raise ValueError(
177 + "distributed_init_method must be set if torch.distributed "
178 + "is not already initialized")
179 + else:
180 + distributed_backend = distributed_backend if distributed_backend else "nccl"
181 + torch.distributed.init_process_group(
182 + backend=distributed_backend,
183 + world_size=parallel_config.world_size,
184 + rank=rank,
185 + init_method=distributed_init_method,
186 + )
187 +
188 + # A small all_reduce for warmup.
189 + torch.distributed.all_reduce(torch.zeros(1))
190 + ensure_model_parallel_initialized(parallel_config.tensor_parallel_size,
191 + parallel_config.pipeline_parallel_size)