@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e90fc21f2e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

[Hardware][Neuron] Refactor neuron support (#3471)

Zhuohan Li committed Mar 21, 2024 at 18:22 UTC e90fc21f2eda7e53f692398ee2c0cb5a0ac19693
33 files changed +615 -549
examples/offline_inference_neuron.py
+3 -2
@@ -12,7 +12,7 @@ sampling_params = SamplingParams(temperature=0.8, top_p=0.95)
12
13 # Create an LLM.
14 llm = LLM(
15 - model="openlm-research/open_llama_3b",
15 + model="TinyLlama/TinyLlama-1.1B-Chat-v1.0",
16 max_num_seqs=8,
17 # The max_model_len and block_size arguments are required to be same as
18 # max sequence length when targeting neuron device.
@@ -24,7 +24,8 @@ llm = LLM(
24 # The device can be automatically detected when AWS Neuron SDK is installed.
25 # The device argument can be either unspecified for automated detection,
26 # or explicitly assigned.
27 - device="neuron")
27 + device="neuron",
28 + tensor_parallel_size=2)
29 # Generate texts from the prompts. The output is a list of RequestOutput objects
30 # that contain the prompt, generated text, and other information.
31 outputs = llm.generate(prompts, sampling_params)
tests/lora/test_worker.py
+1 -1
@@ -33,7 +33,7 @@ def test_worker_apply_lora(sql_lora_files):
33 max_loras=32),
34 distributed_init_method=f"file://{tempfile.mkstemp()[1]}",
35 )
36 - worker.init_model()
36 + worker.init_device()
37 worker.load_model()
38
39 worker.model_runner.set_active_loras([], LoRAMapping([], []))
tests/spec_decode/test_spec_decode_worker.py
+9 -9
@@ -71,7 +71,7 @@ def test_correctly_calls_target_model(k: int, batch_size: int):
71
72 worker = SpecDecodeWorker(draft_worker, target_worker, rejection_sampler,
73 metrics_collector)
74 - worker.init_model()
74 + worker.init_device()
75
76 vocab_size = 32_000
77
@@ -151,7 +151,7 @@ def test_correctly_calls_rejection_sampler(k: int, batch_size: int):
151
152 worker = SpecDecodeWorker(draft_worker, target_worker, rejection_sampler,
153 metrics_collector)
154 - worker.init_model()
154 + worker.init_device()
155
156 proposal_token_ids = torch.randint(low=0,
157 high=vocab_size,
@@ -230,7 +230,7 @@ def test_correctly_formats_output(k: int, batch_size: int):
230
231 worker = SpecDecodeWorker(draft_worker, target_worker, rejection_sampler,
232 metrics_collector)
233 - worker.init_model()
233 + worker.init_device()
234
235 proposal_token_ids = torch.randint(low=0,
236 high=vocab_size,
@@ -342,7 +342,7 @@ def test_collects_metrics(k: int, batch_size: int, returns_metrics: bool):
342
343 worker = SpecDecodeWorker(draft_worker, target_worker, rejection_sampler,
344 metrics_collector)
345 - worker.init_model()
345 + worker.init_device()
346
347 proposal_token_ids = torch.randint(low=0,
348 high=vocab_size,
@@ -486,8 +486,8 @@ def test_empty_input_batch(k: int, batch_size: int):
486
487
488 @torch.inference_mode()
489 -def test_init_model():
490 - """Verify SpecDecodeWorker invokes proposer/scorer worker init_model, as
489 +def test_init_device():
490 + """Verify SpecDecodeWorker invokes proposer/scorer worker init_device, as
491 well as other GPU initialization.
492 """
493 draft_worker = mock_worker(cls=MultiStepWorker)
@@ -499,11 +499,11 @@ def test_init_model():
499 worker = SpecDecodeWorker(draft_worker, target_worker, rejection_sampler,
500 metrics_collector)
501
502 - worker.init_model()
502 + worker.init_device()
503
504 - draft_worker.init_model.assert_called_once()
504 + draft_worker.init_device.assert_called_once()
505
506 - target_worker.init_model.assert_called_once()
506 + target_worker.init_device.assert_called_once()
507
508 metrics_collector.init_gpu_tensors.assert_called_once()
509 rejection_sampler.init_gpu_tensors.assert_called_once()
tests/spec_decode/utils.py
+1 -1
@@ -123,7 +123,7 @@ def create_worker(cls: type,
123 is_driver_worker=is_driver_worker,
124 )
125
126 - worker.init_model()
126 + worker.init_device()
127 worker.load_model()
128
129 cache_config.num_gpu_blocks = num_gpu_blocks
tests/worker/test_swap.py
+1 -1
@@ -30,7 +30,7 @@ def test_swap() -> None:
30 )
31
32 # Initialize the worker.
33 - worker.init_model()
33 + worker.init_device()
34 worker.load_model()
35 worker.init_cache_engine(cache_config)
36 worker.warm_up_model()
vllm/config.py
+2 -15
@@ -474,15 +474,7 @@ class ParallelConfig:
474 placement_group: Optional["PlacementGroup"] = None,
475 ) -> None:
476 self.pipeline_parallel_size = pipeline_parallel_size
477 - if is_neuron():
478 - # For Neuron device support, here we assign TP=1 to avoid sharding
479 - # within vLLM directly. Transformer-neuronx would take
480 - # neuron_tp_degree attribute, and distribute the workload
481 - # to multiple NeuronCores.
482 - self.tensor_parallel_size = 1
483 - self.neuron_tp_degree = tensor_parallel_size
484 - else:
485 - self.tensor_parallel_size = tensor_parallel_size
477 + self.tensor_parallel_size = tensor_parallel_size
478 self.worker_use_ray = worker_use_ray
479 self.max_parallel_loading_workers = max_parallel_loading_workers
480 self.disable_custom_all_reduce = disable_custom_all_reduce
@@ -491,8 +483,7 @@ class ParallelConfig:
483 self.placement_group = placement_group
484
485 self.world_size = pipeline_parallel_size * self.tensor_parallel_size
494 - # Ray worker is not supported for Neuron backend.
495 - if self.world_size > 1 and not is_neuron():
486 + if self.world_size > 1:
487 self.worker_use_ray = True
488 self._verify_args()
489
@@ -591,10 +582,6 @@ class DeviceConfig:
582 # Set device with device type
583 self.device = torch.device(self.device_type)
584
594 - @property
595 - def is_neuron(self):
596 - return self.device_type == "neuron"
597 -
585
586 @dataclass
587 class LoRAConfig:
vllm/engine/async_llm_engine.py
+6 -1
@@ -325,7 +325,12 @@ class AsyncLLMEngine:
325 # Create the engine configs.
326 engine_configs = engine_args.create_engine_configs()
327 parallel_config = engine_configs[2]
328 - if parallel_config.worker_use_ray or engine_args.engine_use_ray:
328 + device_config = engine_configs[4]
329 +
330 + if device_config.device_type == "neuron":
331 + raise NotImplementedError("Neuron is not supported for "
332 + "async engine yet.")
333 + elif parallel_config.worker_use_ray or engine_args.engine_use_ray:
334 initialize_ray_cluster(parallel_config)
335 from vllm.executor.ray_gpu_executor import RayGPUExecutorAsync
336 executor_class = RayGPUExecutorAsync
vllm/engine/llm_engine.py
+5 -1
@@ -125,9 +125,13 @@ class LLMEngine:
125 # Create the engine configs.
126 engine_configs = engine_args.create_engine_configs()
127 parallel_config = engine_configs[2]
128 + device_config = engine_configs[4]
129
130 # Initialize the cluster and specify the executor class.
130 - if parallel_config.worker_use_ray:
131 + if device_config.device_type == "neuron":
132 + from vllm.executor.neuron_executor import NeuronExecutor
133 + executor_class = NeuronExecutor
134 + elif parallel_config.worker_use_ray:
135 initialize_ray_cluster(parallel_config)
136 from vllm.executor.ray_gpu_executor import RayGPUExecutor
137 executor_class = RayGPUExecutor
vllm/executor/gpu_executor.py
+2 -16
@@ -1,4 +1,3 @@
1 -import importlib
1 from typing import Dict, List, Optional
2
3 from vllm.lora.request import LoRARequest
@@ -13,12 +12,6 @@ from vllm.utils import (get_ip, get_open_port, get_distributed_init_method,
12
13 logger = init_logger(__name__)
14
16 -# A map between the device type (in device config) to its worker module.
17 -DEVICE_TO_WORKER_MODULE_MAP = {
18 - "cuda": "vllm.worker.worker",
19 - "neuron": "vllm.worker.neuron_worker",
20 -}
21 -
15
16 class GPUExecutor(ExecutorBase):
17
@@ -44,17 +37,10 @@ class GPUExecutor(ExecutorBase):
37 # Profile the memory usage and initialize the cache.
38 self._init_cache()
39
47 - def _dispatch_worker(self):
48 - worker_module = DEVICE_TO_WORKER_MODULE_MAP[
49 - self.device_config.device_type]
50 - imported_worker = importlib.import_module(worker_module)
51 - Worker = imported_worker.Worker
52 - return Worker
53 -
40 def _init_worker(self):
41 # Lazy import the Worker to avoid importing torch.cuda/xformers
42 # before CUDA_VISIBLE_DEVICES is set in the Worker
57 - Worker = self._dispatch_worker()
43 + from vllm.worker.worker import Worker
44
45 assert self.parallel_config.world_size == 1, (
46 "GPUExecutor only supports single GPU.")
@@ -73,7 +59,7 @@ class GPUExecutor(ExecutorBase):
59 kv_cache_dtype=self.cache_config.cache_dtype,
60 is_driver_worker=True,
61 )
76 - self.driver_worker.init_model()
62 + self.driver_worker.init_device()
63 self.driver_worker.load_model()
64
65 def _init_cache(self) -> None:
vllm/executor/neuron_executor.py new
+80
@@ -0,0 +1,80 @@
1 +from typing import Dict, List, Optional
2 +
3 +from vllm.lora.request import LoRARequest
4 +from vllm.config import (CacheConfig, DeviceConfig, ModelConfig,
5 + ParallelConfig, SchedulerConfig, LoRAConfig)
6 +from vllm.executor.executor_base import ExecutorBase
7 +from vllm.logger import init_logger
8 +from vllm.sequence import SamplerOutput, SequenceGroupMetadata
9 +
10 +logger = init_logger(__name__)
11 +
12 +
13 +class NeuronExecutor(ExecutorBase):
14 +
15 + def __init__(
16 + self,
17 + model_config: ModelConfig,
18 + cache_config: CacheConfig,
19 + parallel_config: ParallelConfig,
20 + scheduler_config: SchedulerConfig,
21 + device_config: DeviceConfig,
22 + lora_config: Optional[LoRAConfig],
23 + ) -> None:
24 + self.model_config = model_config
25 + self.cache_config = cache_config
26 + assert lora_config is None, "LoRA is not supported for Neuron backend."
27 + self.parallel_config = parallel_config
28 + self.scheduler_config = scheduler_config
29 + self.device_config = device_config
30 +
31 + # Set the number of GPU blocks to be the same as the maximum number of
32 + # sequences that can be processed in a single batch. This is equivalent
33 + # to schedule without PagedAttention.
34 + self.cache_config.num_gpu_blocks = self.scheduler_config.max_num_seqs
35 + self.cache_config.num_cpu_blocks = 0
36 +
37 + # Instantiate the worker and load the model to the device.
38 + self._init_worker()
39 +
40 + def _init_worker(self):
41 + from vllm.worker.neuron_worker import NeuronWorker
42 +
43 + self.driver_worker = NeuronWorker(
44 + self.model_config,
45 + self.parallel_config,
46 + self.scheduler_config,
47 + self.device_config,
48 + )
49 + self.driver_worker.init_device()
50 + self.driver_worker.load_model()
51 +
52 + def execute_model(self,
53 + seq_group_metadata_list: List[SequenceGroupMetadata],
54 + blocks_to_swap_in: Dict[int, int],
55 + blocks_to_swap_out: Dict[int, int],
56 + blocks_to_copy: Dict[int, List[int]]) -> SamplerOutput:
57 + assert (blocks_to_swap_in == {} and blocks_to_swap_out == {}
58 + and blocks_to_copy == {}), (
59 + "Cache operations are not supported for Neuron backend.")
60 +
61 + output = self.driver_worker.execute_model(
62 + seq_group_metadata_list=seq_group_metadata_list)
63 + return output
64 +
65 + def add_lora(self, lora_request: LoRARequest) -> bool:
66 + raise NotImplementedError(
67 + "LoRA is not implemented for neuron backend.")
68 +
69 + def remove_lora(self, lora_id: int) -> bool:
70 + raise NotImplementedError(
71 + "LoRA is not implemented for neuron backend.")
72 +
73 + def list_loras(self) -> List[int]:
74 + raise NotImplementedError(
75 + "LoRA is not implemented for neuron backend.")
76 +
77 + def check_health(self) -> None:
78 + # NeuronExecutor will always be healthy as long as
79 + # it's running.
80 + return
vllm/executor/ray_gpu_executor.py
+2 -16
@@ -3,7 +3,6 @@ import copy
3 from collections import defaultdict
4 import os
5 import pickle
6 -import importlib
6 from typing import TYPE_CHECKING, Any, Dict, List, Optional
7
8 from vllm.config import (CacheConfig, DeviceConfig, ModelConfig,
@@ -25,12 +24,6 @@ if TYPE_CHECKING:
24
25 logger = init_logger(__name__)
26
28 -# A map between the device type (in device config) to its worker module.
29 -DEVICE_TO_WORKER_MODULE_MAP = {
30 - "cuda": "vllm.worker.worker",
31 - "neuron": "vllm.worker.neuron_worker",
32 -}
33 -
27 # If the env var is set, it uses the Ray's compiled DAG API
28 # which optimizes the control plane overhead.
29 # Run vLLM with VLLM_USE_RAY_COMPILED_DAG=1 to enable it.
@@ -73,13 +66,6 @@ class RayGPUExecutor(ExecutorBase):
66 if USE_RAY_COMPILED_DAG:
67 self.forward_dag = self._compiled_ray_dag()
68
76 - def _dispatch_worker(self):
77 - worker_module = DEVICE_TO_WORKER_MODULE_MAP[
78 - self.device_config.device_type]
79 - imported_worker = importlib.import_module(worker_module)
80 - Worker = imported_worker.Worker
81 - return Worker
82 -
69 def _init_workers_ray(self, placement_group: "PlacementGroup",
70 **ray_remote_kwargs):
71 if self.parallel_config.tensor_parallel_size == 1:
@@ -155,7 +141,7 @@ class RayGPUExecutor(ExecutorBase):
141
142 # Lazy import the Worker to avoid importing torch.cuda/xformers
143 # before CUDA_VISIBLE_DEVICES is set in the Worker
158 - Worker = self._dispatch_worker()
144 + from vllm.worker.worker import Worker
145
146 model_config = copy.deepcopy(self.model_config)
147 parallel_config = copy.deepcopy(self.parallel_config)
@@ -201,7 +187,7 @@ class RayGPUExecutor(ExecutorBase):
187
188 # FIXME(woosuk): We are not properly initializing cupy NCCL when
189 # we have multiple nodes.
204 - self._run_workers("init_model",
190 + self._run_workers("init_device",
191 cupy_port=get_open_port()
192 if not model_config.enforce_eager else None)
193 self._run_workers(
vllm/lora/layers.py
+2 -2
@@ -799,8 +799,8 @@ class LogitsProcessorWithLoRA(BaseLayerWithLoRA):
799 self.device = device
800
801 @property
802 - def logits_as_hidden_states(self):
803 - return self.base_layer.logits_as_hidden_states
802 + def logits_as_input(self):
803 + return self.base_layer.logits_as_input
804
805 @property
806 def vocab_size(self):
vllm/lora/lora.py
+2 -2
@@ -1,7 +1,7 @@
1 from typing import List, Optional
2
3 import torch
4 -from vllm.utils import in_wsl
4 +from vllm.utils import is_pin_memory_available
5
6
7 class LoRALayerWeights:
@@ -64,7 +64,7 @@ class LoRALayerWeights:
64 dtype: torch.dtype,
65 device: torch.device,
66 embeddings_tensor_dim: Optional[int] = None) -> "LoRALayerWeights":
67 - pin_memory = str(device) == "cpu" and not in_wsl()
67 + pin_memory = str(device) == "cpu" and is_pin_memory_available()
68 lora_a = torch.zeros([input_dim, rank],
69 dtype=dtype,
70 device=device,
vllm/lora/models.py
+2 -2
@@ -11,7 +11,7 @@ import torch
11 from torch import nn
12
13 from vllm.config import LoRAConfig
14 -from vllm.utils import LRUCache, in_wsl
14 +from vllm.utils import LRUCache, is_pin_memory_available
15
16 from vllm.lora.layers import (BaseLayerWithLoRA, LoRAMapping, from_layer,
17 from_layer_logits_processor)
@@ -143,7 +143,7 @@ class LoRAModel:
143 embedding_padding_modules: Optional[List[str]] = None,
144 ) -> "LoRAModel":
145 """Create a LoRAModel from a dictionary of tensors."""
146 - pin_memory = str(device) == "cpu" and not in_wsl()
146 + pin_memory = str(device) == "cpu" and is_pin_memory_available()
147 loras: Dict[str, LoRALayerWeights] = {}
148 for tensor_name, tensor in tensors.items():
149 module_name, is_lora_a = parse_fine_tuned_lora_name(tensor_name)
vllm/model_executor/__init__.py
+1 -2
@@ -1,10 +1,9 @@
1 from vllm.model_executor.input_metadata import InputMetadata
2 from vllm.model_executor.sampling_metadata import SamplingMetadata
3 -from vllm.model_executor.utils import set_random_seed, get_model
3 +from vllm.model_executor.utils import set_random_seed
4
5 __all__ = [
6 "InputMetadata",
7 - "get_model",
7 "SamplingMetadata",
8 "set_random_seed",
9 ]
vllm/model_executor/input_metadata.py
+4 -3
@@ -1,8 +1,9 @@
1 from dataclasses import dataclass, fields
2 -from typing import Optional, List, Any, Dict
2 +from typing import TYPE_CHECKING, Optional, List, Any, Dict
3
4 import torch
5 -from xformers.ops.fmha.attn_bias import AttentionBias
5 +if TYPE_CHECKING:
6 + from xformers.ops.fmha.attn_bias import AttentionBias
7
8
9 @dataclass
@@ -82,7 +83,7 @@ class InputMetadata:
83 # when alibi slopes is used. It is because of the limitation
84 # from xformer API.
85 # will not appear in the __repr__ and __init__
85 - self.attn_bias: Optional[List[AttentionBias]] = None
86 + self.attn_bias: Optional[List["AttentionBias"]] = None
87
88 # Cuda graph is only used for decoding now.
89 if self.use_cuda_graph:
vllm/model_executor/layers/logits_processor.py
+5 -6
@@ -4,8 +4,6 @@ from typing import Optional
4 import torch
5 import torch.nn as nn
6
7 -from vllm.utils import is_neuron
8 -
7 from vllm.model_executor.parallel_utils.communication_op import (
8 tensor_model_parallel_gather)
9 from vllm.model_executor.sampling_metadata import SamplingMetadata
@@ -23,7 +21,8 @@ class LogitsProcessor(nn.Module):
21 def __init__(self,
22 vocab_size: int,
23 org_vocab_size: Optional[int] = None,
26 - scale: Optional[float] = 1.0) -> None:
24 + scale: Optional[float] = 1.0,
25 + logits_as_input: bool = False) -> None:
26 """
27 Args:
28 scale: A scaling factor to apply to the logits.
@@ -31,8 +30,8 @@ class LogitsProcessor(nn.Module):
30 super().__init__()
31 self.scale = scale
32 self.vocab_size = vocab_size
34 - # Transformers-neuronx generate outputs as logits directly.
35 - self.logits_as_hidden_states = is_neuron()
33 + # Whether the input is logits (default is hidden states).
34 + self.logits_as_input = logits_as_input
35 # original vocabulary size (without LoRA).
36 self.org_vocab_size = org_vocab_size or vocab_size
37
@@ -43,7 +42,7 @@ class LogitsProcessor(nn.Module):
42 sampling_metadata: SamplingMetadata,
43 embedding_bias: Optional[torch.Tensor] = None,
44 ) -> torch.Tensor:
46 - if self.logits_as_hidden_states:
45 + if self.logits_as_input:
46 logits = hidden_states
47 else:
48 hidden_states = _prune_hidden_states(hidden_states,
vllm/model_executor/layers/sampler.py
+1 -1
@@ -4,13 +4,13 @@ from typing import Dict, List, Optional, Tuple
4 import torch
5 import torch.nn as nn
6
7 +from vllm.model_executor.layers.ops.sample import sample as sample_triton
8 from vllm.model_executor.sampling_metadata import (SamplingMetadata,
9 SamplingTensors)
10 from vllm.sampling_params import SamplingParams, SamplingType
11 from vllm.sequence import (Logprob, PromptLogprobs, SampleLogprobs,
12 SamplerOutput, SequenceData, SequenceGroupOutput,
13 SequenceOutput)
13 -from vllm.model_executor.layers.ops.sample import (sample as sample_triton)
14
15
16 class Sampler(nn.Module):
vllm/model_executor/models/__init__.py
+1 -14
@@ -4,7 +4,7 @@ from typing import List, Optional, Type
4 import torch.nn as nn
5
6 from vllm.logger import init_logger
7 -from vllm.utils import is_hip, is_neuron
7 +from vllm.utils import is_hip
8
9 logger = init_logger(__name__)
10
@@ -63,12 +63,6 @@ _ROCM_PARTIALLY_SUPPORTED_MODELS = {
63 "Sliding window attention is not yet supported in ROCm's flash attention",
64 }
65
66 -# Models supported by Neuron.
67 -_NEURON_SUPPORTED_MODELS = {
68 - "LlamaForCausalLM": "neuron.llama",
69 - "MistralForCausalLM": "neuron.mistral"
70 -}
71 -
66
67 class ModelRegistry:
68
@@ -85,15 +79,8 @@ class ModelRegistry:
79 logger.warning(
80 f"Model architecture {model_arch} is partially supported "
81 "by ROCm: " + _ROCM_PARTIALLY_SUPPORTED_MODELS[model_arch])
88 - elif is_neuron():
89 - if model_arch not in _NEURON_SUPPORTED_MODELS:
90 - raise ValueError(
91 - f"Model architecture {model_arch} is not supported by "
92 - "Neuron for now.")
82
83 module_name, model_cls_name = _MODELS[model_arch]
95 - if is_neuron():
96 - module_name = _NEURON_SUPPORTED_MODELS[model_arch]
84 module = importlib.import_module(
85 f"vllm.model_executor.models.{module_name}")
86 return getattr(module, model_cls_name, None)
vllm/model_executor/models/neuron/llama.py deleted
-86
@@ -1,86 +0,0 @@
1 -"""Inference-only LLaMA model compatible with HuggingFace weights."""
2 -import os
3 -from typing import List, Optional, Tuple
4 -
5 -import torch
6 -from torch import nn
7 -from transformers import LlamaConfig
8 -
9 -from vllm.model_executor.input_metadata import InputMetadata
10 -from vllm.model_executor.layers.logits_processor import LogitsProcessor
11 -from vllm.model_executor.layers.sampler import Sampler
12 -from vllm.model_executor.sampling_metadata import SamplingMetadata
13 -from vllm.sequence import SamplerOutput
14 -
15 -KVCache = Tuple[torch.Tensor, torch.Tensor]
16 -
17 -
18 -class LlamaForCausalLM(nn.Module):
19 -
20 - def __init__(
21 - self,
22 - config: LlamaConfig,
23 - linear_method=None,
24 - ) -> None:
25 - super().__init__()
26 - self.config = config
27 - self.linear_method = linear_method
28 - self.model = None
29 - self.logits_processor = LogitsProcessor(config.vocab_size)
30 - self.sampler = Sampler()
31 -
32 - def forward(
33 - self,
34 - input_ids: torch.Tensor,
35 - positions: torch.Tensor,
36 - kv_caches: List[KVCache],
37 - input_metadata: InputMetadata,
38 - ) -> torch.Tensor:
39 - with torch.inference_mode():
40 - block_size = self.model.context_buckets[-1]
41 - if input_metadata.is_prompt:
42 - seq_ids = input_metadata.slot_mapping[:, 0] // block_size
43 - else:
44 - seq_ids = input_metadata.block_tables
45 - logits = self.model(input_ids,
46 - cache_ids=positions,
47 - start_ids=seq_ids.flatten())
48 - return logits
49 -
50 - def compute_logits(self, hidden_states: torch.Tensor,
51 - sampling_metadata: SamplingMetadata) -> torch.Tensor:
52 - logits = self.logits_processor(self.model.chkpt_model.lm_head,
53 - hidden_states, sampling_metadata)
54 - return logits
55 -
56 - def sample(
57 - self,
58 - logits: torch.Tensor,
59 - sampling_metadata: SamplingMetadata,
60 - ) -> Optional[SamplerOutput]:
61 - next_tokens = self.sampler(logits, sampling_metadata)
62 - return next_tokens
63 -
64 - def load_weights(self,
65 - model_name_or_path: str,
66 - cache_dir: Optional[str] = None,
67 - load_format: str = "auto",
68 - revision: Optional[str] = None,
69 - **kwargs):
70 - from transformers_neuronx.llama.model import LlamaForSampling
71 -
72 - split_model_dir = f"{model_name_or_path}-split"
73 - if os.path.isdir(os.path.join(model_name_or_path,
74 - "pytorch_model.bin")):
75 - split_model_dir = model_name_or_path
76 - elif not os.path.exists(f"{model_name_or_path}-split"):
77 - from transformers.models.llama import LlamaForCausalLM
78 - from transformers_neuronx.module import save_pretrained_split
79 -
80 - hf_model = LlamaForCausalLM.from_pretrained(model_name_or_path,
81 - low_cpu_mem_usage=True)
82 - save_pretrained_split(hf_model, f"{model_name_or_path}-split")
83 -
84 - self.model = LlamaForSampling.from_pretrained(split_model_dir,
85 - **kwargs)
86 - self.model.to_neuron()
vllm/model_executor/models/neuron/mistral.py deleted
-89
@@ -1,89 +0,0 @@
1 -"""Inference-only Mistral model compatible with HuggingFace weights."""
2 -from typing import List, Optional, Tuple
3 -
4 -import torch
5 -from torch import nn
6 -from transformers import MistralConfig
7 -
8 -from vllm.model_executor.input_metadata import InputMetadata
9 -from vllm.model_executor.layers.logits_processor import LogitsProcessor
10 -from vllm.model_executor.layers.sampler import Sampler
11 -from vllm.model_executor.sampling_metadata import SamplingMetadata
12 -from vllm.sequence import SamplerOutput
13 -import os
14 -
15 -KVCache = Tuple[torch.Tensor, torch.Tensor]
16 -
17 -
18 -class MistralForCausalLM(nn.Module):
19 -
20 - def __init__(
21 - self,
22 - config: MistralConfig,
23 - linear_method=None,
24 - ) -> None:
25 - super().__init__()
26 - self.config = config
27 - self.linear_method = linear_method
28 - self.model = None
29 - self.lm_head = None
30 - self.logits_processor = LogitsProcessor(config.vocab_size)
31 - self.sampler = Sampler()
32 -
33 - def forward(
34 - self,
35 - input_ids: torch.Tensor,
36 - positions: torch.Tensor,
37 - kv_caches: List[KVCache],
38 - input_metadata: InputMetadata,
39 - ) -> SamplerOutput:
40 - with torch.inference_mode():
41 - seq_ids = []
42 - block_size = self.model.context_buckets[-1]
43 - if input_metadata.is_prompt:
44 - seq_ids = input_metadata.slot_mapping[:, 0] // block_size
45 - else:
46 - seq_ids = input_metadata.block_tables
47 -
48 - logits = self.model(input_ids,
49 - cache_ids=positions,
50 - start_ids=seq_ids)
51 - return logits
52 -
53 - def compute_logits(self, hidden_states: torch.Tensor,
54 - sampling_metadata: SamplingMetadata) -> torch.Tensor:
55 - logits = self.logits_processor(self.model.chkpt_model.lm_head,
56 - hidden_states, sampling_metadata)
57 - return logits
58 -
59 - def sample(
60 - self,
61 - logits: torch.Tensor,
62 - sampling_metadata: SamplingMetadata,
63 - ) -> Optional[SamplerOutput]:
64 - next_tokens = self.sampler(logits, sampling_metadata)
65 - return next_tokens
66 -
67 - def load_weights(self,
68 - model_name_or_path: str,
69 - cache_dir: Optional[str] = None,
70 - load_format: str = "auto",
71 - revision: Optional[str] = None,
72 - **kwargs):
73 - from transformers_neuronx.mistral.model import MistralForSampling
74 -
75 - split_model_dir = f"{model_name_or_path}-split"
76 - if os.path.isdir(os.path.join(model_name_or_path,
77 - "pytorch_model.bin")):
78 - split_model_dir = model_name_or_path
79 - elif not os.path.exists(f"{model_name_or_path}-split"):
80 - from transformers import MistralForCausalLM
81 - from transformers_neuronx.module import save_pretrained_split
82 -
83 - hf_model = MistralForCausalLM.from_pretrained(
84 - model_name_or_path, low_cpu_mem_usage=True)
85 - save_pretrained_split(hf_model, f"{model_name_or_path}-split")
86 -
87 - self.model = MistralForSampling.from_pretrained(
88 - split_model_dir, **kwargs)
89 - self.model.to_neuron()
vllm/model_executor/neuron_model_loader.py
+89 -22
@@ -1,12 +1,18 @@
1 -"""Utilities for selecting and loading models."""
2 -from typing import Type
1 +"""Utilities for selecting and loading neuron models."""
2 +import importlib
3 +import os
4 +from typing import Optional, Type
5
6 import torch
7 import torch.nn as nn
8 +import transformers
9 from transformers import PretrainedConfig
10
8 -from vllm.config import ModelConfig, DeviceConfig
9 -from vllm.model_executor.models import ModelRegistry
11 +from vllm.config import ModelConfig, ParallelConfig, SchedulerConfig
12 +from vllm.model_executor.layers.logits_processor import LogitsProcessor
13 +from vllm.model_executor.layers.sampler import Sampler
14 +from vllm.model_executor.sampling_metadata import SamplingMetadata
15 +from vllm.sequence import SamplerOutput
16
17 TORCH_DTYPE_TO_NEURON_AMP = {
18 "auto": "f32",
@@ -20,31 +26,95 @@ TORCH_DTYPE_TO_NEURON_AMP = {
26 torch.float32: "f32",
27 }
28
29 +# Models supported by Neuron.
30 +_NEURON_SUPPORTED_MODELS = {
31 + "LlamaForCausalLM": ("transformers_neuronx.llama.model",
32 + "LlamaForSampling", "LlamaForCausalLM"),
33 + "MistralForCausalLM": ("transformers_neuronx.mistral.model",
34 + "MistralForSampling", "MistralForCausalLM")
35 +}
36 +
37 +
38 +class NeuronCasualLM(nn.Module):
39 +
40 + def __init__(
41 + self,
42 + config: PretrainedConfig,
43 + ) -> None:
44 + super().__init__()
45 + self.config = config
46 + self.model = None
47 + self.logits_processor = LogitsProcessor(config.vocab_size,
48 + logits_as_input=True)
49 + self.sampler = Sampler()
50 +
51 + def forward(
52 + self,
53 + input_ids: torch.Tensor,
54 + positions: torch.Tensor,
55 + input_block_ids: torch.Tensor,
56 + ) -> torch.Tensor:
57 + logits = self.model(input_ids,
58 + cache_ids=positions,
59 + start_ids=input_block_ids)
60 + return logits
61 +
62 + def compute_logits(self, hidden_states: torch.Tensor,
63 + sampling_metadata: SamplingMetadata) -> torch.Tensor:
64 + logits = self.logits_processor(None, hidden_states, sampling_metadata)
65 + return logits
66 +
67 + def sample(
68 + self,
69 + logits: torch.Tensor,
70 + sampling_metadata: SamplingMetadata,
71 + ) -> Optional[SamplerOutput]:
72 + next_tokens = self.sampler(logits, sampling_metadata)
73 + return next_tokens
74 +
75 + def load_weights(self, model_name_or_path: str, **kwargs):
76 + arch = _get_model_architecture(self.config)
77 + neuronx_module_path, neuronx_model_cls, hf_model_cls = (
78 + _NEURON_SUPPORTED_MODELS[arch])
79 + neuronx_module = importlib.import_module(neuronx_module_path)
80 + neuronx_model_cls = getattr(neuronx_module, neuronx_model_cls)
81 +
82 + split_model_dir = f"{model_name_or_path}-split"
83 + if os.path.isdir(os.path.join(model_name_or_path,
84 + "pytorch_model.bin")):
85 + split_model_dir = model_name_or_path
86 + elif not os.path.exists(f"{model_name_or_path}-split"):
87 + hf_model_cls = getattr(transformers, hf_model_cls)
88 + from transformers_neuronx.module import save_pretrained_split
89 +
90 + hf_model = hf_model_cls.from_pretrained(model_name_or_path,
91 + low_cpu_mem_usage=True)
92 + save_pretrained_split(hf_model, f"{model_name_or_path}-split")
93 +
94 + self.model = neuronx_model_cls.from_pretrained(split_model_dir,
95 + **kwargs)
96 + self.model.to_neuron()
97 +
98
99 def _get_model_architecture(config: PretrainedConfig) -> Type[nn.Module]:
100 architectures = getattr(config, "architectures", [])
101 for arch in architectures:
27 - model_cls = ModelRegistry.load_model_cls(arch)
28 - if model_cls is not None:
29 - return model_cls
102 + if arch in _NEURON_SUPPORTED_MODELS:
103 + return arch
104 raise ValueError(
31 - f"Model architectures {architectures} are not supported for now. "
32 - f"Supported architectures: {ModelRegistry.get_supported_archs()}")
105 + f"Model architectures {architectures} are not supported on Neuron "
106 + f"for now. Supported architectures: "
107 + f"{list(_NEURON_SUPPORTED_MODELS.keys())}")
108
109
35 -def get_model(model_config: ModelConfig, device_config: DeviceConfig,
36 - **kwargs) -> nn.Module:
110 +def get_neuron_model(model_config: ModelConfig,
111 + parallel_config: ParallelConfig,
112 + scheduler_config: SchedulerConfig) -> nn.Module:
113 from transformers_neuronx.config import (NeuronConfig,
114 ContinuousBatchingConfig)
115
40 - parallel_config = kwargs.get("parallel_config")
41 - scheduler_config = kwargs.get("scheduler_config")
42 -
43 - model_class = _get_model_architecture(model_config.hf_config)
44 - linear_method = None
45 -
116 # Create a model instance.
47 - model = model_class(model_config.hf_config, linear_method)
117 + model = NeuronCasualLM(model_config.hf_config)
118
119 continuous_batching_config = ContinuousBatchingConfig(
120 batch_size_for_shared_caches=scheduler_config.max_num_seqs)
@@ -54,10 +124,7 @@ def get_model(model_config: ModelConfig, device_config: DeviceConfig,
124 # Load the weights from the cached or downloaded files.
125 model.load_weights(
126 model_config.model,
57 - model_config.download_dir,
58 - model_config.load_format,
59 - model_config.revision,
60 - tp_degree=parallel_config.neuron_tp_degree,
127 + tp_degree=parallel_config.tensor_parallel_size,
128 amp=TORCH_DTYPE_TO_NEURON_AMP[model_config.dtype],
129 neuron_config=neuron_config,
130 context_length_estimate=[scheduler_config.max_model_len],
vllm/model_executor/sampling_metadata.py
+4 -4
@@ -4,11 +4,11 @@ from typing import Dict, List, Optional, Tuple
4 import torch
5 import random
6
7 -from vllm.sampling_params import SamplingParams, SamplingType
8 -from vllm.sequence import SequenceData
9 -from vllm.utils import in_wsl, is_neuron
7 from vllm.model_executor.layers.ops.sample import (
8 get_num_triton_sampler_splits)
9 +from vllm.sampling_params import SamplingParams, SamplingType
10 +from vllm.sequence import SequenceData
11 +from vllm.utils import is_pin_memory_available
12
13 _SAMPLING_EPS = 1e-5
14 _SEED_0_REPLACEMENT = 3403598558
@@ -213,7 +213,7 @@ class SamplingTensors:
213 dtype: torch.dtype) -> "SamplingTensors":
214 # Note that the performance will be very bad without
215 # pinned memory.
216 - pin_memory = not in_wsl() and not is_neuron()
216 + pin_memory = is_pin_memory_available()
217 prompt_max_len = max(len(tokens) for tokens in prompt_tokens)
218 prompt_padded_tokens = [
219 tokens + [vocab_size] * (prompt_max_len - len(tokens))
vllm/model_executor/utils.py
-17
@@ -1,18 +1,10 @@
1 """Utils for model executor."""
2 import random
3 -import importlib
3 from typing import Any, Dict, Optional
4
5 import numpy as np
6 import torch
7
9 -from vllm.config import DeviceConfig, ModelConfig
10 -
11 -DEVICE_TO_MODEL_LOADER_MAP = {
12 - "cuda": "model_loader",
13 - "neuron": "neuron_model_loader",
14 -}
15 -
8
9 def set_random_seed(seed: int) -> None:
10 random.seed(seed)
@@ -41,12 +33,3 @@ def set_weight_attrs(
33 assert not hasattr(
34 weight, key), (f"Overwriting existing tensor attribute: {key}")
35 setattr(weight, key, value)
44 -
45 -
46 -def get_model(model_config: ModelConfig, device_config: DeviceConfig,
47 - **kwargs) -> torch.nn.Module:
48 - model_loader_module = DEVICE_TO_MODEL_LOADER_MAP[device_config.device_type]
49 - imported_model_loader = importlib.import_module(
50 - f"vllm.model_executor.{model_loader_module}")
51 - get_model_fn = imported_model_loader.get_model
52 - return get_model_fn(model_config, device_config, **kwargs)
vllm/spec_decode/metrics.py
+2 -2
@@ -2,7 +2,7 @@ import torch
2 from dataclasses import dataclass
3 from vllm.model_executor.layers.rejection_sampler import RejectionSampler
4 from typing import Optional
5 -from vllm.utils import in_wsl
5 +from vllm.utils import is_pin_memory_available
6 import time
7 from typing import Callable
8
@@ -63,7 +63,7 @@ class AsyncMetricsCollector:
63
64 self._in_flight_copy: Optional[torch.cuda.Event] = None
65
66 - pin_memory = not in_wsl()
66 + pin_memory = is_pin_memory_available()
67 self._aggregate_num_accepted_tokens = torch.tensor(
68 0, dtype=torch.long, device="cpu", pin_memory=pin_memory)
69 self._aggregate_num_emitted_tokens = torch.tensor(
vllm/spec_decode/multi_step_worker.py
+2 -2
@@ -27,8 +27,8 @@ class MultiStepWorker(Worker):
27
28 self._proposer: Optional[DraftModelTop1Proposer] = None
29
30 - def init_model(self):
31 - super().init_model()
30 + def init_device(self):
31 + super().init_device()
32
33 self._proposer = DraftModelTop1Proposer(
34 self,
vllm/spec_decode/spec_decode_worker.py
+3 -3
@@ -79,13 +79,13 @@ class SpecDecodeWorker:
79
80 self.scorer: SpeculativeScorer = None
81
82 - def init_model(self) -> None:
82 + def init_device(self) -> None:
83 """Initialize both scorer and proposer models.
84 """
85 # The scorer worker model is initialized first in case the proposer
86 # model has a smaller TP degree than the target worker.
87 - self.scorer_worker.init_model()
88 - self.proposer_worker.init_model()
87 + self.scorer_worker.init_device()
88 + self.proposer_worker.init_device()
89
90 self._metrics.init_gpu_tensors(self.rank)
91 self.rejection_sampler.init_gpu_tensors(self.rank)
vllm/utils.py
+62 -1
@@ -338,7 +338,27 @@ def create_kv_caches_with_random(
338 return key_caches, value_caches
339
340
341 -class measure_cuda_memory:
341 +@lru_cache
342 +def print_warning_once(msg: str) -> None:
343 + logger.warning(msg)
344 +
345 +
346 +@lru_cache(maxsize=None)
347 +def is_pin_memory_available() -> bool:
348 +
349 + if in_wsl():
350 + # Pinning memory in WSL is not supported.
351 + # https://docs.nvidia.com/cuda/wsl-user-guide/index.html#known-limitations-for-linux-cuda-applications
352 + print_warning_once("Using 'pin_memory=False' as WSL is detected. "
353 + "This may slow down the performance.")
354 + return False
355 + elif is_neuron():
356 + print_warning_once("Pin memory is not supported on Neuron.")
357 + return False
358 + return True
359 +
360 +
361 +class CudaMemoryProfiler:
362
363 def __init__(self, device=None):
364 self.device = device
@@ -360,3 +380,44 @@ class measure_cuda_memory:
380
381 # Force garbage collection
382 gc.collect()
383 +
384 +
385 +def pad_to_max_length(x: List[int], max_len: int, pad: int) -> List[int]:
386 + assert len(x) <= max_len
387 + return x + [pad] * (max_len - len(x))
388 +
389 +
390 +def make_tensor_with_pad(
391 + x: List[List[int]],
392 + max_len: int,
393 + pad: int,
394 + dtype: torch.dtype,
395 + device: Optional[Union[str, torch.device]],
396 +) -> torch.Tensor:
397 + """Make a padded tensor of a 2D inputs.
398 +
399 + The padding is applied to the end of each inner list until it reaches
400 + `max_len`.
401 + """
402 + padded_x = [pad_to_max_length(x_i, max_len, pad) for x_i in x]
403 + return torch.tensor(padded_x, dtype=dtype, device=device)
404 +
405 +
406 +def async_tensor_h2d(
407 + data: list,
408 + dtype: torch.dtype,
409 + target_device: Union[str, torch.device],
410 + pin_memory: bool,
411 +) -> torch.Tensor:
412 + """Asynchronously create a tensor and copy it from host to device."""
413 + t = torch.tensor(data, dtype=dtype, pin_memory=pin_memory, device="cpu")
414 + return t.to(device=target_device, non_blocking=True)
415 +
416 +
417 +def maybe_expand_dim(tensor: torch.Tensor,
418 + target_dims: int,
419 + size: int = 1) -> torch.Tensor:
420 + """Expand the tensor to the target_dims."""
421 + if tensor.ndim < target_dims:
422 + tensor = tensor.view(-1, *([size] * (target_dims - tensor.ndim)))
423 + return tensor
vllm/worker/cache_engine.py
+2 -11
@@ -5,7 +5,7 @@ import torch
5
6 from vllm.config import CacheConfig, ModelConfig, ParallelConfig
7 from vllm.logger import init_logger
8 -from vllm.utils import in_wsl, is_neuron, STR_DTYPE_TO_TORCH_DTYPE
8 +from vllm.utils import is_pin_memory_available, STR_DTYPE_TO_TORCH_DTYPE
9
10 logger = init_logger(__name__)
11
@@ -38,10 +38,6 @@ class CacheEngine:
38 self.num_gpu_blocks = cache_config.num_gpu_blocks
39 self.num_cpu_blocks = cache_config.num_cpu_blocks
40
41 - # Skip initializing KV cache for Neuron backend.
42 - if is_neuron():
43 - return
44 -
41 if cache_config.cache_dtype == "auto":
42 self.dtype = model_config.dtype
43 else:
@@ -90,12 +86,7 @@ class CacheEngine:
86 cpu_cache: List[KVCache] = []
87 key_block_shape = self.get_key_block_shape()
88 value_block_shape = self.get_value_block_shape()
93 - pin_memory = not in_wsl()
94 - if not pin_memory:
95 - # Pinning memory in WSL is not supported.
96 - # https://docs.nvidia.com/cuda/wsl-user-guide/index.html#known-limitations-for-linux-cuda-applications
97 - logger.warning("Using 'pin_memory=False' as WSL is detected. "
98 - "This may slow down the performance.")
89 + pin_memory = is_pin_memory_available()
90 for _ in range(self.num_layers):
91 key_blocks = torch.empty(
92 size=(self.num_cpu_blocks, *key_block_shape),
vllm/worker/model_runner.py
+20 -62
@@ -1,6 +1,6 @@
1 import contextlib
2 import time
3 -from typing import Dict, List, Optional, Tuple, Set, Union
3 +from typing import Dict, List, Optional, Tuple, Set
4
5 import numpy as np
6 import torch
@@ -9,7 +9,8 @@ import torch.nn as nn
9 from vllm.config import (DeviceConfig, ModelConfig, LoRAConfig, ParallelConfig,
10 SchedulerConfig)
11 from vllm.logger import init_logger
12 -from vllm.model_executor import get_model, InputMetadata, SamplingMetadata
12 +from vllm.model_executor import InputMetadata, SamplingMetadata
13 +from vllm.model_executor.model_loader import get_model
14 from vllm.model_executor.parallel_utils import cupy_utils
15 from vllm.model_executor.parallel_utils.communication_op import (
16 broadcast_tensor_dict)
@@ -21,7 +22,9 @@ from vllm.sequence import SamplerOutput, SequenceData, SequenceGroupMetadata
22 from vllm.lora.worker_manager import LRUCacheWorkerLoRAManager
23 from vllm.lora.layers import LoRAMapping
24 from vllm.lora.request import LoRARequest
24 -from vllm.utils import in_wsl, measure_cuda_memory
25 +from vllm.utils import (async_tensor_h2d, CudaMemoryProfiler,
26 + is_pin_memory_available, make_tensor_with_pad,
27 + maybe_expand_dim)
28
29 logger = init_logger(__name__)
30
@@ -79,16 +82,11 @@ class ModelRunner:
82 # The shape of the cached block table will be
83 # (max batch size to capture, max context len to capture / block size).
84 self.graph_block_tables = None # Set after initial profiling.
82 - # cache in_wsl result
83 - self.in_wsl = in_wsl()
85 + self.pin_memory = is_pin_memory_available()
86 self.kv_cache_dtype = kv_cache_dtype
87
86 - # Set enforce_eager to True for Neuron backend, to avoid capturing graph
87 - if self.device_config.is_neuron:
88 - self.model_config.enforce_eager = True
89 -
88 def load_model(self) -> None:
91 - with measure_cuda_memory() as m:
89 + with CudaMemoryProfiler() as m:
90 self.model = get_model(self.model_config,
91 self.device_config,
92 lora_config=self.lora_config,
@@ -238,7 +236,7 @@ class ModelRunner:
236 device=self.device)
237 # Prepare prefix block tables
238 max_prompt_block_table_len = max(len(t) for t in prefix_block_tables)
241 - block_tables = _make_tensor_with_pad(
239 + block_tables = make_tensor_with_pad(
240 prefix_block_tables,
241 max_len=max_prompt_block_table_len,
242 pad=0,
@@ -395,7 +393,7 @@ class ModelRunner:
393 else:
394 max_block_table_len = max(
395 len(block_table) for block_table in block_tables)
398 - block_tables = _make_tensor_with_pad(
396 + block_tables = make_tensor_with_pad(
397 block_tables,
398 max_len=max_block_table_len,
399 pad=0,
@@ -436,7 +434,6 @@ class ModelRunner:
434 categorized_sample_indices = {t: [] for t in SamplingType}
435 categorized_sample_indices_start_idx = 0
436 categorized_sampled_token_indices_start_idx = 0
439 - pin_memory = not self.in_wsl and not self.device_config.is_neuron
437
438 for i, seq_group_metadata in enumerate(seq_group_metadata_list):
439 seq_ids = list(seq_group_metadata.seq_data.keys())
@@ -469,7 +466,7 @@ class ModelRunner:
466
467 if sampling_params.seed is not None:
468 seq_group_metadata.state.generator = torch.Generator(
472 - device="cuda").manual_seed(sampling_params.seed)
469 + device=self.device).manual_seed(sampling_params.seed)
470 else:
471 num_seqs = len(seq_ids)
472 selected_token_indices.extend(
@@ -494,17 +491,17 @@ class ModelRunner:
491 if sampling_params.seed is not None:
492 generators.append(seq_group_metadata.state.generator)
493
497 - selected_token_indices = _async_h2d(selected_token_indices,
498 - dtype=torch.long,
499 - target_device=self.device,
500 - pin_memory=not self.in_wsl)
494 + selected_token_indices = async_tensor_h2d(selected_token_indices,
495 + dtype=torch.long,
496 + target_device=self.device,
497 + pin_memory=self.pin_memory)
498
499 categorized_sample_indices = {
503 - t: _maybe_expand_dim(
504 - _async_h2d(seq_ids,
505 - dtype=torch.int,
506 - target_device=self.device,
507 - pin_memory=pin_memory), 2, 2)
500 + t: maybe_expand_dim(
501 + async_tensor_h2d(seq_ids,
502 + dtype=torch.int,
503 + target_device=self.device,
504 + pin_memory=self.pin_memory), 2, 2)
505 for t, seq_ids in categorized_sample_indices.items()
506 }
507
@@ -910,27 +907,6 @@ def _maybe_cupy_nccl():
907 yield
908
909
913 -def _pad_to_max(x: List[int], max_len: int, pad: int) -> List[int]:
914 - assert len(x) <= max_len
915 - return x + [pad] * (max_len - len(x))
916 -
917 -
918 -def _make_tensor_with_pad(
919 - x: List[List[int]],
920 - max_len: int,
921 - pad: int,
922 - dtype: torch.dtype,
923 - device: Optional[Union[str, torch.device]],
924 -) -> torch.Tensor:
925 - """Make a padded tensor of a 2D inputs.
926 -
927 - The padding is applied to the end of each inner list until it reaches
928 - `max_len`.
929 - """
930 - padded_x = [_pad_to_max(x_i, max_len, pad) for x_i in x]
931 - return torch.tensor(padded_x, dtype=dtype, device=device)
932 -
933 -
910 def _get_graph_batch_size(batch_size: int) -> int:
911 """Returns the padded batch size given actual batch size.
912
@@ -944,21 +920,3 @@ def _get_graph_batch_size(batch_size: int) -> int:
920 else:
921 return ((batch_size + _BATCH_SIZE_ALIGNMENT - 1) //
922 _BATCH_SIZE_ALIGNMENT * _BATCH_SIZE_ALIGNMENT)
947 -
948 -
949 -def _async_h2d(
950 - data: list,
951 - dtype: torch.dtype,
952 - target_device: Union[str, torch.device],
953 - pin_memory: bool,
954 -) -> torch.Tensor:
955 - t = torch.tensor(data, dtype=dtype, pin_memory=pin_memory, device="cpu")
956 - return t.to(device=target_device, non_blocking=True)
957 -
958 -
959 -def _maybe_expand_dim(tensor: torch.Tensor,
960 - target_dims: int,
961 - size: int = 1) -> torch.Tensor:
962 - if tensor.ndim < target_dims:
963 - tensor = tensor.view(-1, *([size] * (target_dims - tensor.ndim)))
964 - return tensor
vllm/worker/neuron_model_runner.py new
+287
@@ -0,0 +1,287 @@
1 +from typing import Dict, List, Optional, Tuple
2 +
3 +import torch
4 +
5 +from vllm.config import (DeviceConfig, ModelConfig, ParallelConfig,
6 + SchedulerConfig)
7 +from vllm.logger import init_logger
8 +from vllm.model_executor import SamplingMetadata
9 +from vllm.model_executor.neuron_model_loader import get_neuron_model
10 +from vllm.sampling_params import SamplingParams, SamplingType
11 +from vllm.sequence import SamplerOutput, SequenceData, SequenceGroupMetadata
12 +from vllm.utils import (async_tensor_h2d, is_pin_memory_available,
13 + make_tensor_with_pad, maybe_expand_dim)
14 +
15 +logger = init_logger(__name__)
16 +
17 +KVCache = Tuple[torch.Tensor, torch.Tensor]
18 +
19 +
20 +class NeuronModelRunner:
21 +
22 + def __init__(
23 + self,
24 + model_config: ModelConfig,
25 + parallel_config: ParallelConfig,
26 + scheduler_config: SchedulerConfig,
27 + device_config: DeviceConfig,
28 + ):
29 + self.model_config = model_config
30 + self.parallel_config = parallel_config
31 + self.scheduler_config = scheduler_config
32 +
33 + if model_config is not None and model_config.get_sliding_window():
34 + logger.warning("Sliding window is not supported on Neuron. "
35 + "The model will run without sliding window.")
36 + self.device_config = (device_config
37 + if device_config is not None else DeviceConfig())
38 + self.device = self.device_config.device
39 + self.model = None
40 + self.pin_memory = is_pin_memory_available()
41 +
42 + def load_model(self) -> None:
43 + self.model = get_neuron_model(self.model_config,
44 + parallel_config=self.parallel_config,
45 + scheduler_config=self.scheduler_config)
46 +
47 + def _prepare_prompt(
48 + self,
49 + seq_group_metadata_list: List[SequenceGroupMetadata],
50 + ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, List[int]]:
51 + assert len(seq_group_metadata_list) > 0
52 + input_tokens: List[List[int]] = []
53 + input_positions: List[List[int]] = []
54 + input_block_ids: List[int] = []
55 +
56 + prompt_lens: List[int] = []
57 + for seq_group_metadata in seq_group_metadata_list:
58 + assert seq_group_metadata.is_prompt
59 + seq_ids = list(seq_group_metadata.seq_data.keys())
60 + assert len(seq_ids) == 1
61 + seq_id = seq_ids[0]
62 +
63 + seq_data = seq_group_metadata.seq_data[seq_id]
64 + prompt_tokens = seq_data.get_token_ids()
65 + prompt_len = len(prompt_tokens)
66 + prompt_lens.append(prompt_len)
67 +
68 + input_tokens.append(prompt_tokens)
69 + input_positions.append(list(range(prompt_len)))
70 +
71 + assert seq_group_metadata.block_tables is not None
72 + block_table = seq_group_metadata.block_tables[seq_id]
73 + assert len(block_table) == 1
74 + input_block_ids.append(block_table[0])
75 +
76 + max_prompt_len = max(prompt_lens)
77 + assert max_prompt_len > 0
78 + input_tokens = make_tensor_with_pad(input_tokens,
79 + max_prompt_len,
80 + pad=0,
81 + dtype=torch.long,
82 + device=self.device)
83 + input_positions = make_tensor_with_pad(input_positions,
84 + max_prompt_len,
85 + pad=0,
86 + dtype=torch.long,
87 + device=self.device)
88 + input_block_ids = torch.tensor(input_block_ids,
89 + dtype=torch.long,
90 + device=self.device)
91 +
92 + return input_tokens, input_positions, input_block_ids, prompt_lens
93 +
94 + def _prepare_decode(
95 + self,
96 + seq_group_metadata_list: List[SequenceGroupMetadata],
97 + ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
98 + assert len(seq_group_metadata_list) > 0
99 + input_tokens: List[List[int]] = []
100 + input_positions: List[List[int]] = []
101 + input_block_ids: List[int] = []
102 + context_lens: List[int] = []
103 +
104 + for seq_group_metadata in seq_group_metadata_list:
105 + assert not seq_group_metadata.is_prompt
106 +
107 + seq_ids = list(seq_group_metadata.seq_data.keys())
108 +
109 + for seq_id in seq_ids:
110 + seq_data = seq_group_metadata.seq_data[seq_id]
111 + generation_token = seq_data.get_last_token_id()
112 + input_tokens.append([generation_token])
113 +
114 + seq_len = seq_data.get_len()
115 + position = seq_len - 1
116 + input_positions.append([position])
117 + context_lens.append(seq_len)
118 +
119 + assert seq_group_metadata.block_tables is not None
120 + block_table = seq_group_metadata.block_tables[seq_id]
121 + assert len(block_table) == 1
122 + input_block_ids.append(block_table[0])
123 +
124 + input_tokens = make_tensor_with_pad(input_tokens,
125 + max_len=1,
126 + pad=0,
127 + dtype=torch.long,
128 + device=self.device)
129 + input_positions = make_tensor_with_pad(input_positions,
130 + max_len=1,
131 + pad=0,
132 + dtype=torch.long,
133 + device=self.device)
134 + context_lens = torch.tensor(context_lens,
135 + dtype=torch.int,
136 + device=self.device)
137 + input_block_ids = torch.tensor(input_block_ids,
138 + dtype=torch.long,
139 + device=self.device)
140 +
141 + return input_tokens, input_positions, input_block_ids
142 +
143 + def _prepare_sample(
144 + self,
145 + seq_group_metadata_list: List[SequenceGroupMetadata],
146 + prompt_lens: List[int],
147 + ) -> SamplingMetadata:
148 + seq_groups: List[Tuple[List[int], SamplingParams]] = []
149 + selected_token_indices: List[int] = []
150 + generators: List[torch.Generator] = []
151 + selected_token_start_idx = 0
152 + categorized_sample_indices = {t: [] for t in SamplingType}
153 + categorized_sample_indices_start_idx = 0
154 + categorized_sampled_token_indices_start_idx = 0
155 +
156 + for i, seq_group_metadata in enumerate(seq_group_metadata_list):
157 + seq_ids = list(seq_group_metadata.seq_data.keys())
158 + sampling_params = seq_group_metadata.sampling_params
159 + seq_groups.append((seq_ids, sampling_params))
160 +
161 + if seq_group_metadata.is_prompt:
162 + assert len(seq_ids) == 1
163 + assert prompt_lens is not None
164 + prompt_len = prompt_lens[i]
165 + if sampling_params.prompt_logprobs is not None:
166 + # NOTE: prompt token positions do not need sample, skip
167 + categorized_sample_indices_start_idx += prompt_len - 1
168 +
169 + categorized_sample_indices[
170 + sampling_params.sampling_type].append([
171 + categorized_sample_indices_start_idx,
172 + categorized_sampled_token_indices_start_idx
173 + ])
174 + categorized_sample_indices_start_idx += 1
175 + categorized_sampled_token_indices_start_idx += 1
176 +
177 + if sampling_params.prompt_logprobs is not None:
178 + selected_token_indices.extend(
179 + range(selected_token_start_idx,
180 + selected_token_start_idx + prompt_len - 1))
181 + selected_token_indices.append(selected_token_start_idx +
182 + prompt_len - 1)
183 + selected_token_start_idx += prompt_len
184 +
185 + if sampling_params.seed is not None:
186 + seq_group_metadata.state.generator = torch.Generator(
187 + device=self.device).manual_seed(sampling_params.seed)
188 + else:
189 + num_seqs = len(seq_ids)
190 + selected_token_indices.extend(
191 + range(selected_token_start_idx,
192 + selected_token_start_idx + num_seqs))
193 + selected_token_start_idx += num_seqs
194 +
195 + categorized_sample_indices[
196 + sampling_params.sampling_type].extend(
197 + zip(
198 + range(
199 + categorized_sample_indices_start_idx,
200 + categorized_sample_indices_start_idx +
201 + num_seqs),
202 + range(
203 + categorized_sampled_token_indices_start_idx,
204 + categorized_sampled_token_indices_start_idx +
205 + num_seqs)))
206 + categorized_sample_indices_start_idx += num_seqs
207 + categorized_sampled_token_indices_start_idx += num_seqs
208 +
209 + if sampling_params.seed is not None:
210 + generators.append(seq_group_metadata.state.generator)
211 +
212 + selected_token_indices = async_tensor_h2d(selected_token_indices,
213 + dtype=torch.long,
214 + target_device=self.device,
215 + pin_memory=self.pin_memory)
216 +
217 + categorized_sample_indices = {
218 + t: maybe_expand_dim(
219 + async_tensor_h2d(seq_ids,
220 + dtype=torch.int,
221 + target_device=self.device,
222 + pin_memory=self.pin_memory), 2, 2)
223 + for t, seq_ids in categorized_sample_indices.items()
224 + }
225 +
226 + seq_data: Dict[int, SequenceData] = {}
227 + for seq_group_metadata in seq_group_metadata_list:
228 + seq_data.update(seq_group_metadata.seq_data)
229 +
230 + sampling_metadata = SamplingMetadata(
231 + seq_groups=seq_groups,
232 + seq_data=seq_data,
233 + prompt_lens=prompt_lens,
234 + selected_token_indices=selected_token_indices,
235 + categorized_sample_indices=categorized_sample_indices,
236 + generators=generators,
237 + )
238 + return sampling_metadata
239 +
240 + def prepare_input_tensors(
241 + self,
242 + seq_group_metadata_list: Optional[List[SequenceGroupMetadata]],
243 + ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, SamplingMetadata]:
244 + # NOTE: We assume that all sequences in the group are all prompts or
245 + # all decodes.
246 + is_prompt = seq_group_metadata_list[0].is_prompt
247 + # Prepare input tensors.
248 + if is_prompt:
249 + (input_tokens, input_positions, input_block_ids,
250 + prompt_lens) = self._prepare_prompt(seq_group_metadata_list)
251 + else:
252 + (input_tokens, input_positions,
253 + input_block_ids) = self._prepare_decode(seq_group_metadata_list)
254 + prompt_lens = []
255 + sampling_metadata = self._prepare_sample(seq_group_metadata_list,
256 + prompt_lens)
257 +
258 + return (input_tokens, input_positions, input_block_ids,
259 + sampling_metadata)
260 +
261 + @torch.inference_mode()
262 + def execute_model(
263 + self,
264 + seq_group_metadata_list: Optional[List[SequenceGroupMetadata]],
265 + ) -> Optional[SamplerOutput]:
266 + (input_tokens, input_positions, input_block_ids, sampling_metadata
267 + ) = self.prepare_input_tensors(seq_group_metadata_list)
268 +
269 + hidden_states = self.model(
270 + input_ids=input_tokens,
271 + positions=input_positions,
272 + input_block_ids=input_block_ids,
273 + )
274 +
275 + # Compute the logits.
276 + logits = self.model.compute_logits(hidden_states, sampling_metadata)
277 +
278 + # Sample the next token.
279 + output = self.model.sample(
280 + logits=logits,
281 + sampling_metadata=sampling_metadata,
282 + )
283 + return output
284 +
285 + @property
286 + def vocab_size(self) -> int:
287 + return self.model_config.get_vocab_size()
vllm/worker/neuron_worker.py
+12 -153
@@ -1,22 +1,17 @@
1 """A Neuron worker class."""
2 -from typing import Dict, List, Optional, Tuple
2 +from typing import List, Optional
3
4 import torch
5 import torch.distributed
6
7 -from vllm.config import (CacheConfig, DeviceConfig, ModelConfig,
8 - ParallelConfig, SchedulerConfig, LoRAConfig)
7 +from vllm.config import (DeviceConfig, ModelConfig, ParallelConfig,
8 + SchedulerConfig)
9 from vllm.model_executor import set_random_seed
10 -from vllm.model_executor.parallel_utils.communication_op import (
11 - broadcast_tensor_dict)
12 -from vllm.model_executor.parallel_utils.parallel_state import (
13 - ensure_model_parallel_initialized)
10 from vllm.sequence import SamplerOutput, SequenceGroupMetadata
15 -from vllm.worker.cache_engine import CacheEngine
16 -from vllm.worker.model_runner import ModelRunner
11 +from vllm.worker.neuron_model_runner import NeuronModelRunner
12
13
19 -class Worker:
14 +class NeuronWorker:
15 """A worker class that executes the model on a group of neuron cores.
16 """
17
@@ -26,168 +21,32 @@ class Worker:
21 parallel_config: ParallelConfig,
22 scheduler_config: SchedulerConfig,
23 device_config: DeviceConfig,
29 - local_rank: int,
30 - rank: int,
31 - distributed_init_method: str,
32 - lora_config: Optional[LoRAConfig] = None,
33 - kv_cache_dtype: Optional[str] = "auto",
34 - is_driver_worker: bool = False,
24 ) -> None:
25 self.model_config = model_config
26 self.parallel_config = parallel_config
27 self.scheduler_config = scheduler_config
28 self.device_config = device_config
40 - self.local_rank = local_rank
41 - self.rank = rank
42 - self.distributed_init_method = distributed_init_method
43 - self.lora_config = lora_config
44 - self.is_driver_worker = is_driver_worker
45 - if self.is_driver_worker:
46 - assert self.rank == 0, "The driver worker must have rank 0."
29
48 - self.model_runner = ModelRunner(model_config,
49 - parallel_config,
50 - scheduler_config,
51 - device_config,
52 - lora_config=self.lora_config,
53 - is_driver_worker=is_driver_worker)
54 - # Uninitialized cache engine. Will be initialized by
55 - # self.init_cache_engine().
56 - self.cache_config = None
57 - self.cache_engine = None
58 - self.cache_events = None
59 - self.gpu_cache = None
30 + self.model_runner = NeuronModelRunner(model_config, parallel_config,
31 + scheduler_config, device_config)
32
61 - def init_model(self) -> None:
62 - # Initialize the distributed environment.
63 - _init_distributed_environment(self.parallel_config,
64 - self.rank,
65 - self.distributed_init_method,
66 - distributed_backend="gloo")
67 -
68 - # Initialize the model.
33 + def init_device(self) -> None:
34 + # Set random seed.
35 set_random_seed(self.model_config.seed)
36
37 def load_model(self):
38 self.model_runner.load_model()
39
74 - @torch.inference_mode()
75 - def profile_num_available_blocks(
76 - self,
77 - block_size: int = 128,
78 - gpu_memory_utilization: float = 0.9,
79 - cpu_swap_space: int = 0,
80 - cache_dtype: str = "float16",
81 - ) -> Tuple[int, int]:
82 - """Simply returns max_num_seqs as num_gpu_blocks, 0 as
83 - num_cpu_blocks."""
84 - num_gpu_blocks = self.scheduler_config.max_num_seqs
85 - num_cpu_blocks = 0
86 - return num_gpu_blocks, num_cpu_blocks
87 -
88 - def init_cache_engine(self, cache_config: CacheConfig) -> None:
89 - self.cache_config = cache_config
90 - self.cache_engine = CacheEngine(self.cache_config, self.model_config,
91 - self.parallel_config)
92 - self.model_runner.set_block_size(self.cache_engine.block_size)
93 -
94 - def warm_up_model(self) -> None:
95 - # Warm up is maintained in transformers-neuronx
96 - pass
97 -
98 - def cache_swap(
99 - self,
100 - blocks_to_swap_in: Dict[int, int],
101 - blocks_to_swap_out: Dict[int, int],
102 - blocks_to_copy: Dict[int, List[int]],
103 - ) -> None:
104 - # Issue cache operations.
105 - issued_cache_op = False
106 - if blocks_to_swap_in:
107 - self.cache_engine.swap_in(blocks_to_swap_in)
108 - issued_cache_op = True
109 - if blocks_to_swap_out:
110 - self.cache_engine.swap_out(blocks_to_swap_out)
111 - issued_cache_op = True
112 - if blocks_to_copy:
113 - self.cache_engine.copy(blocks_to_copy)
114 - issued_cache_op = True
115 -
116 - cache_events = self.cache_events if issued_cache_op else None
117 -
118 - # Wait for cache operations to finish.
119 - if cache_events is not None:
120 - raise NotImplementedError(
121 - "cache operations are not implemented for neuron backend.")
122 -
40 @torch.inference_mode()
41 def execute_model(
42 self,
126 - seq_group_metadata_list: Optional[List[SequenceGroupMetadata]] = None,
127 - blocks_to_swap_in: Optional[Dict[int, int]] = None,
128 - blocks_to_swap_out: Optional[Dict[int, int]] = None,
129 - blocks_to_copy: Optional[Dict[int, List[int]]] = None,
43 + seq_group_metadata_list: List[SequenceGroupMetadata],
44 ) -> Optional[SamplerOutput]:
131 - if self.is_driver_worker:
132 - assert seq_group_metadata_list is not None
133 - num_seq_groups = len(seq_group_metadata_list)
134 - assert blocks_to_swap_in is not None
135 - assert blocks_to_swap_out is not None
136 - assert blocks_to_copy is not None
137 - data = {
138 - "num_seq_groups": num_seq_groups,
139 - "blocks_to_swap_in": blocks_to_swap_in,
140 - "blocks_to_swap_out": blocks_to_swap_out,
141 - "blocks_to_copy": blocks_to_copy,
142 - }
143 - broadcast_tensor_dict(data, src=0)
144 - else:
145 - data = broadcast_tensor_dict(src=0)
146 - num_seq_groups = data["num_seq_groups"]
147 - blocks_to_swap_in = data["blocks_to_swap_in"]
148 - blocks_to_swap_out = data["blocks_to_swap_out"]
149 - blocks_to_copy = data["blocks_to_copy"]
150 -
151 - self.cache_swap(blocks_to_swap_in, blocks_to_swap_out, blocks_to_copy)
45 + num_seq_groups = len(seq_group_metadata_list)
46
47 # If there is no input, we don't need to execute the model.
48 if num_seq_groups == 0:
49 return {}
50
157 - output = self.model_runner.execute_model(seq_group_metadata_list,
158 - self.gpu_cache)
51 + output = self.model_runner.execute_model(seq_group_metadata_list)
52 return output
160 -
161 -
162 -def _init_distributed_environment(
163 - parallel_config: ParallelConfig,
164 - rank: int,
165 - distributed_init_method: Optional[str] = None,
166 - distributed_backend: Optional[str] = None,
167 -) -> None:
168 - """Initialize the distributed environment."""
169 - if torch.distributed.is_initialized():
170 - torch_world_size = torch.distributed.get_world_size()
171 - if torch_world_size != parallel_config.world_size:
172 - raise RuntimeError(
173 - "torch.distributed is already initialized but the torch world "
174 - "size does not match parallel_config.world_size "
175 - f"({torch_world_size} vs. {parallel_config.world_size}).")
176 - elif not distributed_init_method:
177 - raise ValueError(
178 - "distributed_init_method must be set if torch.distributed "
179 - "is not already initialized")
180 - else:
181 - distributed_backend = (distributed_backend
182 - if distributed_backend else "nccl")
183 - torch.distributed.init_process_group(
184 - backend=distributed_backend,
185 - world_size=parallel_config.world_size,
186 - rank=rank,
187 - init_method=distributed_init_method,
188 - )
189 -
190 - # A small all_reduce for warmup.
191 - torch.distributed.all_reduce(torch.zeros(1))
192 - ensure_model_parallel_initialized(parallel_config.tensor_parallel_size,
193 - parallel_config.pipeline_parallel_size)
vllm/worker/worker.py
+2 -2
@@ -67,7 +67,7 @@ class Worker:
67 self.cache_engine = None
68 self.gpu_cache = None
69
70 - def init_model(self, cupy_port: Optional[int] = None) -> None:
70 + def init_device(self, cupy_port: Optional[int] = None) -> None:
71 if self.device_config.device.type == "cuda":
72 # torch.distributed.all_reduce does not free the input tensor until
73 # the synchronization point. This causes the memory usage to grow
@@ -91,7 +91,7 @@ class Worker:
91 # Initialize the distributed environment.
92 init_distributed_environment(self.parallel_config, self.rank,
93 cupy_port, self.distributed_init_method)
94 - # Initialize the model.
94 + # Set random seed.
95 set_random_seed(self.model_config.seed)
96
97 def load_model(self):