A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 10:45 UTC
[Hardware][Neuron] Add on-device sampling support for Neuron (#8746)
Co-authored-by: Ashraf Mahgoub <ashymahg@amazon.com>
Chongming Ni committed
Oct 4, 2024 at 16:42 UTC
cc90419e89c358f906e17a5ec484fbe04092c277
2 files changed
+128
-13
vllm/model_executor/model_loader/neuron.py
+50
-9
@@ -1,4 +1,5 @@
1
"""Utilities for selecting and loading neuron models."""
2
+import copy
3
import importlib
4
import os
5
from typing import Dict, List, Optional, Tuple
@@ -13,6 +14,8 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
14
from vllm.model_executor.layers.quantization import get_quantization_config
15
from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
16
from vllm.model_executor.sampling_metadata import SamplingMetadata
17
+from vllm.sequence import (CompletionSequenceGroupOutput, Logprob,
18
+ SequenceOutput)
19
20
TORCH_DTYPE_TO_NEURON_AMP = {
21
"auto": "f32",
@@ -37,15 +40,18 @@ _NEURON_SUPPORTED_MODELS: Dict[str, Tuple[str, str, str]] = {
40
41
class NeuronCasualLM(nn.Module):
42
40
- def __init__(
41
- self,
42
- config: PretrainedConfig,
43
- ) -> None:
43
+ def __init__(self,
44
+ config: PretrainedConfig,
45
+ on_device_sampling_disabled: bool = False) -> None:
46
super().__init__()
47
self.config = config
48
self.logits_processor = LogitsProcessor(config.vocab_size,
49
logits_as_input=True)
48
- self.sampler = Sampler()
50
+
51
+ self.on_device_sampling_disabled = on_device_sampling_disabled
52
+ if self.on_device_sampling_disabled:
53
+ # Use default sampler
54
+ self.sampler = Sampler()
55
56
# Lazy initialized
57
self.model: nn.Module
@@ -71,8 +77,29 @@ class NeuronCasualLM(nn.Module):
77
logits: torch.Tensor,
78
sampling_metadata: SamplingMetadata,
79
) -> Optional[SamplerOutput]:
74
- next_tokens = self.sampler(logits, sampling_metadata)
75
- return next_tokens
80
+
81
+ if self.on_device_sampling_disabled:
82
+ next_tokens = self.sampler(logits, sampling_metadata)
83
+ return next_tokens
84
+
85
+ # On-device sampling outputs the token ids directly.
86
+ sampled_token_ids = logits.flatten()
87
+ next_tokens = []
88
+ sample_idx = 0
89
+ for seq_group in sampling_metadata.seq_groups:
90
+ samples = []
91
+ for seq_id in seq_group.seq_ids:
92
+ token_id = sampled_token_ids[sample_idx].item()
93
+ samples.append(
94
+ SequenceOutput(parent_seq_id=seq_id,
95
+ output_token=token_id,
96
+ logprobs={token_id: Logprob(token_id)}))
97
+ sample_idx += 1
98
+ next_tokens.append(
99
+ CompletionSequenceGroupOutput(samples=samples,
100
+ prompt_logprobs=None))
101
+
102
+ return SamplerOutput(outputs=next_tokens)
103
104
def load_weights(self, model_name_or_path: str, **kwargs):
105
arch = _get_model_architecture(self.config)
@@ -157,10 +184,22 @@ def _get_default_neuron_config(model_config: ModelConfig,
184
quant=neuron_quantization_config_builder(model_config.quantization)
185
if model_config.quantization else None,
186
continuous_batching=continuous_batching_config,
160
- weight_tiling=bool(model_config.quantization))
187
+ weight_tiling=bool(model_config.quantization),
188
+ on_device_generation=_get_neuron_on_device_generation_config(
189
+ model_config))
190
return default_neuron_args
191
192
193
+def _get_neuron_on_device_generation_config(model_config: ModelConfig):
194
+ if not _is_neuron_on_device_sampling_disabled(model_config):
195
+ return copy.deepcopy(model_config.neuron_sampling_params)
196
+ return None
197
+
198
+
199
+def _is_neuron_on_device_sampling_disabled(model_config: ModelConfig) -> bool:
200
+ return not getattr(model_config, "neuron_sampling_params", None)
201
+
202
+
203
def _get_neuron_config_after_override(default_neuron_config,
204
overridden_neuron_config):
205
from transformers_neuronx.config import NeuronConfig
@@ -174,7 +213,9 @@ def get_neuron_model(model_config: ModelConfig,
213
scheduler_config: SchedulerConfig) -> nn.Module:
214
215
# Create a model instance.
177
- model = NeuronCasualLM(model_config.hf_config)
216
+ model = NeuronCasualLM(
217
+ model_config.hf_config,
218
+ _is_neuron_on_device_sampling_disabled(model_config))
219
220
default_neuron_config_args = _get_default_neuron_config(
221
model_config, parallel_config, scheduler_config)
vllm/worker/neuron_model_runner.py
+78
-4
@@ -1,9 +1,11 @@
1
+import os
2
from dataclasses import dataclass
3
from importlib.util import find_spec
4
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union
5
6
import torch
7
from torch import nn
8
+from transformers_neuronx.config import GenerationConfig
9
10
from vllm.config import (DeviceConfig, ModelConfig, ParallelConfig,
11
SchedulerConfig)
@@ -50,6 +52,9 @@ class ModelInputForNeuron(ModelRunnerInputBase):
52
53
class NeuronModelRunner(ModelRunnerBase[ModelInputForNeuron]):
54
55
+ # NEURON has an upper limit on the top_k
56
+ _MAX_NEURON_SAMPLING_TOP_K = 256
57
+
58
def __init__(
59
self,
60
model_config: ModelConfig,
@@ -76,6 +81,34 @@ class NeuronModelRunner(ModelRunnerBase[ModelInputForNeuron]):
81
# Lazy initialization.
82
self.model: nn.Module # initialize after load_model.
83
84
+ # Once NEURON_ON_DEVICE_SAMPLING_DISABLED is set to a non-zero value,
85
+ # turn off on-device sampling.
86
+ self._on_device_sampling_disabled = int(
87
+ os.getenv("NEURON_ON_DEVICE_SAMPLING_DISABLED", "0"))
88
+
89
+ # NEURON needs to update sampling parameters when request IDs change
90
+ # across batches. This variable stores the previous batch's request IDs
91
+ # to determine if an update is needed.
92
+ self._previous_batch_request_ids: List[str] = []
93
+
94
+ if not self._on_device_sampling_disabled:
95
+ logger.warning(
96
+ "On-device sampling is turned on in Neuron by default, only "
97
+ "top_k, top_p, and temperature are current supported sampling "
98
+ "parameters. To turn off the on-device sampling, please set "
99
+ "the environment variable NEURON_ON_DEVICE_SAMPLING_DISABLED=1."
100
+ )
101
+ self.model_config.neuron_sampling_params = GenerationConfig(
102
+ max_length=self.scheduler_config.max_model_len,
103
+ do_sample=True,
104
+ per_batch_line=True,
105
+ top_k=[self._MAX_NEURON_SAMPLING_TOP_K] \
106
+ * self.scheduler_config.max_num_seqs,
107
+ top_p=[1.0] * self.scheduler_config.max_num_seqs,
108
+ temperature=[1.0] * self.scheduler_config.max_num_seqs,
109
+ dynamic=True,
110
+ global_top_k=self._MAX_NEURON_SAMPLING_TOP_K)
111
+
112
def load_model(self) -> None:
113
if find_spec("transformers_neuronx") is not None:
114
self.model = get_neuron_model(
@@ -215,7 +248,7 @@ class NeuronModelRunner(ModelRunnerBase[ModelInputForNeuron]):
248
else:
249
(input_tokens, input_positions,
250
input_block_ids) = self._prepare_decode(seq_group_metadata_list)
218
- seq_lens = []
251
+ seq_lens = None
252
sampling_metadata = SamplingMetadata.prepare(
253
seq_group_metadata_list,
254
seq_lens,
@@ -227,12 +260,49 @@ class NeuronModelRunner(ModelRunnerBase[ModelInputForNeuron]):
260
self.pin_memory,
261
generators=self.get_generators(finished_requests_ids))
262
263
+ if not self._on_device_sampling_disabled:
264
+ # Once the request IDs are changed in current iteration, we will
265
+ # update the on-device sampling parameters.
266
+ current_batch_request_ids = [
267
+ seq_group_meta_data.request_id
268
+ for seq_group_meta_data in seq_group_metadata_list
269
+ ]
270
+ if current_batch_request_ids != self._previous_batch_request_ids:
271
+ self._update_neuron_sampling_params(sampling_metadata)
272
+ self._previous_batch_request_ids = current_batch_request_ids
273
+
274
return ModelInputForNeuron(input_tokens=input_tokens,
275
input_positions=input_positions,
276
input_block_ids=input_block_ids,
277
sampling_metadata=sampling_metadata,
278
multi_modal_kwargs=multi_modal_kwargs)
279
280
+ def _update_neuron_sampling_params(self,
281
+ sampling_metadata: SamplingMetadata):
282
+ # Update Neuron sampling parameters (GenerationConfig in Neuron)
283
+ current_sampling_params = self.model_config.neuron_sampling_params
284
+ assert current_sampling_params is not None, (
285
+ f"Failed to update sampling_params, "
286
+ f"current sampling params is {current_sampling_params}")
287
+
288
+ top_k = current_sampling_params.top_k
289
+ top_p = current_sampling_params.top_p
290
+ temperature = current_sampling_params.temperature
291
+ for index, sequence_group_to_sample in enumerate(
292
+ sampling_metadata.seq_groups):
293
+ top_k[index] = self._convert_to_neuron_top_k(
294
+ sequence_group_to_sample.sampling_params.top_k)
295
+ top_p[index] = sequence_group_to_sample.sampling_params.top_p
296
+ temperature[index] = \
297
+ sequence_group_to_sample.sampling_params.temperature
298
+
299
+ self.model.model.update_generation_config(current_sampling_params)
300
+
301
+ def _convert_to_neuron_top_k(self, top_k: int) -> int:
302
+ if top_k < 0 or top_k > self._MAX_NEURON_SAMPLING_TOP_K:
303
+ return self._MAX_NEURON_SAMPLING_TOP_K
304
+ return top_k
305
+
306
@torch.inference_mode()
307
def execute_model(
308
self,
@@ -253,9 +323,13 @@ class NeuronModelRunner(ModelRunnerBase[ModelInputForNeuron]):
323
device=self.device),
324
)
325
256
- # Compute the logits.
257
- logits = self.model.compute_logits(hidden_states,
258
- model_input.sampling_metadata)
326
+ # Compute the logits only if the on-device sampling is turned off as
327
+ # on-device sampling outputs the token ids.
328
+ if self._on_device_sampling_disabled:
329
+ logits = self.model.compute_logits(hidden_states,
330
+ model_input.sampling_metadata)
331
+ else:
332
+ logits = hidden_states
333
334
# Sample the next token.
335
output = self.model.sample(