@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / cc90419e89
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 10:45 UTC

[Hardware][Neuron] Add on-device sampling support for Neuron (#8746)

Co-authored-by: Ashraf Mahgoub <ashymahg@amazon.com>

Chongming Ni committed Oct 4, 2024 at 16:42 UTC cc90419e89c358f906e17a5ec484fbe04092c277
2 files changed +128 -13
vllm/model_executor/model_loader/neuron.py
+50 -9
@@ -1,4 +1,5 @@
1 """Utilities for selecting and loading neuron models."""
2 +import copy
3 import importlib
4 import os
5 from typing import Dict, List, Optional, Tuple
@@ -13,6 +14,8 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
14 from vllm.model_executor.layers.quantization import get_quantization_config
15 from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
16 from vllm.model_executor.sampling_metadata import SamplingMetadata
17 +from vllm.sequence import (CompletionSequenceGroupOutput, Logprob,
18 + SequenceOutput)
19
20 TORCH_DTYPE_TO_NEURON_AMP = {
21 "auto": "f32",
@@ -37,15 +40,18 @@ _NEURON_SUPPORTED_MODELS: Dict[str, Tuple[str, str, str]] = {
40
41 class NeuronCasualLM(nn.Module):
42
40 - def __init__(
41 - self,
42 - config: PretrainedConfig,
43 - ) -> None:
43 + def __init__(self,
44 + config: PretrainedConfig,
45 + on_device_sampling_disabled: bool = False) -> None:
46 super().__init__()
47 self.config = config
48 self.logits_processor = LogitsProcessor(config.vocab_size,
49 logits_as_input=True)
48 - self.sampler = Sampler()
50 +
51 + self.on_device_sampling_disabled = on_device_sampling_disabled
52 + if self.on_device_sampling_disabled:
53 + # Use default sampler
54 + self.sampler = Sampler()
55
56 # Lazy initialized
57 self.model: nn.Module
@@ -71,8 +77,29 @@ class NeuronCasualLM(nn.Module):
77 logits: torch.Tensor,
78 sampling_metadata: SamplingMetadata,
79 ) -> Optional[SamplerOutput]:
74 - next_tokens = self.sampler(logits, sampling_metadata)
75 - return next_tokens
80 +
81 + if self.on_device_sampling_disabled:
82 + next_tokens = self.sampler(logits, sampling_metadata)
83 + return next_tokens
84 +
85 + # On-device sampling outputs the token ids directly.
86 + sampled_token_ids = logits.flatten()
87 + next_tokens = []
88 + sample_idx = 0
89 + for seq_group in sampling_metadata.seq_groups:
90 + samples = []
91 + for seq_id in seq_group.seq_ids:
92 + token_id = sampled_token_ids[sample_idx].item()
93 + samples.append(
94 + SequenceOutput(parent_seq_id=seq_id,
95 + output_token=token_id,
96 + logprobs={token_id: Logprob(token_id)}))
97 + sample_idx += 1
98 + next_tokens.append(
99 + CompletionSequenceGroupOutput(samples=samples,
100 + prompt_logprobs=None))
101 +
102 + return SamplerOutput(outputs=next_tokens)
103
104 def load_weights(self, model_name_or_path: str, **kwargs):
105 arch = _get_model_architecture(self.config)
@@ -157,10 +184,22 @@ def _get_default_neuron_config(model_config: ModelConfig,
184 quant=neuron_quantization_config_builder(model_config.quantization)
185 if model_config.quantization else None,
186 continuous_batching=continuous_batching_config,
160 - weight_tiling=bool(model_config.quantization))
187 + weight_tiling=bool(model_config.quantization),
188 + on_device_generation=_get_neuron_on_device_generation_config(
189 + model_config))
190 return default_neuron_args
191
192
193 +def _get_neuron_on_device_generation_config(model_config: ModelConfig):
194 + if not _is_neuron_on_device_sampling_disabled(model_config):
195 + return copy.deepcopy(model_config.neuron_sampling_params)
196 + return None
197 +
198 +
199 +def _is_neuron_on_device_sampling_disabled(model_config: ModelConfig) -> bool:
200 + return not getattr(model_config, "neuron_sampling_params", None)
201 +
202 +
203 def _get_neuron_config_after_override(default_neuron_config,
204 overridden_neuron_config):
205 from transformers_neuronx.config import NeuronConfig
@@ -174,7 +213,9 @@ def get_neuron_model(model_config: ModelConfig,
213 scheduler_config: SchedulerConfig) -> nn.Module:
214
215 # Create a model instance.
177 - model = NeuronCasualLM(model_config.hf_config)
216 + model = NeuronCasualLM(
217 + model_config.hf_config,
218 + _is_neuron_on_device_sampling_disabled(model_config))
219
220 default_neuron_config_args = _get_default_neuron_config(
221 model_config, parallel_config, scheduler_config)
vllm/worker/neuron_model_runner.py
+78 -4
@@ -1,9 +1,11 @@
1 +import os
2 from dataclasses import dataclass
3 from importlib.util import find_spec
4 from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union
5
6 import torch
7 from torch import nn
8 +from transformers_neuronx.config import GenerationConfig
9
10 from vllm.config import (DeviceConfig, ModelConfig, ParallelConfig,
11 SchedulerConfig)
@@ -50,6 +52,9 @@ class ModelInputForNeuron(ModelRunnerInputBase):
52
53 class NeuronModelRunner(ModelRunnerBase[ModelInputForNeuron]):
54
55 + # NEURON has an upper limit on the top_k
56 + _MAX_NEURON_SAMPLING_TOP_K = 256
57 +
58 def __init__(
59 self,
60 model_config: ModelConfig,
@@ -76,6 +81,34 @@ class NeuronModelRunner(ModelRunnerBase[ModelInputForNeuron]):
81 # Lazy initialization.
82 self.model: nn.Module # initialize after load_model.
83
84 + # Once NEURON_ON_DEVICE_SAMPLING_DISABLED is set to a non-zero value,
85 + # turn off on-device sampling.
86 + self._on_device_sampling_disabled = int(
87 + os.getenv("NEURON_ON_DEVICE_SAMPLING_DISABLED", "0"))
88 +
89 + # NEURON needs to update sampling parameters when request IDs change
90 + # across batches. This variable stores the previous batch's request IDs
91 + # to determine if an update is needed.
92 + self._previous_batch_request_ids: List[str] = []
93 +
94 + if not self._on_device_sampling_disabled:
95 + logger.warning(
96 + "On-device sampling is turned on in Neuron by default, only "
97 + "top_k, top_p, and temperature are current supported sampling "
98 + "parameters. To turn off the on-device sampling, please set "
99 + "the environment variable NEURON_ON_DEVICE_SAMPLING_DISABLED=1."
100 + )
101 + self.model_config.neuron_sampling_params = GenerationConfig(
102 + max_length=self.scheduler_config.max_model_len,
103 + do_sample=True,
104 + per_batch_line=True,
105 + top_k=[self._MAX_NEURON_SAMPLING_TOP_K] \
106 + * self.scheduler_config.max_num_seqs,
107 + top_p=[1.0] * self.scheduler_config.max_num_seqs,
108 + temperature=[1.0] * self.scheduler_config.max_num_seqs,
109 + dynamic=True,
110 + global_top_k=self._MAX_NEURON_SAMPLING_TOP_K)
111 +
112 def load_model(self) -> None:
113 if find_spec("transformers_neuronx") is not None:
114 self.model = get_neuron_model(
@@ -215,7 +248,7 @@ class NeuronModelRunner(ModelRunnerBase[ModelInputForNeuron]):
248 else:
249 (input_tokens, input_positions,
250 input_block_ids) = self._prepare_decode(seq_group_metadata_list)
218 - seq_lens = []
251 + seq_lens = None
252 sampling_metadata = SamplingMetadata.prepare(
253 seq_group_metadata_list,
254 seq_lens,
@@ -227,12 +260,49 @@ class NeuronModelRunner(ModelRunnerBase[ModelInputForNeuron]):
260 self.pin_memory,
261 generators=self.get_generators(finished_requests_ids))
262
263 + if not self._on_device_sampling_disabled:
264 + # Once the request IDs are changed in current iteration, we will
265 + # update the on-device sampling parameters.
266 + current_batch_request_ids = [
267 + seq_group_meta_data.request_id
268 + for seq_group_meta_data in seq_group_metadata_list
269 + ]
270 + if current_batch_request_ids != self._previous_batch_request_ids:
271 + self._update_neuron_sampling_params(sampling_metadata)
272 + self._previous_batch_request_ids = current_batch_request_ids
273 +
274 return ModelInputForNeuron(input_tokens=input_tokens,
275 input_positions=input_positions,
276 input_block_ids=input_block_ids,
277 sampling_metadata=sampling_metadata,
278 multi_modal_kwargs=multi_modal_kwargs)
279
280 + def _update_neuron_sampling_params(self,
281 + sampling_metadata: SamplingMetadata):
282 + # Update Neuron sampling parameters (GenerationConfig in Neuron)
283 + current_sampling_params = self.model_config.neuron_sampling_params
284 + assert current_sampling_params is not None, (
285 + f"Failed to update sampling_params, "
286 + f"current sampling params is {current_sampling_params}")
287 +
288 + top_k = current_sampling_params.top_k
289 + top_p = current_sampling_params.top_p
290 + temperature = current_sampling_params.temperature
291 + for index, sequence_group_to_sample in enumerate(
292 + sampling_metadata.seq_groups):
293 + top_k[index] = self._convert_to_neuron_top_k(
294 + sequence_group_to_sample.sampling_params.top_k)
295 + top_p[index] = sequence_group_to_sample.sampling_params.top_p
296 + temperature[index] = \
297 + sequence_group_to_sample.sampling_params.temperature
298 +
299 + self.model.model.update_generation_config(current_sampling_params)
300 +
301 + def _convert_to_neuron_top_k(self, top_k: int) -> int:
302 + if top_k < 0 or top_k > self._MAX_NEURON_SAMPLING_TOP_K:
303 + return self._MAX_NEURON_SAMPLING_TOP_K
304 + return top_k
305 +
306 @torch.inference_mode()
307 def execute_model(
308 self,
@@ -253,9 +323,13 @@ class NeuronModelRunner(ModelRunnerBase[ModelInputForNeuron]):
323 device=self.device),
324 )
325
256 - # Compute the logits.
257 - logits = self.model.compute_logits(hidden_states,
258 - model_input.sampling_metadata)
326 + # Compute the logits only if the on-device sampling is turned off as
327 + # on-device sampling outputs the token ids.
328 + if self._on_device_sampling_disabled:
329 + logits = self.model.compute_logits(hidden_states,
330 + model_input.sampling_metadata)
331 + else:
332 + logits = hidden_states
333
334 # Sample the next token.
335 output = self.model.sample(