A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
[Neuron] Adding support for adding/ overriding neuron configuration a… (#8062)
Co-authored-by: Harsha Bikki <harbikh@amazon.com>
Harsha vardhan manoj Bikki committed
Sep 4, 2024 at 16:33 UTC
008cf886c9361e696f70a15a282d72b58686468a
8 files changed
+243
-42
examples/offline_inference_neuron_int8_quantization.py
new
+50
@@ -0,0 +1,50 @@
1
+import os
2
+
3
+from vllm import LLM, SamplingParams
4
+
5
+# creates XLA hlo graphs for all the context length buckets.
6
+os.environ['NEURON_CONTEXT_LENGTH_BUCKETS'] = "128,512,1024,2048"
7
+# creates XLA hlo graphs for all the token gen buckets.
8
+os.environ['NEURON_TOKEN_GEN_BUCKETS'] = "128,512,1024,2048"
9
+# Quantizes neuron model weight to int8 ,
10
+# The default config for quantization is int8 dtype.
11
+os.environ['NEURON_QUANT_DTYPE'] = "s8"
12
+
13
+# Sample prompts.
14
+prompts = [
15
+ "Hello, my name is",
16
+ "The president of the United States is",
17
+ "The capital of France is",
18
+ "The future of AI is",
19
+]
20
+# Create a sampling params object.
21
+sampling_params = SamplingParams(temperature=0.8, top_p=0.95)
22
+
23
+# Create an LLM.
24
+llm = LLM(
25
+ model="TinyLlama/TinyLlama-1.1B-Chat-v1.0",
26
+ max_num_seqs=8,
27
+ # The max_model_len and block_size arguments are required to be same as
28
+ # max sequence length when targeting neuron device.
29
+ # Currently, this is a known limitation in continuous batching support
30
+ # in transformers-neuronx.
31
+ # TODO(liangfu): Support paged-attention in transformers-neuronx.
32
+ max_model_len=2048,
33
+ block_size=2048,
34
+ # The device can be automatically detected when AWS Neuron SDK is installed.
35
+ # The device argument can be either unspecified for automated detection,
36
+ # or explicitly assigned.
37
+ device="neuron",
38
+ quantization="neuron_quant",
39
+ override_neuron_config={
40
+ "cast_logits_dtype": "bfloat16",
41
+ },
42
+ tensor_parallel_size=2)
43
+# Generate texts from the prompts. The output is a list of RequestOutput objects
44
+# that contain the prompt, generated text, and other information.
45
+outputs = llm.generate(prompts, sampling_params)
46
+# Print the outputs.
47
+for output in outputs:
48
+ prompt = output.prompt
49
+ generated_text = output.outputs[0].text
50
+ print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
vllm/config.py
+41
-28
@@ -1,8 +1,8 @@
1
import enum
2
import json
3
from dataclasses import dataclass, field, fields
4
-from typing import (TYPE_CHECKING, ClassVar, List, Mapping, Optional, Tuple,
5
- Type, Union)
4
+from typing import (TYPE_CHECKING, Any, ClassVar, Dict, List, Mapping,
5
+ Optional, Tuple, Type, Union)
6
7
import torch
8
from transformers import PretrainedConfig
@@ -115,35 +115,39 @@ class ModelConfig:
115
the model name will be the same as `model`.
116
limit_mm_per_prompt: Maximum number of data instances per modality
117
per prompt. Only applicable for multimodal models.
118
+ override_neuron_config: Initialize non default neuron config or
119
+ override default neuron config that are specific to Neuron devices,
120
+ this argument will be used to configure the neuron config that
121
+ can not be gathered from the vllm arguments.
122
"""
123
124
def __init__(
121
- self,
122
- model: str,
123
- tokenizer: str,
124
- tokenizer_mode: str,
125
- trust_remote_code: bool,
126
- dtype: Union[str, torch.dtype],
127
- seed: int,
128
- revision: Optional[str] = None,
129
- code_revision: Optional[str] = None,
130
- rope_scaling: Optional[dict] = None,
131
- rope_theta: Optional[float] = None,
132
- tokenizer_revision: Optional[str] = None,
133
- max_model_len: Optional[int] = None,
134
- spec_target_max_model_len: Optional[int] = None,
135
- quantization: Optional[str] = None,
136
- quantization_param_path: Optional[str] = None,
137
- enforce_eager: Optional[bool] = None,
138
- max_context_len_to_capture: Optional[int] = None,
139
- max_seq_len_to_capture: Optional[int] = None,
140
- max_logprobs: int = 20,
141
- disable_sliding_window: bool = False,
142
- skip_tokenizer_init: bool = False,
143
- served_model_name: Optional[Union[str, List[str]]] = None,
144
- limit_mm_per_prompt: Optional[Mapping[str, int]] = None,
145
- use_async_output_proc: bool = True,
146
- ) -> None:
125
+ self,
126
+ model: str,
127
+ tokenizer: str,
128
+ tokenizer_mode: str,
129
+ trust_remote_code: bool,
130
+ dtype: Union[str, torch.dtype],
131
+ seed: int,
132
+ revision: Optional[str] = None,
133
+ code_revision: Optional[str] = None,
134
+ rope_scaling: Optional[dict] = None,
135
+ rope_theta: Optional[float] = None,
136
+ tokenizer_revision: Optional[str] = None,
137
+ max_model_len: Optional[int] = None,
138
+ spec_target_max_model_len: Optional[int] = None,
139
+ quantization: Optional[str] = None,
140
+ quantization_param_path: Optional[str] = None,
141
+ enforce_eager: Optional[bool] = None,
142
+ max_context_len_to_capture: Optional[int] = None,
143
+ max_seq_len_to_capture: Optional[int] = None,
144
+ max_logprobs: int = 20,
145
+ disable_sliding_window: bool = False,
146
+ skip_tokenizer_init: bool = False,
147
+ served_model_name: Optional[Union[str, List[str]]] = None,
148
+ limit_mm_per_prompt: Optional[Mapping[str, int]] = None,
149
+ use_async_output_proc: bool = True,
150
+ override_neuron_config: Optional[Dict[str, Any]] = None) -> None:
151
self.model = model
152
self.tokenizer = tokenizer
153
self.tokenizer_mode = tokenizer_mode
@@ -227,6 +231,9 @@ class ModelConfig:
231
limit_mm_per_prompt)
232
if not self.skip_tokenizer_init:
233
self._verify_tokenizer_mode()
234
+
235
+ self.override_neuron_config = override_neuron_config if is_neuron(
236
+ ) else None
237
self._verify_embedding_mode()
238
self._verify_quantization()
239
self._verify_cuda_graph()
@@ -275,6 +282,7 @@ class ModelConfig:
282
"experts_int8"
283
]
284
tpu_supported_quantization = ["tpu_int8"]
285
+ neuron_supported_quantization = ["neuron_quant"]
286
if self.quantization is not None:
287
self.quantization = self.quantization.lower()
288
@@ -329,6 +337,11 @@ class ModelConfig:
337
"Using AWQ quantization with ROCm, but VLLM_USE_TRITON_AWQ"
338
" is not set, enabling VLLM_USE_TRITON_AWQ.")
339
envs.VLLM_USE_TRITON_AWQ = True
340
+ if is_neuron(
341
+ ) and self.quantization not in neuron_supported_quantization:
342
+ raise ValueError(
343
+ f"{self.quantization} quantization is currently not "
344
+ f"supported in Neuron Backend.")
345
346
def _verify_cuda_graph(self) -> None:
347
if self.max_seq_len_to_capture is None:
vllm/engine/arg_utils.py
+14
-3
@@ -2,8 +2,8 @@ import argparse
2
import dataclasses
3
import json
4
from dataclasses import dataclass
5
-from typing import (TYPE_CHECKING, Dict, List, Mapping, Optional, Tuple, Type,
6
- Union)
5
+from typing import (TYPE_CHECKING, Any, Dict, List, Mapping, Optional, Tuple,
6
+ Type, Union)
7
8
import torch
9
@@ -149,6 +149,7 @@ class EngineArgs:
149
otlp_traces_endpoint: Optional[str] = None
150
collect_detailed_traces: Optional[str] = None
151
disable_async_output_proc: bool = False
152
+ override_neuron_config: Optional[Dict[str, Any]] = None
153
154
def __post_init__(self):
155
if self.tokenizer is None:
@@ -742,6 +743,16 @@ class EngineArgs:
743
default=EngineArgs.disable_async_output_proc,
744
help="Disable async output processing. This may result in "
745
"lower performance.")
746
+ parser.add_argument(
747
+ '--override-neuron-config',
748
+ type=lambda configs: {
749
+ str(key): value
750
+ for key, value in
751
+ (config.split(':') for config in configs.split(','))
752
+ },
753
+ default=None,
754
+ help="override or set neuron device configuration.")
755
+
756
return parser
757
758
@classmethod
@@ -802,7 +813,7 @@ class EngineArgs:
813
served_model_name=self.served_model_name,
814
limit_mm_per_prompt=self.limit_mm_per_prompt,
815
use_async_output_proc=not self.disable_async_output_proc,
805
- )
816
+ override_neuron_config=self.override_neuron_config)
817
cache_config = CacheConfig(
818
block_size=self.block_size if self.device != "neuron" else
819
self.max_model_len, # neuron needs block_size = max_model_len
vllm/engine/llm_engine.py
+2
@@ -214,6 +214,7 @@ class LLMEngine:
214
"Initializing an LLM engine (v%s) with config: "
215
"model=%r, speculative_config=%r, tokenizer=%r, "
216
"skip_tokenizer_init=%s, tokenizer_mode=%s, revision=%s, "
217
+ "override_neuron_config=%s, "
218
"rope_scaling=%r, rope_theta=%r, tokenizer_revision=%s, "
219
"trust_remote_code=%s, dtype=%s, max_seq_len=%d, "
220
"download_dir=%r, load_format=%s, tensor_parallel_size=%d, "
@@ -232,6 +233,7 @@ class LLMEngine:
233
model_config.skip_tokenizer_init,
234
model_config.tokenizer_mode,
235
model_config.revision,
236
+ model_config.override_neuron_config,
237
model_config.rope_scaling,
238
model_config.rope_theta,
239
model_config.tokenizer_revision,
vllm/model_executor/layers/quantization/__init__.py
+3
@@ -22,6 +22,8 @@ from vllm.model_executor.layers.quantization.gptq_marlin import (
22
from vllm.model_executor.layers.quantization.gptq_marlin_24 import (
23
GPTQMarlin24Config)
24
from vllm.model_executor.layers.quantization.marlin import MarlinConfig
25
+from vllm.model_executor.layers.quantization.neuron_quant import (
26
+ NeuronQuantConfig)
27
from vllm.model_executor.layers.quantization.qqq import QQQConfig
28
from vllm.model_executor.layers.quantization.squeezellm import SqueezeLLMConfig
29
from vllm.model_executor.layers.quantization.tpu_int8 import Int8TpuConfig
@@ -46,6 +48,7 @@ QUANTIZATION_METHODS: Dict[str, Type[QuantizationConfig]] = {
48
"bitsandbytes": BitsAndBytesConfig,
49
"qqq": QQQConfig,
50
"experts_int8": ExpertsInt8Config,
51
+ "neuron_quant": NeuronQuantConfig,
52
}
53
54
vllm/model_executor/layers/quantization/neuron_quant.py
new
+67
@@ -0,0 +1,67 @@
1
+import os
2
+from importlib.util import find_spec
3
+from typing import Any, Dict, List, Optional
4
+
5
+from torch.nn import Module
6
+
7
+from vllm.model_executor.layers.quantization.base_config import (
8
+ QuantizationConfig)
9
+
10
+SUPPORTED_QUANT_DTYPE_LIST = ['s8', 'f8e4m3fn']
11
+
12
+
13
+class NeuronQuantConfig(QuantizationConfig):
14
+ """Int8 Quantization Config class for Neuron Backend."""
15
+
16
+ def __init__(
17
+ self,
18
+ dequant_dtype: str = "f16",
19
+ quantize_method: str = "vector_dynamic",
20
+ ) -> None:
21
+ self.quant_dtype = os.getenv("NEURON_QUANT_DTYPE", "s8")
22
+ if self.quant_dtype not in SUPPORTED_QUANT_DTYPE_LIST:
23
+ raise ValueError(
24
+ f"Neuron quantization datatype {self.quant_dtype} is not valid,"
25
+ f"the quantization datatype should match one of the below types"
26
+ f"{SUPPORTED_QUANT_DTYPE_LIST}")
27
+ self.dequant_dtype = dequant_dtype
28
+ self.quantize_method = quantize_method
29
+
30
+ def get_name(self) -> str:
31
+ return "neuron_quant"
32
+
33
+ def get_supported_act_dtypes(self) -> List[str]:
34
+ return SUPPORTED_QUANT_DTYPE_LIST
35
+
36
+ @classmethod
37
+ def get_min_capability(cls) -> int:
38
+ raise NotImplementedError(
39
+ "This function should not be called with Neuron Backend")
40
+
41
+ @staticmethod
42
+ def get_config_filenames() -> List[str]:
43
+ return []
44
+
45
+ @classmethod
46
+ def from_config(cls, config: Dict[str, Any]) -> "NeuronQuantConfig":
47
+ quantize_method = cls.get_from_keys(config, ["quantize_method"])
48
+ dequant_dtype = cls.get_from_keys(config, ["dequant_dtype"])
49
+ return cls(dequant_dtype=dequant_dtype,
50
+ quantize_method=quantize_method)
51
+
52
+ def get_quant_method(self, layer: Module, prefix: str) -> Optional[Any]:
53
+ if find_spec("transformers_neuronx") is not None:
54
+ return self.get_quantization_config()
55
+ else:
56
+ raise NotImplementedError(
57
+ "Neuron Quantization is only supported through"
58
+ " transformers_neuronx.")
59
+
60
+ def get_scaled_act_names(self) -> List[str]:
61
+ return []
62
+
63
+ def get_quantization_config(self):
64
+ from transformers_neuronx.config import QuantizationConfig
65
+ return QuantizationConfig(quant_dtype=self.quant_dtype,
66
+ dequant_dtype=self.dequant_dtype,
67
+ quantize_method=self.quantize_method)
vllm/model_executor/model_loader/neuron.py
+57
-8
@@ -10,6 +10,7 @@ from transformers import PretrainedConfig
10
11
from vllm.config import ModelConfig, ParallelConfig, SchedulerConfig
12
from vllm.model_executor.layers.logits_processor import LogitsProcessor
13
+from vllm.model_executor.layers.quantization import get_quantization_config
14
from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
15
from vllm.model_executor.sampling_metadata import SamplingMetadata
16
@@ -81,8 +82,7 @@ class NeuronCasualLM(nn.Module):
82
neuronx_model_cls = getattr(neuronx_module, neuronx_model_cls_name)
83
84
split_model_dir = f"{model_name_or_path}-split"
84
- if os.path.isdir(os.path.join(model_name_or_path,
85
- "pytorch_model.bin")):
85
+ if _is_pretrained_neuron_checkpoint(model_name_or_path):
86
split_model_dir = model_name_or_path
87
elif not os.path.exists(f"{model_name_or_path}-split"):
88
hf_model_cls = getattr(transformers, hf_model_cls_name)
@@ -97,6 +97,23 @@ class NeuronCasualLM(nn.Module):
97
self.model.to_neuron()
98
99
100
+def _is_pretrained_neuron_checkpoint(model_name_or_path: str) -> bool:
101
+ # Checking if the neuron checkpoint is saved in the old format.
102
+ if os.path.isdir(os.path.join(model_name_or_path, "pytorch_model.bin")):
103
+ return True
104
+ # Checking if the neuron checkpoint is saved in the new format.
105
+ pretrained_split_files = ["config.json", "generation_config.json"]
106
+ pretrained_split_format = ".safetensors"
107
+ for file in pretrained_split_files:
108
+ file_path = os.path.join(model_name_or_path, file)
109
+ if not os.path.isfile(file_path):
110
+ return False
111
+ for file in os.listdir(model_name_or_path):
112
+ if file.endswith(pretrained_split_format):
113
+ return True
114
+ return False
115
+
116
+
117
def _get_model_architecture(config: PretrainedConfig) -> str:
118
architectures = getattr(config, "architectures", [])
119
for arch in architectures:
@@ -119,19 +136,51 @@ def _get_buckets(env: str, default_value: List[int]) -> List[int]:
136
return buckets_list
137
138
139
+def _get_default_neuron_config(model_config: ModelConfig,
140
+ parallel_config: ParallelConfig,
141
+ scheduler_config: SchedulerConfig):
142
+ from transformers_neuronx.config import ContinuousBatchingConfig
143
+ from transformers_neuronx.constants import LAYOUT_BSH
144
+
145
+ continuous_batching_config = ContinuousBatchingConfig(
146
+ batch_size_for_shared_caches=scheduler_config.max_num_seqs)
147
+ quant_config = dict(
148
+ dequant_dtype=TORCH_DTYPE_TO_NEURON_AMP[model_config.dtype],
149
+ quantize_method="vector_dynamic")
150
+ neuron_quantization_config_builder = lambda quant: get_quantization_config(
151
+ quant).from_config(quant_config).get_quant_method(None, "")
152
+ # TODO: Add Paged attention config to the default neuron arguments.
153
+ default_neuron_args = dict(
154
+ collectives_layout=LAYOUT_BSH,
155
+ attention_layout=LAYOUT_BSH,
156
+ fuse_qkv=True,
157
+ quant=neuron_quantization_config_builder(model_config.quantization)
158
+ if model_config.quantization else None,
159
+ continuous_batching=continuous_batching_config,
160
+ weight_tiling=bool(model_config.quantization))
161
+ return default_neuron_args
162
+
163
+
164
+def _get_neuron_config_after_override(default_neuron_config,
165
+ overridden_neuron_config):
166
+ from transformers_neuronx.config import NeuronConfig
167
+ overridden_neuron_config = overridden_neuron_config or {}
168
+ default_neuron_config.update(overridden_neuron_config)
169
+ return NeuronConfig(**default_neuron_config)
170
+
171
+
172
def get_neuron_model(model_config: ModelConfig,
173
parallel_config: ParallelConfig,
174
scheduler_config: SchedulerConfig) -> nn.Module:
125
- from transformers_neuronx.config import (ContinuousBatchingConfig,
126
- NeuronConfig)
175
176
# Create a model instance.
177
model = NeuronCasualLM(model_config.hf_config)
178
131
- continuous_batching_config = ContinuousBatchingConfig(
132
- batch_size_for_shared_caches=scheduler_config.max_num_seqs)
133
- neuron_config = NeuronConfig(
134
- continuous_batching=continuous_batching_config)
179
+ default_neuron_config_args = _get_default_neuron_config(
180
+ model_config, parallel_config, scheduler_config)
181
+
182
+ neuron_config = _get_neuron_config_after_override(
183
+ default_neuron_config_args, model_config.override_neuron_config)
184
185
context_length_estimates = _get_buckets("NEURON_CONTEXT_LENGTH_BUCKETS",
186
[scheduler_config.max_model_len])
vllm/worker/neuron_model_runner.py
+9
-3
@@ -1,4 +1,5 @@
1
from dataclasses import dataclass
2
+from importlib.util import find_spec
3
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union
4
5
import torch
@@ -76,9 +77,14 @@ class NeuronModelRunner(ModelRunnerBase[ModelInputForNeuron]):
77
self.model: nn.Module # initialize after load_model.
78
79
def load_model(self) -> None:
79
- self.model = get_neuron_model(self.model_config,
80
- parallel_config=self.parallel_config,
81
- scheduler_config=self.scheduler_config)
80
+ if find_spec("transformers_neuronx") is not None:
81
+ self.model = get_neuron_model(
82
+ self.model_config,
83
+ parallel_config=self.parallel_config,
84
+ scheduler_config=self.scheduler_config)
85
+ else:
86
+ raise NotImplementedError(
87
+ "Supports only Transformer-NeuronX based models.")
88
89
def _prepare_prompt(
90
self,