A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
[Neuron] Adding support for context-lenght, token-gen buckets. (#7885)
Co-authored-by: Harsha Bikki <harbikh@amazon.com>
Harsha vardhan manoj Bikki committed
Aug 29, 2024 at 13:58 UTC
257afc37c5b3e4c6d491d105337387989b013aee
2 files changed
+33
-11
examples/offline_inference_neuron.py
+9
-2
@@ -1,5 +1,12 @@
1
+import os
2
+
3
from vllm import LLM, SamplingParams
4
5
+# creates XLA hlo graphs for all the context length buckets.
6
+os.environ['NEURON_CONTEXT_LENGTH_BUCKETS'] = "128,512,1024,2048"
7
+# creates XLA hlo graphs for all the token gen buckets.
8
+os.environ['NEURON_TOKEN_GEN_BUCKETS'] = "128,512,1024,2048"
9
+
10
# Sample prompts.
11
prompts = [
12
"Hello, my name is",
@@ -19,8 +26,8 @@ llm = LLM(
26
# Currently, this is a known limitation in continuous batching support
27
# in transformers-neuronx.
28
# TODO(liangfu): Support paged-attention in transformers-neuronx.
22
- max_model_len=128,
23
- block_size=128,
29
+ max_model_len=2048,
30
+ block_size=2048,
31
# The device can be automatically detected when AWS Neuron SDK is installed.
32
# The device argument can be either unspecified for automated detection,
33
# or explicitly assigned.
vllm/model_executor/model_loader/neuron.py
+24
-9
@@ -1,7 +1,7 @@
1
"""Utilities for selecting and loading neuron models."""
2
import importlib
3
import os
4
-from typing import Dict, Optional, Tuple
4
+from typing import Dict, List, Optional, Tuple
5
6
import torch
7
import torch.nn as nn
@@ -109,6 +109,17 @@ def _get_model_architecture(config: PretrainedConfig) -> str:
109
f"{list(_NEURON_SUPPORTED_MODELS.keys())}")
110
111
112
+def _get_buckets(env: str, default_value: List[int]) -> List[int]:
113
+ env_value = os.getenv(env)
114
+ if env_value is None:
115
+ return default_value
116
+ buckets_remove_empty = filter(
117
+ lambda x: x is not None and len(x.strip()) > 0, env_value.split(","))
118
+ buckets_int = map(int, buckets_remove_empty)
119
+ buckets_list = list(buckets_int)
120
+ return buckets_list
121
+
122
+
123
def get_neuron_model(model_config: ModelConfig,
124
parallel_config: ParallelConfig,
125
scheduler_config: SchedulerConfig) -> nn.Module:
@@ -123,14 +134,18 @@ def get_neuron_model(model_config: ModelConfig,
134
neuron_config = NeuronConfig(
135
continuous_batching=continuous_batching_config)
136
137
+ context_length_estimates = _get_buckets("NEURON_CONTEXT_LENGTH_BUCKETS",
138
+ [scheduler_config.max_model_len])
139
+ n_positions = _get_buckets("NEURON_TOKEN_GEN_BUCKETS",
140
+ [scheduler_config.max_model_len])
141
+
142
# Load the weights from the cached or downloaded files.
127
- model.load_weights(
128
- model_config.model,
129
- tp_degree=parallel_config.tensor_parallel_size,
130
- amp=TORCH_DTYPE_TO_NEURON_AMP[model_config.dtype],
131
- neuron_config=neuron_config,
132
- context_length_estimate=[scheduler_config.max_model_len],
133
- n_positions=[scheduler_config.max_model_len],
134
- batch_size=scheduler_config.max_num_seqs)
143
+ model.load_weights(model_config.model,
144
+ tp_degree=parallel_config.tensor_parallel_size,
145
+ amp=TORCH_DTYPE_TO_NEURON_AMP[model_config.dtype],
146
+ neuron_config=neuron_config,
147
+ context_length_estimate=context_length_estimates,
148
+ n_positions=n_positions,
149
+ batch_size=scheduler_config.max_num_seqs)
150
151
return model.eval()