@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 257afc37c5
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 17:00 UTC

[Neuron] Adding support for context-lenght, token-gen buckets. (#7885)

Co-authored-by: Harsha Bikki <harbikh@amazon.com>

Harsha vardhan manoj Bikki committed Aug 29, 2024 at 13:58 UTC 257afc37c5b3e4c6d491d105337387989b013aee
2 files changed +33 -11
examples/offline_inference_neuron.py
+9 -2
@@ -1,5 +1,12 @@
1 +import os
2 +
3 from vllm import LLM, SamplingParams
4
5 +# creates XLA hlo graphs for all the context length buckets.
6 +os.environ['NEURON_CONTEXT_LENGTH_BUCKETS'] = "128,512,1024,2048"
7 +# creates XLA hlo graphs for all the token gen buckets.
8 +os.environ['NEURON_TOKEN_GEN_BUCKETS'] = "128,512,1024,2048"
9 +
10 # Sample prompts.
11 prompts = [
12 "Hello, my name is",
@@ -19,8 +26,8 @@ llm = LLM(
26 # Currently, this is a known limitation in continuous batching support
27 # in transformers-neuronx.
28 # TODO(liangfu): Support paged-attention in transformers-neuronx.
22 - max_model_len=128,
23 - block_size=128,
29 + max_model_len=2048,
30 + block_size=2048,
31 # The device can be automatically detected when AWS Neuron SDK is installed.
32 # The device argument can be either unspecified for automated detection,
33 # or explicitly assigned.
vllm/model_executor/model_loader/neuron.py
+24 -9
@@ -1,7 +1,7 @@
1 """Utilities for selecting and loading neuron models."""
2 import importlib
3 import os
4 -from typing import Dict, Optional, Tuple
4 +from typing import Dict, List, Optional, Tuple
5
6 import torch
7 import torch.nn as nn
@@ -109,6 +109,17 @@ def _get_model_architecture(config: PretrainedConfig) -> str:
109 f"{list(_NEURON_SUPPORTED_MODELS.keys())}")
110
111
112 +def _get_buckets(env: str, default_value: List[int]) -> List[int]:
113 + env_value = os.getenv(env)
114 + if env_value is None:
115 + return default_value
116 + buckets_remove_empty = filter(
117 + lambda x: x is not None and len(x.strip()) > 0, env_value.split(","))
118 + buckets_int = map(int, buckets_remove_empty)
119 + buckets_list = list(buckets_int)
120 + return buckets_list
121 +
122 +
123 def get_neuron_model(model_config: ModelConfig,
124 parallel_config: ParallelConfig,
125 scheduler_config: SchedulerConfig) -> nn.Module:
@@ -123,14 +134,18 @@ def get_neuron_model(model_config: ModelConfig,
134 neuron_config = NeuronConfig(
135 continuous_batching=continuous_batching_config)
136
137 + context_length_estimates = _get_buckets("NEURON_CONTEXT_LENGTH_BUCKETS",
138 + [scheduler_config.max_model_len])
139 + n_positions = _get_buckets("NEURON_TOKEN_GEN_BUCKETS",
140 + [scheduler_config.max_model_len])
141 +
142 # Load the weights from the cached or downloaded files.
127 - model.load_weights(
128 - model_config.model,
129 - tp_degree=parallel_config.tensor_parallel_size,
130 - amp=TORCH_DTYPE_TO_NEURON_AMP[model_config.dtype],
131 - neuron_config=neuron_config,
132 - context_length_estimate=[scheduler_config.max_model_len],
133 - n_positions=[scheduler_config.max_model_len],
134 - batch_size=scheduler_config.max_num_seqs)
143 + model.load_weights(model_config.model,
144 + tp_degree=parallel_config.tensor_parallel_size,
145 + amp=TORCH_DTYPE_TO_NEURON_AMP[model_config.dtype],
146 + neuron_config=neuron_config,
147 + context_length_estimate=context_length_estimates,
148 + n_positions=n_positions,
149 + batch_size=scheduler_config.max_num_seqs)
150
151 return model.eval()