A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 23:30 UTC
[Platform] add pre_register_and_update function (#12432)
Signed-off-by: wangxiyuan <wangxiyuan1007@gmail.com>
wangxiyuan committed
Feb 11, 2025 at 22:06 UTC
2e3b969ec0d46e2cfff041a07f29a2ca4bb82bbd
3 files changed
+41
-1
vllm/config.py
+2
-1
@@ -3057,7 +3057,8 @@ class VllmConfig:
3057
kv_transfer_config: KVTransferConfig = field(default=None,
3058
init=True) # type: ignore
3059
# some opaque config, only used to provide additional information
3060
- # for the hash computation, mainly used for testing and debugging.
3060
+ # for the hash computation, mainly used for testing, debugging or out of
3061
+ # tree config registration.
3062
additional_config: SupportsHash = field(default=None,
3063
init=True) # type: ignore
3064
instance_id: str = ""
vllm/engine/arg_utils.py
+21
@@ -20,6 +20,7 @@ from vllm.config import (CacheConfig, CompilationConfig, ConfigFormat,
20
from vllm.executor.executor_base import ExecutorBase
21
from vllm.logger import init_logger
22
from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
23
+from vllm.plugins import load_general_plugins
24
from vllm.transformers_utils.utils import check_gguf_file
25
from vllm.usage.usage_lib import UsageContext
26
from vllm.utils import FlexibleArgumentParser, StoreBoolean
@@ -203,6 +204,8 @@ class EngineArgs:
204
205
calculate_kv_scales: Optional[bool] = None
206
207
+ additional_config: Optional[Dict[str, Any]] = None
208
+
209
def __post_init__(self):
210
if not self.tokenizer:
211
self.tokenizer = self.model
@@ -984,6 +987,14 @@ class EngineArgs:
987
'be loaded from the model checkpoint if available. '
988
'Otherwise, the scales will default to 1.0.')
989
990
+ parser.add_argument(
991
+ "--additional-config",
992
+ type=json.loads,
993
+ default=None,
994
+ help="Additional config for specified platform in JSON format. "
995
+ "Different platforms may support different configs. Make sure the "
996
+ "configs are valid for the platform you are using. The input format"
997
+ " is like '{\"config_key\":\"config_value\"}'")
998
return parser
999
1000
@classmethod
@@ -1044,6 +1055,9 @@ class EngineArgs:
1055
def create_engine_config(self,
1056
usage_context: Optional[UsageContext] = None
1057
) -> VllmConfig:
1058
+ from vllm.platforms import current_platform
1059
+ current_platform.pre_register_and_update()
1060
+
1061
if envs.VLLM_USE_V1:
1062
self._override_v1_engine_args(usage_context)
1063
@@ -1287,6 +1301,7 @@ class EngineArgs:
1301
prompt_adapter_config=prompt_adapter_config,
1302
compilation_config=self.compilation_config,
1303
kv_transfer_config=self.kv_transfer_config,
1304
+ additional_config=self.additional_config,
1305
)
1306
1307
if envs.VLLM_USE_V1:
@@ -1347,6 +1362,12 @@ class AsyncEngineArgs(EngineArgs):
1362
parser.add_argument('--disable-log-requests',
1363
action='store_true',
1364
help='Disable logging requests.')
1365
+ # Initialize plugin to update the parser, for example, The plugin may
1366
+ # adding a new kind of quantization method to --quantization argument or
1367
+ # a new device to --device argument.
1368
+ load_general_plugins()
1369
+ from vllm.platforms import current_platform
1370
+ current_platform.pre_register_and_update(parser)
1371
return parser
1372
1373
vllm/platforms/interface.py
+18
@@ -13,8 +13,10 @@ from vllm.logger import init_logger
13
14
if TYPE_CHECKING:
15
from vllm.config import VllmConfig
16
+ from vllm.utils import FlexibleArgumentParser
17
else:
18
VllmConfig = None
19
+ FlexibleArgumentParser = None
20
21
logger = init_logger(__name__)
22
@@ -223,6 +225,22 @@ class Platform:
225
np.random.seed(seed)
226
torch.manual_seed(seed)
227
228
+ @classmethod
229
+ def pre_register_and_update(cls,
230
+ parser: Optional[FlexibleArgumentParser] = None
231
+ ) -> None:
232
+ """
233
+ Do some pre-registeration or update action for the current platform.
234
+
235
+ This function is called before global VllmConfig is initialized or cli
236
+ arguments are parsed. It's used for out-of-tree platforms to register or
237
+ update the configuration.
238
+
239
+ For example, the out-of-tree quantization config can be imported and
240
+ registered here dynamically.
241
+ """
242
+ pass
243
+
244
@classmethod
245
def check_and_update_config(cls, vllm_config: VllmConfig) -> None:
246
"""