@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 2e3b969ec0
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 23:30 UTC

[Platform] add pre_register_and_update function (#12432)

Signed-off-by: wangxiyuan <wangxiyuan1007@gmail.com>

wangxiyuan committed Feb 11, 2025 at 22:06 UTC 2e3b969ec0d46e2cfff041a07f29a2ca4bb82bbd
3 files changed +41 -1
vllm/config.py
+2 -1
@@ -3057,7 +3057,8 @@ class VllmConfig:
3057 kv_transfer_config: KVTransferConfig = field(default=None,
3058 init=True) # type: ignore
3059 # some opaque config, only used to provide additional information
3060 - # for the hash computation, mainly used for testing and debugging.
3060 + # for the hash computation, mainly used for testing, debugging or out of
3061 + # tree config registration.
3062 additional_config: SupportsHash = field(default=None,
3063 init=True) # type: ignore
3064 instance_id: str = ""
vllm/engine/arg_utils.py
+21
@@ -20,6 +20,7 @@ from vllm.config import (CacheConfig, CompilationConfig, ConfigFormat,
20 from vllm.executor.executor_base import ExecutorBase
21 from vllm.logger import init_logger
22 from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
23 +from vllm.plugins import load_general_plugins
24 from vllm.transformers_utils.utils import check_gguf_file
25 from vllm.usage.usage_lib import UsageContext
26 from vllm.utils import FlexibleArgumentParser, StoreBoolean
@@ -203,6 +204,8 @@ class EngineArgs:
204
205 calculate_kv_scales: Optional[bool] = None
206
207 + additional_config: Optional[Dict[str, Any]] = None
208 +
209 def __post_init__(self):
210 if not self.tokenizer:
211 self.tokenizer = self.model
@@ -984,6 +987,14 @@ class EngineArgs:
987 'be loaded from the model checkpoint if available. '
988 'Otherwise, the scales will default to 1.0.')
989
990 + parser.add_argument(
991 + "--additional-config",
992 + type=json.loads,
993 + default=None,
994 + help="Additional config for specified platform in JSON format. "
995 + "Different platforms may support different configs. Make sure the "
996 + "configs are valid for the platform you are using. The input format"
997 + " is like '{\"config_key\":\"config_value\"}'")
998 return parser
999
1000 @classmethod
@@ -1044,6 +1055,9 @@ class EngineArgs:
1055 def create_engine_config(self,
1056 usage_context: Optional[UsageContext] = None
1057 ) -> VllmConfig:
1058 + from vllm.platforms import current_platform
1059 + current_platform.pre_register_and_update()
1060 +
1061 if envs.VLLM_USE_V1:
1062 self._override_v1_engine_args(usage_context)
1063
@@ -1287,6 +1301,7 @@ class EngineArgs:
1301 prompt_adapter_config=prompt_adapter_config,
1302 compilation_config=self.compilation_config,
1303 kv_transfer_config=self.kv_transfer_config,
1304 + additional_config=self.additional_config,
1305 )
1306
1307 if envs.VLLM_USE_V1:
@@ -1347,6 +1362,12 @@ class AsyncEngineArgs(EngineArgs):
1362 parser.add_argument('--disable-log-requests',
1363 action='store_true',
1364 help='Disable logging requests.')
1365 + # Initialize plugin to update the parser, for example, The plugin may
1366 + # adding a new kind of quantization method to --quantization argument or
1367 + # a new device to --device argument.
1368 + load_general_plugins()
1369 + from vllm.platforms import current_platform
1370 + current_platform.pre_register_and_update(parser)
1371 return parser
1372
1373
vllm/platforms/interface.py
+18
@@ -13,8 +13,10 @@ from vllm.logger import init_logger
13
14 if TYPE_CHECKING:
15 from vllm.config import VllmConfig
16 + from vllm.utils import FlexibleArgumentParser
17 else:
18 VllmConfig = None
19 + FlexibleArgumentParser = None
20
21 logger = init_logger(__name__)
22
@@ -223,6 +225,22 @@ class Platform:
225 np.random.seed(seed)
226 torch.manual_seed(seed)
227
228 + @classmethod
229 + def pre_register_and_update(cls,
230 + parser: Optional[FlexibleArgumentParser] = None
231 + ) -> None:
232 + """
233 + Do some pre-registeration or update action for the current platform.
234 +
235 + This function is called before global VllmConfig is initialized or cli
236 + arguments are parsed. It's used for out-of-tree platforms to register or
237 + update the configuration.
238 +
239 + For example, the out-of-tree quantization config can be imported and
240 + registered here dynamically.
241 + """
242 + pass
243 +
244 @classmethod
245 def check_and_update_config(cls, vllm_config: VllmConfig) -> None:
246 """