@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 73b9083e99
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 08:15 UTC

[misc] improve cloudpickle registration and tests (#10202)

Signed-off-by: youkaichao <youkaichao@gmail.com>

youkaichao committed Nov 10, 2024 at 16:10 UTC 73b9083e99c02c6ba91f6be9479b88e7e9a94cdf
3 files changed +50 -31
tests/distributed/test_pipeline_parallel.py
+20 -6
@@ -32,6 +32,8 @@ class PPTestOptions(NamedTuple):
32 multi_node_only: bool
33 trust_remote_code: bool
34 tokenizer_mode: Optional[str]
35 + load_format: Optional[str] = None
36 + hf_overrides: Optional[str] = None
37
38
39 @dataclass
@@ -50,6 +52,8 @@ class PPTestSettings:
52 task: TaskOption = "auto",
53 trust_remote_code: bool = False,
54 tokenizer_mode: Optional[str] = None,
55 + load_format: Optional[str] = None,
56 + hf_overrides: Optional[str] = None,
57 ):
58 return PPTestSettings(
59 parallel_setups=[
@@ -78,7 +82,9 @@ class PPTestSettings:
82 task=task,
83 test_options=PPTestOptions(multi_node_only=multi_node_only,
84 trust_remote_code=trust_remote_code,
81 - tokenizer_mode=tokenizer_mode),
85 + tokenizer_mode=tokenizer_mode,
86 + load_format=load_format,
87 + hf_overrides=hf_overrides),
88 )
89
90 @staticmethod
@@ -90,6 +96,8 @@ class PPTestSettings:
96 multi_node_only: bool = False,
97 trust_remote_code: bool = False,
98 tokenizer_mode: Optional[str] = None,
99 + load_format: Optional[str] = None,
100 + hf_overrides: Optional[str] = None,
101 ):
102 return PPTestSettings(
103 parallel_setups=[
@@ -102,7 +110,9 @@ class PPTestSettings:
110 task=task,
111 test_options=PPTestOptions(multi_node_only=multi_node_only,
112 trust_remote_code=trust_remote_code,
105 - tokenizer_mode=tokenizer_mode),
113 + tokenizer_mode=tokenizer_mode,
114 + load_format=load_format,
115 + hf_overrides=hf_overrides),
116 )
117
118 def iter_params(self, model_name: str):
@@ -161,9 +171,8 @@ TEXT_GENERATION_MODELS = {
171 "facebook/opt-iml-max-1.3b": PPTestSettings.fast(),
172 "OrionStarAI/Orion-14B-Chat": PPTestSettings.fast(trust_remote_code=True),
173 "microsoft/phi-2": PPTestSettings.fast(),
164 - "microsoft/Phi-3-mini-4k-instruct": PPTestSettings.detailed(trust_remote_code=True, multi_node_only=True), # noqa: E501
174 + "microsoft/Phi-3.5-MoE-instruct": PPTestSettings.detailed(trust_remote_code=True, multi_node_only=True, load_format="dummy", hf_overrides='{"num_hidden_layers": 4, "hidden_size": 512, "intermediate_size": 800, "num_attention_heads": 4, "num_key_value_heads": 1}'), # noqa: E501
175 "microsoft/Phi-3-small-8k-instruct": PPTestSettings.fast(trust_remote_code=True), # noqa: E501
166 - "microsoft/Phi-3.5-MoE-instruct": PPTestSettings.fast(trust_remote_code=True), # noqa: E501
176 "adept/persimmon-8b-chat": PPTestSettings.fast(),
177 "Qwen/Qwen-7B-Chat": PPTestSettings.fast(trust_remote_code=True),
178 "Qwen/Qwen2-7B-Instruct": PPTestSettings.fast(),
@@ -214,9 +223,9 @@ MULTIMODAL_MODELS = {
223 # NOTE: You can update this on your local machine to run specific tests
224 TEST_MODELS = [
225 # [LANGUAGE GENERATION]
226 + "microsoft/Phi-3.5-MoE-instruct",
227 "meta-llama/Meta-Llama-3-8B",
228 "ibm/PowerLM-3b",
219 - "microsoft/Phi-3-mini-4k-instruct",
229 # [LANGUAGE EMBEDDING]
230 "intfloat/e5-mistral-7b-instruct",
231 "BAAI/bge-multilingual-gemma2",
@@ -238,7 +247,8 @@ def _compare_tp(
247 method: Literal["generate", "encode"],
248 ):
249 tp_size, pp_size, eager_mode, chunked_prefill = parallel_setup
241 - multi_node_only, trust_remote_code, tokenizer_mode = test_options
250 + multi_node_only, trust_remote_code, tokenizer_mode, \
251 + load_format, hf_overrides = test_options
252
253 if num_gpus_available < tp_size * pp_size:
254 pytest.skip(f"Need at least {tp_size} x {pp_size} GPUs")
@@ -267,6 +277,10 @@ def _compare_tp(
277 common_args.append("--trust-remote-code")
278 if tokenizer_mode:
279 common_args.extend(["--tokenizer-mode", tokenizer_mode])
280 + if load_format:
281 + common_args.extend(["--load-format", load_format])
282 + if hf_overrides:
283 + common_args.extend(["--hf-overrides", hf_overrides])
284
285 if (distributed_backend == "ray" and tp_size == 2 and pp_size == 2
286 and chunked_prefill):
vllm/engine/arg_utils.py
-4
@@ -19,8 +19,6 @@ from vllm.logger import init_logger
19 from vllm.model_executor.layers.pooler import PoolingType
20 from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
21 from vllm.platforms import current_platform
22 -from vllm.transformers_utils.config import (
23 - maybe_register_config_serialize_by_value)
22 from vllm.transformers_utils.utils import check_gguf_file
23 from vllm.utils import FlexibleArgumentParser, StoreBoolean
24
@@ -1013,8 +1011,6 @@ class EngineArgs:
1011 "supported for multimodal models and has been disabled.")
1012 self.enable_prefix_caching = False
1013
1016 - maybe_register_config_serialize_by_value(self.trust_remote_code)
1017 -
1014 cache_config = CacheConfig(
1015 # neuron needs block_size = max_model_len
1016 block_size=self.block_size if self.device != "neuron" else
vllm/transformers_utils/config.py
+30 -21
@@ -234,6 +234,9 @@ def get_config(
234
235 patch_rope_scaling(config)
236
237 + if trust_remote_code:
238 + maybe_register_config_serialize_by_value()
239 +
240 return config
241
242
@@ -389,33 +392,39 @@ def get_sentence_transformer_tokenizer_config(model: str,
392 return None
393
394
392 -def maybe_register_config_serialize_by_value(trust_remote_code: bool) -> None:
395 +def maybe_register_config_serialize_by_value() -> None:
396 """Try to register HF model configuration class to serialize by value
397
395 - With trust_remote_code, the config class is typically an instance of a
396 - custom class imported from the HF modules cache. The class will not be
397 - importable in spawned workers by default (and won't exist at all on
398 - other nodes), which breaks serialization of the config.
398 + If trust_remote_code is set, and the model's config file specifies an
399 + `AutoConfig` class, then the config class is typically an instance of
400 + a custom class imported from the HF modules cache.
401 +
402 + Examples:
403 +
404 + >>> from transformers import AutoConfig
405 + >>> klass = AutoConfig.from_pretrained('meta-llama/Meta-Llama-3-8B', trust_remote_code=True)
406 + >>> klass.__class__ # transformers.models.llama.configuration_llama.LlamaConfig
407 + >>> import transformers_modules # error, not initialized
408 + >>> klass = AutoConfig.from_pretrained('deepseek-ai/DeepSeek-V2.5', trust_remote_code=True)
409 + >>> import transformers_modules # success, initialized
410 + >>> klass.__class__ # transformers_modules.deepseek-ai.DeepSeek-V2.5.98b11844770b2c3ffc18b175c758a803640f4e77.configuration_deepseek.DeepseekV2Config
411 +
412 + In the DeepSeek example, the config class is an instance of a custom
413 + class that is not serializable by default. This class will not be
414 + importable in spawned workers, and won't exist at all on
415 + other nodes, which breaks serialization of the config.
416
417 In this function we tell the cloudpickle serialization library to pass
418 instances of these generated classes by value instead of by reference,
419 i.e. the class definition is serialized along with its data so that the
403 - class module does not need to be importable on the receiving end. This
404 - registration only works if the modules cache has already been
405 - initialized.
406 -
420 + class module does not need to be importable on the receiving end.
421
422 See: https://github.com/cloudpipe/cloudpickle?tab=readme-ov-file#overriding-pickles-serialization-mechanism-for-importable-constructs
409 - """
410 - if not trust_remote_code:
411 - return
412 -
423 + """ # noqa
424 try:
425 import transformers_modules
426 except ImportError:
416 - logger.debug("Could not import transformers_modules used for remote"
417 - " code. If remote code is not needed remove"
418 - " `--trust-remote-code`.")
427 + # the config does not need trust_remote_code
428 return
429
430 try:
@@ -428,19 +437,19 @@ def maybe_register_config_serialize_by_value(trust_remote_code: bool) -> None:
437 ray.cloudpickle.register_pickle_by_value(transformers_modules)
438
439 # multiprocessing uses pickle to serialize arguments when using spawn
431 - # Here we get pickle to use cloudpickle to serialize ModelConfig objects
440 + # Here we get pickle to use cloudpickle to serialize config objects
441 # that contain instances of the custom config class to avoid
442 # serialization problems if the generated module (and model) has a `.`
443 # in its name
444 import multiprocessing
445 import pickle
446
438 - from vllm.config import ModelConfig
447 + from vllm.config import VllmConfig
448
440 - def _reduce_modelconfig(mc: ModelConfig):
441 - return (pickle.loads, (cloudpickle.dumps(mc), ))
449 + def _reduce_config(config: VllmConfig):
450 + return (pickle.loads, (cloudpickle.dumps(config), ))
451
443 - multiprocessing.reducer.register(ModelConfig, _reduce_modelconfig)
452 + multiprocessing.reducer.register(VllmConfig, _reduce_config)
453
454 except Exception as e:
455 logger.warning(