A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 08:15 UTC
[misc] improve cloudpickle registration and tests (#10202)
Signed-off-by: youkaichao <youkaichao@gmail.com>
youkaichao committed
Nov 10, 2024 at 16:10 UTC
73b9083e99c02c6ba91f6be9479b88e7e9a94cdf
3 files changed
+50
-31
tests/distributed/test_pipeline_parallel.py
+20
-6
@@ -32,6 +32,8 @@ class PPTestOptions(NamedTuple):
32
multi_node_only: bool
33
trust_remote_code: bool
34
tokenizer_mode: Optional[str]
35
+ load_format: Optional[str] = None
36
+ hf_overrides: Optional[str] = None
37
38
39
@dataclass
@@ -50,6 +52,8 @@ class PPTestSettings:
52
task: TaskOption = "auto",
53
trust_remote_code: bool = False,
54
tokenizer_mode: Optional[str] = None,
55
+ load_format: Optional[str] = None,
56
+ hf_overrides: Optional[str] = None,
57
):
58
return PPTestSettings(
59
parallel_setups=[
@@ -78,7 +82,9 @@ class PPTestSettings:
82
task=task,
83
test_options=PPTestOptions(multi_node_only=multi_node_only,
84
trust_remote_code=trust_remote_code,
81
- tokenizer_mode=tokenizer_mode),
85
+ tokenizer_mode=tokenizer_mode,
86
+ load_format=load_format,
87
+ hf_overrides=hf_overrides),
88
)
89
90
@staticmethod
@@ -90,6 +96,8 @@ class PPTestSettings:
96
multi_node_only: bool = False,
97
trust_remote_code: bool = False,
98
tokenizer_mode: Optional[str] = None,
99
+ load_format: Optional[str] = None,
100
+ hf_overrides: Optional[str] = None,
101
):
102
return PPTestSettings(
103
parallel_setups=[
@@ -102,7 +110,9 @@ class PPTestSettings:
110
task=task,
111
test_options=PPTestOptions(multi_node_only=multi_node_only,
112
trust_remote_code=trust_remote_code,
105
- tokenizer_mode=tokenizer_mode),
113
+ tokenizer_mode=tokenizer_mode,
114
+ load_format=load_format,
115
+ hf_overrides=hf_overrides),
116
)
117
118
def iter_params(self, model_name: str):
@@ -161,9 +171,8 @@ TEXT_GENERATION_MODELS = {
171
"facebook/opt-iml-max-1.3b": PPTestSettings.fast(),
172
"OrionStarAI/Orion-14B-Chat": PPTestSettings.fast(trust_remote_code=True),
173
"microsoft/phi-2": PPTestSettings.fast(),
164
- "microsoft/Phi-3-mini-4k-instruct": PPTestSettings.detailed(trust_remote_code=True, multi_node_only=True), # noqa: E501
174
+ "microsoft/Phi-3.5-MoE-instruct": PPTestSettings.detailed(trust_remote_code=True, multi_node_only=True, load_format="dummy", hf_overrides='{"num_hidden_layers": 4, "hidden_size": 512, "intermediate_size": 800, "num_attention_heads": 4, "num_key_value_heads": 1}'), # noqa: E501
175
"microsoft/Phi-3-small-8k-instruct": PPTestSettings.fast(trust_remote_code=True), # noqa: E501
166
- "microsoft/Phi-3.5-MoE-instruct": PPTestSettings.fast(trust_remote_code=True), # noqa: E501
176
"adept/persimmon-8b-chat": PPTestSettings.fast(),
177
"Qwen/Qwen-7B-Chat": PPTestSettings.fast(trust_remote_code=True),
178
"Qwen/Qwen2-7B-Instruct": PPTestSettings.fast(),
@@ -214,9 +223,9 @@ MULTIMODAL_MODELS = {
223
# NOTE: You can update this on your local machine to run specific tests
224
TEST_MODELS = [
225
# [LANGUAGE GENERATION]
226
+ "microsoft/Phi-3.5-MoE-instruct",
227
"meta-llama/Meta-Llama-3-8B",
228
"ibm/PowerLM-3b",
219
- "microsoft/Phi-3-mini-4k-instruct",
229
# [LANGUAGE EMBEDDING]
230
"intfloat/e5-mistral-7b-instruct",
231
"BAAI/bge-multilingual-gemma2",
@@ -238,7 +247,8 @@ def _compare_tp(
247
method: Literal["generate", "encode"],
248
):
249
tp_size, pp_size, eager_mode, chunked_prefill = parallel_setup
241
- multi_node_only, trust_remote_code, tokenizer_mode = test_options
250
+ multi_node_only, trust_remote_code, tokenizer_mode, \
251
+ load_format, hf_overrides = test_options
252
253
if num_gpus_available < tp_size * pp_size:
254
pytest.skip(f"Need at least {tp_size} x {pp_size} GPUs")
@@ -267,6 +277,10 @@ def _compare_tp(
277
common_args.append("--trust-remote-code")
278
if tokenizer_mode:
279
common_args.extend(["--tokenizer-mode", tokenizer_mode])
280
+ if load_format:
281
+ common_args.extend(["--load-format", load_format])
282
+ if hf_overrides:
283
+ common_args.extend(["--hf-overrides", hf_overrides])
284
285
if (distributed_backend == "ray" and tp_size == 2 and pp_size == 2
286
and chunked_prefill):
vllm/engine/arg_utils.py
-4
@@ -19,8 +19,6 @@ from vllm.logger import init_logger
19
from vllm.model_executor.layers.pooler import PoolingType
20
from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
21
from vllm.platforms import current_platform
22
-from vllm.transformers_utils.config import (
23
- maybe_register_config_serialize_by_value)
22
from vllm.transformers_utils.utils import check_gguf_file
23
from vllm.utils import FlexibleArgumentParser, StoreBoolean
24
@@ -1013,8 +1011,6 @@ class EngineArgs:
1011
"supported for multimodal models and has been disabled.")
1012
self.enable_prefix_caching = False
1013
1016
- maybe_register_config_serialize_by_value(self.trust_remote_code)
1017
-
1014
cache_config = CacheConfig(
1015
# neuron needs block_size = max_model_len
1016
block_size=self.block_size if self.device != "neuron" else
vllm/transformers_utils/config.py
+30
-21
@@ -234,6 +234,9 @@ def get_config(
234
235
patch_rope_scaling(config)
236
237
+ if trust_remote_code:
238
+ maybe_register_config_serialize_by_value()
239
+
240
return config
241
242
@@ -389,33 +392,39 @@ def get_sentence_transformer_tokenizer_config(model: str,
392
return None
393
394
392
-def maybe_register_config_serialize_by_value(trust_remote_code: bool) -> None:
395
+def maybe_register_config_serialize_by_value() -> None:
396
"""Try to register HF model configuration class to serialize by value
397
395
- With trust_remote_code, the config class is typically an instance of a
396
- custom class imported from the HF modules cache. The class will not be
397
- importable in spawned workers by default (and won't exist at all on
398
- other nodes), which breaks serialization of the config.
398
+ If trust_remote_code is set, and the model's config file specifies an
399
+ `AutoConfig` class, then the config class is typically an instance of
400
+ a custom class imported from the HF modules cache.
401
+
402
+ Examples:
403
+
404
+ >>> from transformers import AutoConfig
405
+ >>> klass = AutoConfig.from_pretrained('meta-llama/Meta-Llama-3-8B', trust_remote_code=True)
406
+ >>> klass.__class__ # transformers.models.llama.configuration_llama.LlamaConfig
407
+ >>> import transformers_modules # error, not initialized
408
+ >>> klass = AutoConfig.from_pretrained('deepseek-ai/DeepSeek-V2.5', trust_remote_code=True)
409
+ >>> import transformers_modules # success, initialized
410
+ >>> klass.__class__ # transformers_modules.deepseek-ai.DeepSeek-V2.5.98b11844770b2c3ffc18b175c758a803640f4e77.configuration_deepseek.DeepseekV2Config
411
+
412
+ In the DeepSeek example, the config class is an instance of a custom
413
+ class that is not serializable by default. This class will not be
414
+ importable in spawned workers, and won't exist at all on
415
+ other nodes, which breaks serialization of the config.
416
417
In this function we tell the cloudpickle serialization library to pass
418
instances of these generated classes by value instead of by reference,
419
i.e. the class definition is serialized along with its data so that the
403
- class module does not need to be importable on the receiving end. This
404
- registration only works if the modules cache has already been
405
- initialized.
406
-
420
+ class module does not need to be importable on the receiving end.
421
422
See: https://github.com/cloudpipe/cloudpickle?tab=readme-ov-file#overriding-pickles-serialization-mechanism-for-importable-constructs
409
- """
410
- if not trust_remote_code:
411
- return
412
-
423
+ """ # noqa
424
try:
425
import transformers_modules
426
except ImportError:
416
- logger.debug("Could not import transformers_modules used for remote"
417
- " code. If remote code is not needed remove"
418
- " `--trust-remote-code`.")
427
+ # the config does not need trust_remote_code
428
return
429
430
try:
@@ -428,19 +437,19 @@ def maybe_register_config_serialize_by_value(trust_remote_code: bool) -> None:
437
ray.cloudpickle.register_pickle_by_value(transformers_modules)
438
439
# multiprocessing uses pickle to serialize arguments when using spawn
431
- # Here we get pickle to use cloudpickle to serialize ModelConfig objects
440
+ # Here we get pickle to use cloudpickle to serialize config objects
441
# that contain instances of the custom config class to avoid
442
# serialization problems if the generated module (and model) has a `.`
443
# in its name
444
import multiprocessing
445
import pickle
446
438
- from vllm.config import ModelConfig
447
+ from vllm.config import VllmConfig
448
440
- def _reduce_modelconfig(mc: ModelConfig):
441
- return (pickle.loads, (cloudpickle.dumps(mc), ))
449
+ def _reduce_config(config: VllmConfig):
450
+ return (pickle.loads, (cloudpickle.dumps(config), ))
451
443
- multiprocessing.reducer.register(ModelConfig, _reduce_modelconfig)
452
+ multiprocessing.reducer.register(VllmConfig, _reduce_config)
453
454
except Exception as e:
455
logger.warning(