A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[Bugfix]: serialize config by value for --trust-remote-code (#6751)
Signed-off-by: Travis Johnson <tsjohnso@us.ibm.com> Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com>
Travis Johnson committed
Oct 21, 2024 at 20:46 UTC
b729901139c93edd9ef8d48a16d269f070d8ba42
4 files changed
+103
-28
tests/distributed/test_pipeline_parallel.py
+35
-28
@@ -28,19 +28,25 @@ class ParallelSetup(NamedTuple):
28
chunked_prefill: bool
29
30
31
+class PPTestOptions(NamedTuple):
32
+ multi_node_only: bool
33
+ trust_remote_code: bool
34
+ tokenizer_mode: Optional[str]
35
+
36
+
37
@dataclass
38
class PPTestSettings:
39
parallel_setups: List[ParallelSetup]
40
distributed_backends: List[str]
41
task: TaskOption
36
- trust_remote_code: bool
37
- tokenizer_mode: Optional[str]
42
+ test_options: PPTestOptions
43
44
@staticmethod
45
def detailed(
46
*,
47
tp_base: int = 1,
48
pp_base: int = 2,
49
+ multi_node_only: bool = False,
50
task: TaskOption = "auto",
51
trust_remote_code: bool = False,
52
tokenizer_mode: Optional[str] = None,
@@ -70,8 +76,9 @@ class PPTestSettings:
76
],
77
distributed_backends=["mp", "ray"],
78
task=task,
73
- trust_remote_code=trust_remote_code,
74
- tokenizer_mode=tokenizer_mode,
79
+ test_options=PPTestOptions(multi_node_only=multi_node_only,
80
+ trust_remote_code=trust_remote_code,
81
+ tokenizer_mode=tokenizer_mode),
82
)
83
84
@staticmethod
@@ -80,6 +87,7 @@ class PPTestSettings:
87
tp_base: int = 1,
88
pp_base: int = 2,
89
task: TaskOption = "auto",
90
+ multi_node_only: bool = False,
91
trust_remote_code: bool = False,
92
tokenizer_mode: Optional[str] = None,
93
):
@@ -92,15 +100,18 @@ class PPTestSettings:
100
],
101
distributed_backends=["mp"],
102
task=task,
95
- trust_remote_code=trust_remote_code,
96
- tokenizer_mode=tokenizer_mode,
103
+ test_options=PPTestOptions(multi_node_only=multi_node_only,
104
+ trust_remote_code=trust_remote_code,
105
+ tokenizer_mode=tokenizer_mode),
106
)
107
108
def iter_params(self, model_name: str):
109
+ opts = self.test_options
110
+
111
for parallel_setup in self.parallel_setups:
112
for distributed_backend in self.distributed_backends:
113
yield (model_name, parallel_setup, distributed_backend,
103
- self.task, self.trust_remote_code, self.tokenizer_mode)
114
+ self.task, opts)
115
116
117
# NOTE: You can adjust tp_base and/or pp_base locally to fit the model in GPU
@@ -110,6 +121,7 @@ class PPTestSettings:
121
GENERATION_MODEL_SETTINGS = {
122
# [DETAILED TESTS]
123
"meta-llama/Meta-Llama-3-8B": PPTestSettings.detailed(),
124
+ "microsoft/Phi-3-mini-4k-instruct": PPTestSettings.detailed(trust_remote_code=True, multi_node_only=True), # noqa: E501
125
# [FAST TESTS]
126
# Uses Llama
127
# "BAAI/AquilaChat-7B": PPTestSettings.fast(),
@@ -151,10 +163,8 @@ GENERATION_MODEL_SETTINGS = {
163
"facebook/opt-iml-max-1.3b": PPTestSettings.fast(),
164
"OrionStarAI/Orion-14B-Chat": PPTestSettings.fast(trust_remote_code=True),
165
"microsoft/phi-2": PPTestSettings.fast(),
154
- "microsoft/Phi-3-mini-4k-instruct": PPTestSettings.fast(),
166
"microsoft/Phi-3-small-8k-instruct": PPTestSettings.fast(trust_remote_code=True), # noqa: E501
156
- # FIXME: https://github.com/vllm-project/vllm/issues/8553
157
- # "microsoft/Phi-3.5-MoE-instruct": PPTestSettings.fast(trust_remote_code=True), # noqa: E501
167
+ "microsoft/Phi-3.5-MoE-instruct": PPTestSettings.fast(trust_remote_code=True), # noqa: E501
168
"adept/persimmon-8b-chat": PPTestSettings.fast(),
169
"Qwen/Qwen-7B-Chat": PPTestSettings.fast(trust_remote_code=True),
170
"Qwen/Qwen2-beta-7B-Chat": PPTestSettings.fast(),
@@ -205,6 +215,7 @@ TEST_MODELS = [
215
# [LANGUAGE GENERATION]
216
"meta-llama/Meta-Llama-3-8B",
217
"ibm/PowerLM-3b",
218
+ "microsoft/Phi-3-mini-4k-instruct",
219
# [LANGUAGE EMBEDDING]
220
"intfloat/e5-mistral-7b-instruct",
221
"BAAI/bge-multilingual-gemma2",
@@ -220,19 +231,21 @@ def _compare_tp(
231
parallel_setup: ParallelSetup,
232
distributed_backend: str,
233
task: TaskOption,
223
- trust_remote_code: bool,
224
- tokenizer_mode: Optional[str],
234
+ test_options: PPTestOptions,
235
num_gpus_available: int,
236
*,
227
- method: Literal["generate", "encode"] = "encode",
237
+ method: Literal["generate", "encode"],
238
):
239
tp_size, pp_size, eager_mode, chunked_prefill = parallel_setup
240
+ multi_node_only, trust_remote_code, tokenizer_mode = test_options
241
242
if num_gpus_available < tp_size * pp_size:
243
pytest.skip(f"Need at least {tp_size} x {pp_size} GPUs")
244
if VLLM_MULTI_NODE and distributed_backend == "mp":
245
pytest.skip("Skipping multi-node pipeline parallel test for "
246
"multiprocessing distributed backend")
247
+ if multi_node_only and not VLLM_MULTI_NODE:
248
+ pytest.skip("Not in multi-node setting")
249
250
common_args = [
251
# use half precision for speed and memory savings in CI environment
@@ -307,7 +320,7 @@ def _compare_tp(
320
321
@pytest.mark.parametrize(
322
("model_name", "parallel_setup", "distributed_backend", "task",
310
- "trust_remote_code", "tokenizer_mode"),
323
+ "test_options"),
324
[
325
params for model_name, settings in GENERATION_MODEL_SETTINGS.items()
326
for params in settings.iter_params(model_name)
@@ -320,23 +333,21 @@ def test_tp_language_generation(
333
parallel_setup: ParallelSetup,
334
distributed_backend: str,
335
task: TaskOption,
323
- trust_remote_code: bool,
324
- tokenizer_mode: Optional[str],
336
+ test_options: PPTestOptions,
337
num_gpus_available,
338
):
339
_compare_tp(model_name,
340
parallel_setup,
341
distributed_backend,
342
task,
331
- trust_remote_code,
332
- tokenizer_mode,
343
+ test_options,
344
num_gpus_available,
345
method="generate")
346
347
348
@pytest.mark.parametrize(
349
("model_name", "parallel_setup", "distributed_backend", "task",
339
- "trust_remote_code", "tokenizer_mode"),
350
+ "test_options"),
351
[
352
params for model_name, settings in EMBEDDING_MODEL_SETTINGS.items()
353
for params in settings.iter_params(model_name)
@@ -349,23 +360,21 @@ def test_tp_language_embedding(
360
parallel_setup: ParallelSetup,
361
distributed_backend: str,
362
task: TaskOption,
352
- trust_remote_code: bool,
353
- tokenizer_mode: Optional[str],
363
+ test_options: PPTestOptions,
364
num_gpus_available,
365
):
366
_compare_tp(model_name,
367
parallel_setup,
368
distributed_backend,
369
task,
360
- trust_remote_code,
361
- tokenizer_mode,
370
+ test_options,
371
num_gpus_available,
372
method="encode")
373
374
375
@pytest.mark.parametrize(
376
("model_name", "parallel_setup", "distributed_backend", "task",
368
- "trust_remote_code", "tokenizer_mode"),
377
+ "test_options"),
378
[
379
params for model_name, settings in MULTIMODAL_MODEL_SETTINGS.items()
380
for params in settings.iter_params(model_name)
@@ -378,15 +387,13 @@ def test_tp_multimodal_generation(
387
parallel_setup: ParallelSetup,
388
distributed_backend: str,
389
task: TaskOption,
381
- trust_remote_code: bool,
382
- tokenizer_mode: Optional[str],
390
+ test_options: PPTestOptions,
391
num_gpus_available,
392
):
393
_compare_tp(model_name,
394
parallel_setup,
395
distributed_backend,
396
task,
389
- trust_remote_code,
390
- tokenizer_mode,
397
+ test_options,
398
num_gpus_available,
399
method="generate")
vllm/engine/arg_utils.py
+4
@@ -16,6 +16,8 @@ from vllm.config import (CacheConfig, ConfigFormat, DecodingConfig,
16
from vllm.executor.executor_base import ExecutorBase
17
from vllm.logger import init_logger
18
from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
19
+from vllm.transformers_utils.config import (
20
+ maybe_register_config_serialize_by_value)
21
from vllm.transformers_utils.utils import check_gguf_file
22
from vllm.utils import FlexibleArgumentParser
23
@@ -924,6 +926,8 @@ class EngineArgs:
926
"supported for multimodal models and has been disabled.")
927
self.enable_prefix_caching = False
928
929
+ maybe_register_config_serialize_by_value(self.trust_remote_code)
930
+
931
cache_config = CacheConfig(
932
# neuron needs block_size = max_model_len
933
block_size=self.block_size if self.device != "neuron" else
vllm/transformers_utils/config.py
+62
@@ -232,6 +232,68 @@ def get_config(
232
return config
233
234
235
+def maybe_register_config_serialize_by_value(trust_remote_code: bool) -> None:
236
+ """Try to register HF model configuration class to serialize by value
237
+
238
+ With trust_remote_code, the config class is typically an instance of a
239
+ custom class imported from the HF modules cache. The class will not be
240
+ importable in spawned workers by default (and won't exist at all on
241
+ other nodes), which breaks serialization of the config.
242
+
243
+ In this function we tell the cloudpickle serialization library to pass
244
+ instances of these generated classes by value instead of by reference,
245
+ i.e. the class definition is serialized along with its data so that the
246
+ class module does not need to be importable on the receiving end. This
247
+ registration only works if the modules cache has already been
248
+ initialized.
249
+
250
+
251
+ See: https://github.com/cloudpipe/cloudpickle?tab=readme-ov-file#overriding-pickles-serialization-mechanism-for-importable-constructs
252
+ """
253
+ if not trust_remote_code:
254
+ return
255
+
256
+ try:
257
+ import transformers_modules
258
+ except ImportError:
259
+ logger.debug("Could not import transformers_modules used for remote"
260
+ " code. If remote code is not needed remove"
261
+ " `--trust-remote-code`.")
262
+ return
263
+
264
+ try:
265
+ import cloudpickle
266
+ cloudpickle.register_pickle_by_value(transformers_modules)
267
+
268
+ # ray vendors its own version of cloudpickle
269
+ from vllm.executor.ray_utils import ray
270
+ if ray:
271
+ ray.cloudpickle.register_pickle_by_value(transformers_modules)
272
+
273
+ # multiprocessing uses pickle to serialize arguments when using spawn
274
+ # Here we get pickle to use cloudpickle to serialize ModelConfig objects
275
+ # that contain instances of the custom config class to avoid
276
+ # serialization problems if the generated module (and model) has a `.`
277
+ # in its name
278
+ import multiprocessing
279
+ import pickle
280
+
281
+ from vllm.config import ModelConfig
282
+
283
+ def _reduce_modelconfig(mc: ModelConfig):
284
+ return (pickle.loads, (cloudpickle.dumps(mc), ))
285
+
286
+ multiprocessing.reducer.register(ModelConfig, _reduce_modelconfig)
287
+
288
+ except Exception as e:
289
+ logger.warning(
290
+ "Unable to register remote classes used by"
291
+ " trust_remote_code with by-value serialization. This may"
292
+ " lead to a later error. If remote code is not needed"
293
+ " remove `--trust-remote-code`",
294
+ exc_info=e)
295
+
296
+
297
def load_params_config(model, revision) -> PretrainedConfig:
298
# This function loads a params.json config which
299
# should be used when loading models in mistral format
vllm/utils.py
+2
@@ -968,6 +968,8 @@ def flatten_2d_lists(lists: List[List[T]]) -> List[T]:
968
return [item for sublist in lists for item in sublist]
969
970
971
+# TODO: This function can be removed if transformer_modules classes are
972
+# serialized by value when communicating between processes
973
def init_cached_hf_modules() -> None:
974
"""
975
Lazy initialization of the Hugging Face modules.