@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / b729901139
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[Bugfix]: serialize config by value for --trust-remote-code (#6751)

Signed-off-by: Travis Johnson <tsjohnso@us.ibm.com> Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com>

Travis Johnson committed Oct 21, 2024 at 20:46 UTC b729901139c93edd9ef8d48a16d269f070d8ba42
4 files changed +103 -28
tests/distributed/test_pipeline_parallel.py
+35 -28
@@ -28,19 +28,25 @@ class ParallelSetup(NamedTuple):
28 chunked_prefill: bool
29
30
31 +class PPTestOptions(NamedTuple):
32 + multi_node_only: bool
33 + trust_remote_code: bool
34 + tokenizer_mode: Optional[str]
35 +
36 +
37 @dataclass
38 class PPTestSettings:
39 parallel_setups: List[ParallelSetup]
40 distributed_backends: List[str]
41 task: TaskOption
36 - trust_remote_code: bool
37 - tokenizer_mode: Optional[str]
42 + test_options: PPTestOptions
43
44 @staticmethod
45 def detailed(
46 *,
47 tp_base: int = 1,
48 pp_base: int = 2,
49 + multi_node_only: bool = False,
50 task: TaskOption = "auto",
51 trust_remote_code: bool = False,
52 tokenizer_mode: Optional[str] = None,
@@ -70,8 +76,9 @@ class PPTestSettings:
76 ],
77 distributed_backends=["mp", "ray"],
78 task=task,
73 - trust_remote_code=trust_remote_code,
74 - tokenizer_mode=tokenizer_mode,
79 + test_options=PPTestOptions(multi_node_only=multi_node_only,
80 + trust_remote_code=trust_remote_code,
81 + tokenizer_mode=tokenizer_mode),
82 )
83
84 @staticmethod
@@ -80,6 +87,7 @@ class PPTestSettings:
87 tp_base: int = 1,
88 pp_base: int = 2,
89 task: TaskOption = "auto",
90 + multi_node_only: bool = False,
91 trust_remote_code: bool = False,
92 tokenizer_mode: Optional[str] = None,
93 ):
@@ -92,15 +100,18 @@ class PPTestSettings:
100 ],
101 distributed_backends=["mp"],
102 task=task,
95 - trust_remote_code=trust_remote_code,
96 - tokenizer_mode=tokenizer_mode,
103 + test_options=PPTestOptions(multi_node_only=multi_node_only,
104 + trust_remote_code=trust_remote_code,
105 + tokenizer_mode=tokenizer_mode),
106 )
107
108 def iter_params(self, model_name: str):
109 + opts = self.test_options
110 +
111 for parallel_setup in self.parallel_setups:
112 for distributed_backend in self.distributed_backends:
113 yield (model_name, parallel_setup, distributed_backend,
103 - self.task, self.trust_remote_code, self.tokenizer_mode)
114 + self.task, opts)
115
116
117 # NOTE: You can adjust tp_base and/or pp_base locally to fit the model in GPU
@@ -110,6 +121,7 @@ class PPTestSettings:
121 GENERATION_MODEL_SETTINGS = {
122 # [DETAILED TESTS]
123 "meta-llama/Meta-Llama-3-8B": PPTestSettings.detailed(),
124 + "microsoft/Phi-3-mini-4k-instruct": PPTestSettings.detailed(trust_remote_code=True, multi_node_only=True), # noqa: E501
125 # [FAST TESTS]
126 # Uses Llama
127 # "BAAI/AquilaChat-7B": PPTestSettings.fast(),
@@ -151,10 +163,8 @@ GENERATION_MODEL_SETTINGS = {
163 "facebook/opt-iml-max-1.3b": PPTestSettings.fast(),
164 "OrionStarAI/Orion-14B-Chat": PPTestSettings.fast(trust_remote_code=True),
165 "microsoft/phi-2": PPTestSettings.fast(),
154 - "microsoft/Phi-3-mini-4k-instruct": PPTestSettings.fast(),
166 "microsoft/Phi-3-small-8k-instruct": PPTestSettings.fast(trust_remote_code=True), # noqa: E501
156 - # FIXME: https://github.com/vllm-project/vllm/issues/8553
157 - # "microsoft/Phi-3.5-MoE-instruct": PPTestSettings.fast(trust_remote_code=True), # noqa: E501
167 + "microsoft/Phi-3.5-MoE-instruct": PPTestSettings.fast(trust_remote_code=True), # noqa: E501
168 "adept/persimmon-8b-chat": PPTestSettings.fast(),
169 "Qwen/Qwen-7B-Chat": PPTestSettings.fast(trust_remote_code=True),
170 "Qwen/Qwen2-beta-7B-Chat": PPTestSettings.fast(),
@@ -205,6 +215,7 @@ TEST_MODELS = [
215 # [LANGUAGE GENERATION]
216 "meta-llama/Meta-Llama-3-8B",
217 "ibm/PowerLM-3b",
218 + "microsoft/Phi-3-mini-4k-instruct",
219 # [LANGUAGE EMBEDDING]
220 "intfloat/e5-mistral-7b-instruct",
221 "BAAI/bge-multilingual-gemma2",
@@ -220,19 +231,21 @@ def _compare_tp(
231 parallel_setup: ParallelSetup,
232 distributed_backend: str,
233 task: TaskOption,
223 - trust_remote_code: bool,
224 - tokenizer_mode: Optional[str],
234 + test_options: PPTestOptions,
235 num_gpus_available: int,
236 *,
227 - method: Literal["generate", "encode"] = "encode",
237 + method: Literal["generate", "encode"],
238 ):
239 tp_size, pp_size, eager_mode, chunked_prefill = parallel_setup
240 + multi_node_only, trust_remote_code, tokenizer_mode = test_options
241
242 if num_gpus_available < tp_size * pp_size:
243 pytest.skip(f"Need at least {tp_size} x {pp_size} GPUs")
244 if VLLM_MULTI_NODE and distributed_backend == "mp":
245 pytest.skip("Skipping multi-node pipeline parallel test for "
246 "multiprocessing distributed backend")
247 + if multi_node_only and not VLLM_MULTI_NODE:
248 + pytest.skip("Not in multi-node setting")
249
250 common_args = [
251 # use half precision for speed and memory savings in CI environment
@@ -307,7 +320,7 @@ def _compare_tp(
320
321 @pytest.mark.parametrize(
322 ("model_name", "parallel_setup", "distributed_backend", "task",
310 - "trust_remote_code", "tokenizer_mode"),
323 + "test_options"),
324 [
325 params for model_name, settings in GENERATION_MODEL_SETTINGS.items()
326 for params in settings.iter_params(model_name)
@@ -320,23 +333,21 @@ def test_tp_language_generation(
333 parallel_setup: ParallelSetup,
334 distributed_backend: str,
335 task: TaskOption,
323 - trust_remote_code: bool,
324 - tokenizer_mode: Optional[str],
336 + test_options: PPTestOptions,
337 num_gpus_available,
338 ):
339 _compare_tp(model_name,
340 parallel_setup,
341 distributed_backend,
342 task,
331 - trust_remote_code,
332 - tokenizer_mode,
343 + test_options,
344 num_gpus_available,
345 method="generate")
346
347
348 @pytest.mark.parametrize(
349 ("model_name", "parallel_setup", "distributed_backend", "task",
339 - "trust_remote_code", "tokenizer_mode"),
350 + "test_options"),
351 [
352 params for model_name, settings in EMBEDDING_MODEL_SETTINGS.items()
353 for params in settings.iter_params(model_name)
@@ -349,23 +360,21 @@ def test_tp_language_embedding(
360 parallel_setup: ParallelSetup,
361 distributed_backend: str,
362 task: TaskOption,
352 - trust_remote_code: bool,
353 - tokenizer_mode: Optional[str],
363 + test_options: PPTestOptions,
364 num_gpus_available,
365 ):
366 _compare_tp(model_name,
367 parallel_setup,
368 distributed_backend,
369 task,
360 - trust_remote_code,
361 - tokenizer_mode,
370 + test_options,
371 num_gpus_available,
372 method="encode")
373
374
375 @pytest.mark.parametrize(
376 ("model_name", "parallel_setup", "distributed_backend", "task",
368 - "trust_remote_code", "tokenizer_mode"),
377 + "test_options"),
378 [
379 params for model_name, settings in MULTIMODAL_MODEL_SETTINGS.items()
380 for params in settings.iter_params(model_name)
@@ -378,15 +387,13 @@ def test_tp_multimodal_generation(
387 parallel_setup: ParallelSetup,
388 distributed_backend: str,
389 task: TaskOption,
381 - trust_remote_code: bool,
382 - tokenizer_mode: Optional[str],
390 + test_options: PPTestOptions,
391 num_gpus_available,
392 ):
393 _compare_tp(model_name,
394 parallel_setup,
395 distributed_backend,
396 task,
389 - trust_remote_code,
390 - tokenizer_mode,
397 + test_options,
398 num_gpus_available,
399 method="generate")
vllm/engine/arg_utils.py
+4
@@ -16,6 +16,8 @@ from vllm.config import (CacheConfig, ConfigFormat, DecodingConfig,
16 from vllm.executor.executor_base import ExecutorBase
17 from vllm.logger import init_logger
18 from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
19 +from vllm.transformers_utils.config import (
20 + maybe_register_config_serialize_by_value)
21 from vllm.transformers_utils.utils import check_gguf_file
22 from vllm.utils import FlexibleArgumentParser
23
@@ -924,6 +926,8 @@ class EngineArgs:
926 "supported for multimodal models and has been disabled.")
927 self.enable_prefix_caching = False
928
929 + maybe_register_config_serialize_by_value(self.trust_remote_code)
930 +
931 cache_config = CacheConfig(
932 # neuron needs block_size = max_model_len
933 block_size=self.block_size if self.device != "neuron" else
vllm/transformers_utils/config.py
+62
@@ -232,6 +232,68 @@ def get_config(
232 return config
233
234
235 +def maybe_register_config_serialize_by_value(trust_remote_code: bool) -> None:
236 + """Try to register HF model configuration class to serialize by value
237 +
238 + With trust_remote_code, the config class is typically an instance of a
239 + custom class imported from the HF modules cache. The class will not be
240 + importable in spawned workers by default (and won't exist at all on
241 + other nodes), which breaks serialization of the config.
242 +
243 + In this function we tell the cloudpickle serialization library to pass
244 + instances of these generated classes by value instead of by reference,
245 + i.e. the class definition is serialized along with its data so that the
246 + class module does not need to be importable on the receiving end. This
247 + registration only works if the modules cache has already been
248 + initialized.
249 +
250 +
251 + See: https://github.com/cloudpipe/cloudpickle?tab=readme-ov-file#overriding-pickles-serialization-mechanism-for-importable-constructs
252 + """
253 + if not trust_remote_code:
254 + return
255 +
256 + try:
257 + import transformers_modules
258 + except ImportError:
259 + logger.debug("Could not import transformers_modules used for remote"
260 + " code. If remote code is not needed remove"
261 + " `--trust-remote-code`.")
262 + return
263 +
264 + try:
265 + import cloudpickle
266 + cloudpickle.register_pickle_by_value(transformers_modules)
267 +
268 + # ray vendors its own version of cloudpickle
269 + from vllm.executor.ray_utils import ray
270 + if ray:
271 + ray.cloudpickle.register_pickle_by_value(transformers_modules)
272 +
273 + # multiprocessing uses pickle to serialize arguments when using spawn
274 + # Here we get pickle to use cloudpickle to serialize ModelConfig objects
275 + # that contain instances of the custom config class to avoid
276 + # serialization problems if the generated module (and model) has a `.`
277 + # in its name
278 + import multiprocessing
279 + import pickle
280 +
281 + from vllm.config import ModelConfig
282 +
283 + def _reduce_modelconfig(mc: ModelConfig):
284 + return (pickle.loads, (cloudpickle.dumps(mc), ))
285 +
286 + multiprocessing.reducer.register(ModelConfig, _reduce_modelconfig)
287 +
288 + except Exception as e:
289 + logger.warning(
290 + "Unable to register remote classes used by"
291 + " trust_remote_code with by-value serialization. This may"
292 + " lead to a later error. If remote code is not needed"
293 + " remove `--trust-remote-code`",
294 + exc_info=e)
295 +
296 +
297 def load_params_config(model, revision) -> PretrainedConfig:
298 # This function loads a params.json config which
299 # should be used when loading models in mistral format
vllm/utils.py
+2
@@ -968,6 +968,8 @@ def flatten_2d_lists(lists: List[List[T]]) -> List[T]:
968 return [item for sublist in lists for item in sublist]
969
970
971 +# TODO: This function can be removed if transformer_modules classes are
972 +# serialized by value when communicating between processes
973 def init_cached_hf_modules() -> None:
974 """
975 Lazy initialization of the Hugging Face modules.