A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 01:00 UTC
Fix inadvertently silenced PP tests for `mp`, add DeepSeek V2/V3 model family to PP tests (#20831)
Signed-off-by: Seiji Eicher <seiji@anyscale.com>
Seiji Eicher committed
Jul 16, 2025 at 00:14 UTC
d0dc4cfca48c2734da18ec42d6bba1346cbfc400
1 file changed
+17
-7
tests/distributed/test_pipeline_parallel.py
+17
-7
@@ -14,8 +14,9 @@ from typing import Literal, NamedTuple, Optional
14
15
import pytest
16
17
-from vllm.config import TaskOption
17
+from vllm.config import _FLOAT16_NOT_SUPPORTED_MODELS, TaskOption
18
from vllm.logger import init_logger
19
+from vllm.transformers_utils.config import get_config
20
21
from ..models.registry import HF_EXAMPLE_MODELS
22
from ..utils import compare_two_settings, create_new_process_for_each_test
@@ -158,7 +159,7 @@ TEXT_GENERATION_MODELS = {
159
"databricks/dbrx-instruct": PPTestSettings.fast(load_format="dummy"),
160
"Deci/DeciLM-7B-instruct": PPTestSettings.fast(),
161
"deepseek-ai/deepseek-llm-7b-chat": PPTestSettings.fast(),
161
- "deepseek-ai/DeepSeek-V2-Lite-Chat": PPTestSettings.fast(),
162
+ "deepseek-ai/DeepSeek-V2-Lite-Chat": PPTestSettings.fast(tp_base=2),
163
"LGAI-EXAONE/EXAONE-3.0-7.8B-Instruct": PPTestSettings.fast(),
164
"tiiuae/falcon-7b": PPTestSettings.fast(),
165
"google/gemma-1.1-2b-it": PPTestSettings.fast(),
@@ -210,9 +211,11 @@ TEXT_GENERATION_MODELS = {
211
212
EMBEDDING_MODELS = { # type: ignore[var-annotated]
213
# [Text-only]
213
- "intfloat/e5-mistral-7b-instruct": PPTestSettings.fast(),
214
- "BAAI/bge-multilingual-gemma2": PPTestSettings.fast(),
215
- "Qwen/Qwen2.5-Math-RM-72B": PPTestSettings.fast(load_format="dummy"),
214
+ "intfloat/e5-mistral-7b-instruct": PPTestSettings.fast(task="embed"),
215
+ "BAAI/bge-multilingual-gemma2": PPTestSettings.fast(task="embed"),
216
+ "Qwen/Qwen2.5-Math-RM-72B": PPTestSettings.fast(
217
+ load_format="dummy", task="embed"
218
+ ),
219
}
220
221
MULTIMODAL_MODELS = {
@@ -248,6 +251,7 @@ TEST_MODELS = [
251
"meta-llama/Llama-3.2-1B-Instruct",
252
"ArthurZ/Ilama-3.2-1B",
253
"ibm/PowerLM-3b",
254
+ "deepseek-ai/DeepSeek-V2-Lite-Chat",
255
# [LANGUAGE EMBEDDING]
256
"intfloat/e5-mistral-7b-instruct",
257
"BAAI/bge-multilingual-gemma2",
@@ -287,6 +291,11 @@ def _compare_tp(
291
trust_remote_code = model_info.trust_remote_code
292
tokenizer_mode = model_info.tokenizer_mode
293
hf_overrides = model_info.hf_overrides
294
+ hf_config = get_config(model_id, trust_remote_code)
295
+
296
+ dtype = "float16"
297
+ if hf_config.model_type in _FLOAT16_NOT_SUPPORTED_MODELS:
298
+ dtype = "bfloat16"
299
300
if load_format == "dummy":
301
# Avoid OOM
@@ -316,7 +325,7 @@ def _compare_tp(
325
common_args = [
326
# use half precision for speed and memory savings in CI environment
327
"--dtype",
319
- "float16",
328
+ dtype,
329
"--max-model-len",
330
"2048",
331
"--max-num-seqs",
@@ -338,6 +347,7 @@ def _compare_tp(
347
common_args.extend(["--hf-overrides", json.dumps(hf_overrides)])
348
349
specific_case = tp_size == 2 and pp_size == 2 and chunked_prefill
350
+ testing_ray_compiled_graph = False
351
if distributed_backend == "ray" and (vllm_major_version == "1"
352
or specific_case):
353
# For V1, test Ray Compiled Graph for all the tests
@@ -351,6 +361,7 @@ def _compare_tp(
361
# Temporary. Currently when zeromq + SPMD is used, it does not properly
362
# terminate because of a Ray Compiled Graph issue.
363
common_args.append("--disable-frontend-multiprocessing")
364
+ testing_ray_compiled_graph = True
365
elif distributed_backend == "mp":
366
# Both V0/V1 of multiprocessing executor support PP
367
pp_env = {
@@ -394,7 +405,6 @@ def _compare_tp(
405
tp_env,
406
method=method)
407
except Exception:
397
- testing_ray_compiled_graph = pp_env is not None
408
if testing_ray_compiled_graph and vllm_major_version == "0":
409
# Ray Compiled Graph tests are flaky for V0,
410
# so we don't want to fail the test