@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 252357793d
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[ci][distributed] try to fix pp test (#7054)

youkaichao committed Aug 1, 2024 at 22:03 UTC 252357793dd1fe9d30c34e68e4b8b2143a4c5138
4 files changed +45 -3
tests/distributed/test_pipeline_parallel.py
+3 -1
@@ -9,7 +9,7 @@ import os
9
10 import pytest
11
12 -from ..utils import compare_two_settings
12 +from ..utils import compare_two_settings, fork_new_process_for_each_test
13
14 VLLM_MULTI_NODE = os.getenv("VLLM_MULTI_NODE", "0") == "1"
15
@@ -28,6 +28,7 @@ VLLM_MULTI_NODE = os.getenv("VLLM_MULTI_NODE", "0") == "1"
28 (1, 4, 0, 1, "meta-llama/Meta-Llama-3-8B", "mp"),
29 (1, 4, 1, 0, "meta-llama/Meta-Llama-3-8B", "mp"),
30 ])
31 +@fork_new_process_for_each_test
32 def test_compare_tp(TP_SIZE, PP_SIZE, EAGER_MODE, CHUNKED_PREFILL, MODEL_NAME,
33 DIST_BACKEND):
34 if VLLM_MULTI_NODE and DIST_BACKEND == "mp":
@@ -77,6 +78,7 @@ def test_compare_tp(TP_SIZE, PP_SIZE, EAGER_MODE, CHUNKED_PREFILL, MODEL_NAME,
78 "FLASH_ATTN",
79 "FLASHINFER",
80 ])
81 +@fork_new_process_for_each_test
82 def test_pp_cudagraph(PP_SIZE, MODEL_NAME, ATTN_BACKEND):
83 cudagraph_args = [
84 # use half precision for speed and memory savings in CI environment
tests/utils.py
+39
@@ -1,4 +1,6 @@
1 +import functools
2 import os
3 +import signal
4 import subprocess
5 import sys
6 import time
@@ -336,3 +338,40 @@ def wait_for_gpu_memory_to_clear(devices: List[int],
338 f'{dur_s=:.02f} ({threshold_bytes/2**30=})')
339
340 time.sleep(5)
341 +
342 +
343 +def fork_new_process_for_each_test(f):
344 +
345 + @functools.wraps(f)
346 + def wrapper(*args, **kwargs):
347 + # Make the process the leader of its own process group
348 + # to avoid sending SIGTERM to the parent process
349 + os.setpgrp()
350 + from _pytest.outcomes import Skipped
351 + pid = os.fork()
352 + if pid == 0:
353 + try:
354 + f(*args, **kwargs)
355 + except Skipped as e:
356 + # convert Skipped to exit code 0
357 + print(str(e))
358 + os._exit(0)
359 + except Exception:
360 + import traceback
361 + traceback.print_exc()
362 + os._exit(1)
363 + else:
364 + os._exit(0)
365 + else:
366 + pgid = os.getpgid(pid)
367 + _pid, _exitcode = os.waitpid(pid, 0)
368 + # ignore SIGTERM signal itself
369 + old_singla_handler = signal.signal(signal.SIGTERM, signal.SIG_IGN)
370 + # kill all child processes
371 + os.killpg(pgid, signal.SIGTERM)
372 + # restore the signal handler
373 + signal.signal(signal.SIGTERM, old_singla_handler)
374 + assert _exitcode == 0, (f"function {f} failed when called with"
375 + f" args {args} and kwargs {kwargs}")
376 +
377 + return wrapper
vllm/transformers_utils/tokenizer_group/ray_tokenizer_group.py
+1 -1
@@ -3,7 +3,7 @@ import os
3 from typing import List, Optional
4
5 try:
6 - from ray.exceptions import ActorDiedError
6 + from ray.exceptions import ActorDiedError # type: ignore
7 except ImportError:
8 # For older versions of Ray
9 from ray.exceptions import RayActorError as ActorDiedError # type: ignore
vllm/utils.py
+2 -1
@@ -928,7 +928,8 @@ def error_on_invalid_device_count_status():
928 with contextlib.suppress(Exception):
929 # future pytorch will fix the issue, device_count will not be cached
930 # at that time, `.cache_info().currsize` will error out
931 - cache_entries = torch.cuda.device_count.cache_info().currsize
931 + cache_entries = torch.cuda.device_count.cache_info( # type: ignore
932 + ).currsize
933 if cache_entries != 0:
934 # the function is already called, and the result is cached
935 remembered = torch.cuda.device_count()