A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[ci][distributed] try to fix pp test (#7054)
youkaichao committed
Aug 1, 2024 at 22:03 UTC
252357793dd1fe9d30c34e68e4b8b2143a4c5138
4 files changed
+45
-3
tests/distributed/test_pipeline_parallel.py
+3
-1
@@ -9,7 +9,7 @@ import os
9
10
import pytest
11
12
-from ..utils import compare_two_settings
12
+from ..utils import compare_two_settings, fork_new_process_for_each_test
13
14
VLLM_MULTI_NODE = os.getenv("VLLM_MULTI_NODE", "0") == "1"
15
@@ -28,6 +28,7 @@ VLLM_MULTI_NODE = os.getenv("VLLM_MULTI_NODE", "0") == "1"
28
(1, 4, 0, 1, "meta-llama/Meta-Llama-3-8B", "mp"),
29
(1, 4, 1, 0, "meta-llama/Meta-Llama-3-8B", "mp"),
30
])
31
+@fork_new_process_for_each_test
32
def test_compare_tp(TP_SIZE, PP_SIZE, EAGER_MODE, CHUNKED_PREFILL, MODEL_NAME,
33
DIST_BACKEND):
34
if VLLM_MULTI_NODE and DIST_BACKEND == "mp":
@@ -77,6 +78,7 @@ def test_compare_tp(TP_SIZE, PP_SIZE, EAGER_MODE, CHUNKED_PREFILL, MODEL_NAME,
78
"FLASH_ATTN",
79
"FLASHINFER",
80
])
81
+@fork_new_process_for_each_test
82
def test_pp_cudagraph(PP_SIZE, MODEL_NAME, ATTN_BACKEND):
83
cudagraph_args = [
84
# use half precision for speed and memory savings in CI environment
tests/utils.py
+39
@@ -1,4 +1,6 @@
1
+import functools
2
import os
3
+import signal
4
import subprocess
5
import sys
6
import time
@@ -336,3 +338,40 @@ def wait_for_gpu_memory_to_clear(devices: List[int],
338
f'{dur_s=:.02f} ({threshold_bytes/2**30=})')
339
340
time.sleep(5)
341
+
342
+
343
+def fork_new_process_for_each_test(f):
344
+
345
+ @functools.wraps(f)
346
+ def wrapper(*args, **kwargs):
347
+ # Make the process the leader of its own process group
348
+ # to avoid sending SIGTERM to the parent process
349
+ os.setpgrp()
350
+ from _pytest.outcomes import Skipped
351
+ pid = os.fork()
352
+ if pid == 0:
353
+ try:
354
+ f(*args, **kwargs)
355
+ except Skipped as e:
356
+ # convert Skipped to exit code 0
357
+ print(str(e))
358
+ os._exit(0)
359
+ except Exception:
360
+ import traceback
361
+ traceback.print_exc()
362
+ os._exit(1)
363
+ else:
364
+ os._exit(0)
365
+ else:
366
+ pgid = os.getpgid(pid)
367
+ _pid, _exitcode = os.waitpid(pid, 0)
368
+ # ignore SIGTERM signal itself
369
+ old_singla_handler = signal.signal(signal.SIGTERM, signal.SIG_IGN)
370
+ # kill all child processes
371
+ os.killpg(pgid, signal.SIGTERM)
372
+ # restore the signal handler
373
+ signal.signal(signal.SIGTERM, old_singla_handler)
374
+ assert _exitcode == 0, (f"function {f} failed when called with"
375
+ f" args {args} and kwargs {kwargs}")
376
+
377
+ return wrapper
vllm/transformers_utils/tokenizer_group/ray_tokenizer_group.py
+1
-1
@@ -3,7 +3,7 @@ import os
3
from typing import List, Optional
4
5
try:
6
- from ray.exceptions import ActorDiedError
6
+ from ray.exceptions import ActorDiedError # type: ignore
7
except ImportError:
8
# For older versions of Ray
9
from ray.exceptions import RayActorError as ActorDiedError # type: ignore
vllm/utils.py
+2
-1
@@ -928,7 +928,8 @@ def error_on_invalid_device_count_status():
928
with contextlib.suppress(Exception):
929
# future pytorch will fix the issue, device_count will not be cached
930
# at that time, `.cache_info().currsize` will error out
931
- cache_entries = torch.cuda.device_count.cache_info().currsize
931
+ cache_entries = torch.cuda.device_count.cache_info( # type: ignore
932
+ ).currsize
933
if cache_entries != 0:
934
# the function is already called, and the result is cached
935
remembered = torch.cuda.device_count()