@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 334d64d1e8
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 05:45 UTC

[ci] add vllm_test_utils (#10659)

Signed-off-by: youkaichao <youkaichao@gmail.com>

youkaichao committed Nov 26, 2024 at 00:20 UTC 334d64d1e816cc7c9fa2f67e22d24638e63c8e15
14 files changed +113 -61
Dockerfile
+4
@@ -191,6 +191,10 @@ ADD . /vllm-workspace/
191 RUN --mount=type=cache,target=/root/.cache/pip \
192 python3 -m pip install -r requirements-dev.txt
193
194 +# install development dependencies (for testing)
195 +RUN --mount=type=cache,target=/root/.cache/pip \
196 + python3 -m pip install -e tests/vllm_test_utils
197 +
198 # enable fast downloads from hf (for testing)
199 RUN --mount=type=cache,target=/root/.cache/pip \
200 python3 -m pip install hf_transfer
Dockerfile.cpu
+4
@@ -62,4 +62,8 @@ WORKDIR /workspace/
62
63 RUN ln -s /workspace/vllm/tests && ln -s /workspace/vllm/examples && ln -s /workspace/vllm/benchmarks
64
65 +# install development dependencies (for testing)
66 +RUN --mount=type=cache,target=/root/.cache/pip \
67 + pip install -e tests/vllm_test_utils
68 +
69 ENTRYPOINT ["python3", "-m", "vllm.entrypoints.openai.api_server"]
Dockerfile.hpu
+3
@@ -11,6 +11,9 @@ ENV PT_HPU_ENABLE_LAZY_COLLECTIVES=true
11
12 RUN VLLM_TARGET_DEVICE=hpu python3 setup.py install
13
14 +# install development dependencies (for testing)
15 +RUN python3 -m pip install -e tests/vllm_test_utils
16 +
17 WORKDIR /workspace/
18
19 RUN ln -s /workspace/vllm/tests && ln -s /workspace/vllm/examples && ln -s /workspace/vllm/benchmarks
Dockerfile.neuron
+3
@@ -38,4 +38,7 @@ ENV VLLM_TARGET_DEVICE neuron
38 RUN --mount=type=bind,source=.git,target=.git \
39 pip install --no-build-isolation -v -e .
40
41 +# install development dependencies (for testing)
42 +RUN python3 -m pip install -e tests/vllm_test_utils
43 +
44 CMD ["/bin/bash"]
Dockerfile.openvino
+3
@@ -22,4 +22,7 @@ RUN PIP_EXTRA_INDEX_URL="https://download.pytorch.org/whl/cpu" VLLM_TARGET_DEVIC
22 COPY examples/ /workspace/examples
23 COPY benchmarks/ /workspace/benchmarks
24
25 +# install development dependencies (for testing)
26 +RUN python3 -m pip install -e tests/vllm_test_utils
27 +
28 CMD ["/bin/bash"]
Dockerfile.ppc64le
+3
@@ -29,6 +29,9 @@ RUN --mount=type=cache,target=/root/.cache/pip \
29 RUN --mount=type=bind,source=.git,target=.git \
30 VLLM_TARGET_DEVICE=cpu python3 setup.py install
31
32 +# install development dependencies (for testing)
33 +RUN python3 -m pip install -e tests/vllm_test_utils
34 +
35 WORKDIR /workspace/
36
37 RUN ln -s /workspace/vllm/tests && ln -s /workspace/vllm/examples && ln -s /workspace/vllm/benchmarks
Dockerfile.rocm
+3
@@ -168,4 +168,7 @@ RUN --mount=type=cache,target=/root/.cache/pip \
168 if ls libs/*.whl; then \
169 python3 -m pip install libs/*.whl; fi
170
171 +# install development dependencies (for testing)
172 +RUN python3 -m pip install -e tests/vllm_test_utils
173 +
174 CMD ["/bin/bash"]
Dockerfile.tpu
+3
@@ -22,4 +22,7 @@ RUN --mount=type=cache,target=/root/.cache/pip \
22 -r requirements-tpu.txt
23 RUN python3 setup.py develop
24
25 +# install development dependencies (for testing)
26 +RUN python3 -m pip install -e tests/vllm_test_utils
27 +
28 CMD ["/bin/bash"]
Dockerfile.xpu
+2 -1
@@ -64,5 +64,6 @@ RUN --mount=type=cache,target=/root/.cache/pip \
64
65 ENV VLLM_USAGE_SOURCE production-docker-image \
66 TRITON_XPU_PROFILE 1
67 -
67 +# install development dependencies (for testing)
68 +RUN python3 -m pip install -e tests/vllm_test_utils
69 ENTRYPOINT ["python3", "-m", "vllm.entrypoints.openai.api_server"]
tests/entrypoints/llm/test_lazy_outlines.py
+16 -7
@@ -1,12 +1,12 @@
1 import sys
2
3 +from vllm_test_utils import blame
4 +
5 from vllm import LLM, SamplingParams
6 from vllm.distributed import cleanup_dist_env_and_memory
7
8
7 -def test_lazy_outlines(sample_regex):
8 - """If users don't use guided decoding, outlines should not be imported.
9 - """
9 +def run_normal():
10 prompts = [
11 "Hello, my name is",
12 "The president of the United States is",
@@ -25,13 +25,12 @@ def test_lazy_outlines(sample_regex):
25 generated_text = output.outputs[0].text
26 print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
27
28 - # make sure outlines is not imported
29 - assert 'outlines' not in sys.modules
30 -
28 # Destroy the LLM object and free up the GPU memory.
29 del llm
30 cleanup_dist_env_and_memory()
31
32 +
33 +def run_lmfe(sample_regex):
34 # Create an LLM with guided decoding enabled.
35 llm = LLM(model="facebook/opt-125m",
36 enforce_eager=True,
@@ -51,5 +50,15 @@ def test_lazy_outlines(sample_regex):
50 generated_text = output.outputs[0].text
51 print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
52
53 +
54 +def test_lazy_outlines(sample_regex):
55 + """If users don't use guided decoding, outlines should not be imported.
56 + """
57 # make sure outlines is not imported
55 - assert 'outlines' not in sys.modules
58 + module_name = "outlines"
59 + with blame(lambda: module_name in sys.modules) as result:
60 + run_normal()
61 + run_lmfe(sample_regex)
62 + assert not result.found, (
63 + f"Module {module_name} is already imported, the"
64 + f" first import location is:\n{result.trace_stack}")
tests/test_lazy_torch_compile.py
+1 -53
@@ -1,61 +1,9 @@
1 # Description: Test the lazy import module
2 # The utility function cannot be placed in `vllm.utils`
3 # this needs to be a standalone script
4 -
5 -import contextlib
6 -import dataclasses
4 import sys
8 -import traceback
9 -from typing import Callable, Generator
10 -
11 -
12 -@dataclasses.dataclass
13 -class BlameResult:
14 - found: bool = False
15 - trace_stack: str = ""
16 -
17 -
18 -@contextlib.contextmanager
19 -def blame(func: Callable) -> Generator[BlameResult, None, None]:
20 - """
21 - Trace the function calls to find the first function that satisfies the
22 - condition. The trace stack will be stored in the result.
23 -
24 - Usage:
25 -
26 - ```python
27 - with blame(lambda: some_condition()) as result:
28 - # do something
29 -
30 - if result.found:
31 - print(result.trace_stack)
32 - """
33 - result = BlameResult()
34 -
35 - def _trace_calls(frame, event, arg=None):
36 - nonlocal result
37 - if event in ['call', 'return']:
38 - # for every function call or return
39 - try:
40 - # Temporarily disable the trace function
41 - sys.settrace(None)
42 - # check condition here
43 - if not result.found and func():
44 - result.found = True
45 - result.trace_stack = "".join(traceback.format_stack())
46 - # Re-enable the trace function
47 - sys.settrace(_trace_calls)
48 - except NameError:
49 - # modules are deleted during shutdown
50 - pass
51 - return _trace_calls
52 -
53 - sys.settrace(_trace_calls)
54 -
55 - yield result
56 -
57 - sys.settrace(None)
5
6 +from vllm_test_utils import blame
7
8 module_name = "torch._inductor.async_compile"
9
tests/vllm_test_utils/setup.py new
+7
@@ -0,0 +1,7 @@
1 +from setuptools import setup
2 +
3 +setup(
4 + name='vllm_test_utils',
5 + version='0.1',
6 + packages=['vllm_test_utils'],
7 +)
tests/vllm_test_utils/vllm_test_utils/__init__.py new
+8
@@ -0,0 +1,8 @@
1 +"""
2 +vllm_utils is a package for vLLM testing utilities.
3 +It does not import any vLLM modules.
4 +"""
5 +
6 +from .blame import BlameResult, blame
7 +
8 +__all__ = ["blame", "BlameResult"]
tests/vllm_test_utils/vllm_test_utils/blame.py new
+53
@@ -0,0 +1,53 @@
1 +import contextlib
2 +import dataclasses
3 +import sys
4 +import traceback
5 +from typing import Callable, Generator
6 +
7 +
8 +@dataclasses.dataclass
9 +class BlameResult:
10 + found: bool = False
11 + trace_stack: str = ""
12 +
13 +
14 +@contextlib.contextmanager
15 +def blame(func: Callable) -> Generator[BlameResult, None, None]:
16 + """
17 + Trace the function calls to find the first function that satisfies the
18 + condition. The trace stack will be stored in the result.
19 +
20 + Usage:
21 +
22 + ```python
23 + with blame(lambda: some_condition()) as result:
24 + # do something
25 +
26 + if result.found:
27 + print(result.trace_stack)
28 + """
29 + result = BlameResult()
30 +
31 + def _trace_calls(frame, event, arg=None):
32 + nonlocal result
33 + if event in ['call', 'return']:
34 + # for every function call or return
35 + try:
36 + # Temporarily disable the trace function
37 + sys.settrace(None)
38 + # check condition here
39 + if not result.found and func():
40 + result.found = True
41 + result.trace_stack = "".join(traceback.format_stack())
42 + # Re-enable the trace function
43 + sys.settrace(_trace_calls)
44 + except NameError:
45 + # modules are deleted during shutdown
46 + pass
47 + return _trace_calls
48 +
49 + sys.settrace(_trace_calls)
50 +
51 + yield result
52 +
53 + sys.settrace(None)