A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 05:45 UTC
[ci] add vllm_test_utils (#10659)
Signed-off-by: youkaichao <youkaichao@gmail.com>
youkaichao committed
Nov 26, 2024 at 00:20 UTC
334d64d1e816cc7c9fa2f67e22d24638e63c8e15
14 files changed
+113
-61
Dockerfile
+4
@@ -191,6 +191,10 @@ ADD . /vllm-workspace/
191
RUN --mount=type=cache,target=/root/.cache/pip \
192
python3 -m pip install -r requirements-dev.txt
193
194
+# install development dependencies (for testing)
195
+RUN --mount=type=cache,target=/root/.cache/pip \
196
+ python3 -m pip install -e tests/vllm_test_utils
197
+
198
# enable fast downloads from hf (for testing)
199
RUN --mount=type=cache,target=/root/.cache/pip \
200
python3 -m pip install hf_transfer
Dockerfile.cpu
+4
@@ -62,4 +62,8 @@ WORKDIR /workspace/
62
63
RUN ln -s /workspace/vllm/tests && ln -s /workspace/vllm/examples && ln -s /workspace/vllm/benchmarks
64
65
+# install development dependencies (for testing)
66
+RUN --mount=type=cache,target=/root/.cache/pip \
67
+ pip install -e tests/vllm_test_utils
68
+
69
ENTRYPOINT ["python3", "-m", "vllm.entrypoints.openai.api_server"]
Dockerfile.hpu
+3
@@ -11,6 +11,9 @@ ENV PT_HPU_ENABLE_LAZY_COLLECTIVES=true
11
12
RUN VLLM_TARGET_DEVICE=hpu python3 setup.py install
13
14
+# install development dependencies (for testing)
15
+RUN python3 -m pip install -e tests/vllm_test_utils
16
+
17
WORKDIR /workspace/
18
19
RUN ln -s /workspace/vllm/tests && ln -s /workspace/vllm/examples && ln -s /workspace/vllm/benchmarks
Dockerfile.neuron
+3
@@ -38,4 +38,7 @@ ENV VLLM_TARGET_DEVICE neuron
38
RUN --mount=type=bind,source=.git,target=.git \
39
pip install --no-build-isolation -v -e .
40
41
+# install development dependencies (for testing)
42
+RUN python3 -m pip install -e tests/vllm_test_utils
43
+
44
CMD ["/bin/bash"]
Dockerfile.openvino
+3
@@ -22,4 +22,7 @@ RUN PIP_EXTRA_INDEX_URL="https://download.pytorch.org/whl/cpu" VLLM_TARGET_DEVIC
22
COPY examples/ /workspace/examples
23
COPY benchmarks/ /workspace/benchmarks
24
25
+# install development dependencies (for testing)
26
+RUN python3 -m pip install -e tests/vllm_test_utils
27
+
28
CMD ["/bin/bash"]
Dockerfile.ppc64le
+3
@@ -29,6 +29,9 @@ RUN --mount=type=cache,target=/root/.cache/pip \
29
RUN --mount=type=bind,source=.git,target=.git \
30
VLLM_TARGET_DEVICE=cpu python3 setup.py install
31
32
+# install development dependencies (for testing)
33
+RUN python3 -m pip install -e tests/vllm_test_utils
34
+
35
WORKDIR /workspace/
36
37
RUN ln -s /workspace/vllm/tests && ln -s /workspace/vllm/examples && ln -s /workspace/vllm/benchmarks
Dockerfile.rocm
+3
@@ -168,4 +168,7 @@ RUN --mount=type=cache,target=/root/.cache/pip \
168
if ls libs/*.whl; then \
169
python3 -m pip install libs/*.whl; fi
170
171
+# install development dependencies (for testing)
172
+RUN python3 -m pip install -e tests/vllm_test_utils
173
+
174
CMD ["/bin/bash"]
Dockerfile.tpu
+3
@@ -22,4 +22,7 @@ RUN --mount=type=cache,target=/root/.cache/pip \
22
-r requirements-tpu.txt
23
RUN python3 setup.py develop
24
25
+# install development dependencies (for testing)
26
+RUN python3 -m pip install -e tests/vllm_test_utils
27
+
28
CMD ["/bin/bash"]
Dockerfile.xpu
+2
-1
@@ -64,5 +64,6 @@ RUN --mount=type=cache,target=/root/.cache/pip \
64
65
ENV VLLM_USAGE_SOURCE production-docker-image \
66
TRITON_XPU_PROFILE 1
67
-
67
+# install development dependencies (for testing)
68
+RUN python3 -m pip install -e tests/vllm_test_utils
69
ENTRYPOINT ["python3", "-m", "vllm.entrypoints.openai.api_server"]
tests/entrypoints/llm/test_lazy_outlines.py
+16
-7
@@ -1,12 +1,12 @@
1
import sys
2
3
+from vllm_test_utils import blame
4
+
5
from vllm import LLM, SamplingParams
6
from vllm.distributed import cleanup_dist_env_and_memory
7
8
7
-def test_lazy_outlines(sample_regex):
8
- """If users don't use guided decoding, outlines should not be imported.
9
- """
9
+def run_normal():
10
prompts = [
11
"Hello, my name is",
12
"The president of the United States is",
@@ -25,13 +25,12 @@ def test_lazy_outlines(sample_regex):
25
generated_text = output.outputs[0].text
26
print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
27
28
- # make sure outlines is not imported
29
- assert 'outlines' not in sys.modules
30
-
28
# Destroy the LLM object and free up the GPU memory.
29
del llm
30
cleanup_dist_env_and_memory()
31
32
+
33
+def run_lmfe(sample_regex):
34
# Create an LLM with guided decoding enabled.
35
llm = LLM(model="facebook/opt-125m",
36
enforce_eager=True,
@@ -51,5 +50,15 @@ def test_lazy_outlines(sample_regex):
50
generated_text = output.outputs[0].text
51
print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
52
53
+
54
+def test_lazy_outlines(sample_regex):
55
+ """If users don't use guided decoding, outlines should not be imported.
56
+ """
57
# make sure outlines is not imported
55
- assert 'outlines' not in sys.modules
58
+ module_name = "outlines"
59
+ with blame(lambda: module_name in sys.modules) as result:
60
+ run_normal()
61
+ run_lmfe(sample_regex)
62
+ assert not result.found, (
63
+ f"Module {module_name} is already imported, the"
64
+ f" first import location is:\n{result.trace_stack}")
tests/test_lazy_torch_compile.py
+1
-53
@@ -1,61 +1,9 @@
1
# Description: Test the lazy import module
2
# The utility function cannot be placed in `vllm.utils`
3
# this needs to be a standalone script
4
-
5
-import contextlib
6
-import dataclasses
4
import sys
8
-import traceback
9
-from typing import Callable, Generator
10
-
11
-
12
-@dataclasses.dataclass
13
-class BlameResult:
14
- found: bool = False
15
- trace_stack: str = ""
16
-
17
-
18
-@contextlib.contextmanager
19
-def blame(func: Callable) -> Generator[BlameResult, None, None]:
20
- """
21
- Trace the function calls to find the first function that satisfies the
22
- condition. The trace stack will be stored in the result.
23
-
24
- Usage:
25
-
26
- ```python
27
- with blame(lambda: some_condition()) as result:
28
- # do something
29
-
30
- if result.found:
31
- print(result.trace_stack)
32
- """
33
- result = BlameResult()
34
-
35
- def _trace_calls(frame, event, arg=None):
36
- nonlocal result
37
- if event in ['call', 'return']:
38
- # for every function call or return
39
- try:
40
- # Temporarily disable the trace function
41
- sys.settrace(None)
42
- # check condition here
43
- if not result.found and func():
44
- result.found = True
45
- result.trace_stack = "".join(traceback.format_stack())
46
- # Re-enable the trace function
47
- sys.settrace(_trace_calls)
48
- except NameError:
49
- # modules are deleted during shutdown
50
- pass
51
- return _trace_calls
52
-
53
- sys.settrace(_trace_calls)
54
-
55
- yield result
56
-
57
- sys.settrace(None)
5
6
+from vllm_test_utils import blame
7
8
module_name = "torch._inductor.async_compile"
9
tests/vllm_test_utils/setup.py
new
+7
@@ -0,0 +1,7 @@
1
+from setuptools import setup
2
+
3
+setup(
4
+ name='vllm_test_utils',
5
+ version='0.1',
6
+ packages=['vllm_test_utils'],
7
+)
tests/vllm_test_utils/vllm_test_utils/__init__.py
new
+8
@@ -0,0 +1,8 @@
1
+"""
2
+vllm_utils is a package for vLLM testing utilities.
3
+It does not import any vLLM modules.
4
+"""
5
+
6
+from .blame import BlameResult, blame
7
+
8
+__all__ = ["blame", "BlameResult"]
tests/vllm_test_utils/vllm_test_utils/blame.py
new
+53
@@ -0,0 +1,53 @@
1
+import contextlib
2
+import dataclasses
3
+import sys
4
+import traceback
5
+from typing import Callable, Generator
6
+
7
+
8
+@dataclasses.dataclass
9
+class BlameResult:
10
+ found: bool = False
11
+ trace_stack: str = ""
12
+
13
+
14
+@contextlib.contextmanager
15
+def blame(func: Callable) -> Generator[BlameResult, None, None]:
16
+ """
17
+ Trace the function calls to find the first function that satisfies the
18
+ condition. The trace stack will be stored in the result.
19
+
20
+ Usage:
21
+
22
+ ```python
23
+ with blame(lambda: some_condition()) as result:
24
+ # do something
25
+
26
+ if result.found:
27
+ print(result.trace_stack)
28
+ """
29
+ result = BlameResult()
30
+
31
+ def _trace_calls(frame, event, arg=None):
32
+ nonlocal result
33
+ if event in ['call', 'return']:
34
+ # for every function call or return
35
+ try:
36
+ # Temporarily disable the trace function
37
+ sys.settrace(None)
38
+ # check condition here
39
+ if not result.found and func():
40
+ result.found = True
41
+ result.trace_stack = "".join(traceback.format_stack())
42
+ # Re-enable the trace function
43
+ sys.settrace(_trace_calls)
44
+ except NameError:
45
+ # modules are deleted during shutdown
46
+ pass
47
+ return _trace_calls
48
+
49
+ sys.settrace(_trace_calls)
50
+
51
+ yield result
52
+
53
+ sys.settrace(None)