@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 350f9e107f
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[CI/Build] Move `test_utils.py` to `tests/utils.py` (#4425)

Since #4335 was merged, I've noticed that the definition of ServerRunner in the tests is the same as in the test for OpenAI API. I have moved the class to the test utilities to avoid code duplication. (Although it only has been repeated twice so far, I will add another similar test suite in #4200 which would duplicate the code a third time) Also, I have moved the test utilities file (test_utils.py) to under the test directory (tests/utils.py), since none of its code is actually used in the main package. Note that I have added __init__.py to each test subpackage and updated the ray.init() call in the test utilities file in order to relative import tests/utils.py.

Cyrus Leung committed May 13, 2024 at 22:50 UTC 350f9e107f0c00e59be1b970f96395494ed68b48
34 files changed +138 -164
.buildkite/test-pipeline.yaml
+11 -13
@@ -24,28 +24,26 @@ steps:
24 command: pytest -v -s core
25
26 - label: Distributed Comm Ops Test
27 - command: pytest -v -s test_comm_ops.py
28 - working_dir: "/vllm-workspace/tests/distributed"
27 + command: pytest -v -s distributed/test_comm_ops.py
28 + working_dir: "/vllm-workspace/tests"
29 num_gpus: 2
30
31 - label: Distributed Tests
32 - working_dir: "/vllm-workspace/tests/distributed"
33 -
34 - num_gpus: 2 # only support 1 or 2 for now.
32 + working_dir: "/vllm-workspace/tests"
33 + num_gpus: 2
34 mirror_hardwares: [amd]
36 -
35 commands:
38 - - pytest -v -s test_pynccl_library.py
39 - - TEST_DIST_MODEL=facebook/opt-125m pytest -v -s test_basic_distributed_correctness.py
40 - - TEST_DIST_MODEL=meta-llama/Llama-2-7b-hf pytest -v -s test_basic_distributed_correctness.py
41 - - TEST_DIST_MODEL=facebook/opt-125m pytest -v -s test_chunked_prefill_distributed.py
42 - - TEST_DIST_MODEL=meta-llama/Llama-2-7b-hf pytest -v -s test_chunked_prefill_distributed.py
36 + - pytest -v -s distributed/test_pynccl_library.py
37 + - TEST_DIST_MODEL=facebook/opt-125m pytest -v -s distributed/test_basic_distributed_correctness.py
38 + - TEST_DIST_MODEL=meta-llama/Llama-2-7b-hf pytest -v -s distributed/test_basic_distributed_correctness.py
39 + - TEST_DIST_MODEL=facebook/opt-125m pytest -v -s distributed/test_chunked_prefill_distributed.py
40 + - TEST_DIST_MODEL=meta-llama/Llama-2-7b-hf pytest -v -s distributed/test_chunked_prefill_distributed.py
41
42 - label: Distributed Tests (Multiple Groups)
45 - working_dir: "/vllm-workspace/tests/distributed"
43 + working_dir: "/vllm-workspace/tests"
44 num_gpus: 4
45 commands:
48 - - pytest -v -s test_pynccl.py
46 + - pytest -v -s distributed/test_pynccl.py
47
48 - label: Engine Test
49 #mirror_hardwares: [amd]
tests/async_engine/__init__.py
tests/async_engine/test_openapi_server_ray.py
+3 -48
@@ -1,61 +1,16 @@
1 -# imports for guided decoding tests
2 -import os
3 -import subprocess
4 -import sys
5 -import time
6 -
1 import openai # use the official client for correctness check
2 import pytest
3 # using Ray for overall ease of process management, parallel requests,
4 # and debugging.
5 import ray
12 -import requests
6
14 -MAX_SERVER_START_WAIT_S = 600 # wait for server to start for 60 seconds
7 +from ..utils import ServerRunner
8 +
9 # any model with a chat template should work here
10 MODEL_NAME = "facebook/opt-125m"
11
12
19 -@ray.remote(num_gpus=1)
20 -class ServerRunner:
21 -
22 - def __init__(self, args):
23 - env = os.environ.copy()
24 - env["PYTHONUNBUFFERED"] = "1"
25 - self.proc = subprocess.Popen(
26 - ["python3", "-m", "vllm.entrypoints.openai.api_server"] + args,
27 - env=env,
28 - stdout=sys.stdout,
29 - stderr=sys.stderr,
30 - )
31 - self._wait_for_server()
32 -
33 - def ready(self):
34 - return True
35 -
36 - def _wait_for_server(self):
37 - # run health check
38 - start = time.time()
39 - while True:
40 - try:
41 - if requests.get(
42 - "http://localhost:8000/health").status_code == 200:
43 - break
44 - except Exception as err:
45 - if self.proc.poll() is not None:
46 - raise RuntimeError("Server exited unexpectedly.") from err
47 -
48 - time.sleep(0.5)
49 - if time.time() - start > MAX_SERVER_START_WAIT_S:
50 - raise RuntimeError(
51 - "Server failed to start in time.") from err
52 -
53 - def __del__(self):
54 - if hasattr(self, "proc"):
55 - self.proc.terminate()
56 -
57 -
58 -@pytest.fixture(scope="session")
13 +@pytest.fixture(scope="module")
14 def server():
15 ray.init()
16 server_runner = ServerRunner.remote([
tests/basic_correctness/__init__.py
tests/core/block/e2e/__init__.py
tests/core/block/e2e/conftest.py
+2 -1
@@ -1,9 +1,10 @@
1 import pytest
2
3 -from tests.conftest import cleanup
3 from vllm import LLM
4 from vllm.model_executor.utils import set_random_seed
5
6 +from ....conftest import cleanup
7 +
8
9 @pytest.fixture
10 def baseline_llm_generator(common_llm_kwargs, per_test_common_llm_kwargs,
tests/distributed/__init__.py
tests/distributed/test_basic_distributed_correctness.py
+4 -2
@@ -4,10 +4,12 @@ by one. The solution is to pass arguments (model name) by environment
4 variables.
5 Run:
6 ```sh
7 +cd $VLLM_PATH/tests
8 +
9 TEST_DIST_MODEL=facebook/opt-125m pytest \
8 - test_basic_distributed_correctness.py
10 + distributed/test_basic_distributed_correctness.py
11 TEST_DIST_MODEL=meta-llama/Llama-2-7b-hf \
10 - test_basic_distributed_correctness.py
12 + distributed/test_basic_distributed_correctness.py
13 ```
14 """
15 import os
tests/distributed/test_comm_ops.py
+3 -2
@@ -11,8 +11,9 @@ import torch
11 from vllm.distributed import (broadcast_tensor_dict,
12 tensor_model_parallel_all_gather,
13 tensor_model_parallel_all_reduce)
14 -from vllm.test_utils import (init_test_distributed_environment,
15 - multi_process_tensor_parallel)
14 +
15 +from ..utils import (init_test_distributed_environment,
16 + multi_process_tensor_parallel)
17
18
19 @ray.remote(num_gpus=1, max_calls=1)
tests/distributed/test_custom_all_reduce.py
+3 -2
@@ -10,8 +10,9 @@ from vllm.distributed.communication_op import ( # noqa
10 graph_capture, tensor_model_parallel_all_reduce)
11 from vllm.distributed.parallel_state import (get_tensor_model_parallel_group,
12 get_tp_ca_communicator)
13 -from vllm.test_utils import (init_test_distributed_environment,
14 - multi_process_tensor_parallel)
13 +
14 +from ..utils import (init_test_distributed_environment,
15 + multi_process_tensor_parallel)
16
17 random.seed(42)
18 test_sizes = [random.randint(1024, 2048 * 1024) for _ in range(8)]
tests/engine/__init__.py
tests/engine/output_processor/__init__.py
tests/engine/output_processor/test_multi_step.py
+2 -1
@@ -4,7 +4,6 @@ from unittest.mock import MagicMock
4 import pytest
5 from transformers import PreTrainedTokenizer
6
7 -from tests.core.utils import create_seq_group
7 from vllm.core.scheduler import Scheduler
8 from vllm.engine.output_processor.multi_step import MultiStepOutputProcessor
9 from vllm.engine.output_processor.stop_checker import StopChecker
@@ -14,6 +13,8 @@ from vllm.sequence import (CompletionSequenceGroupOutput, Logprob,
13 from vllm.transformers_utils.detokenizer import Detokenizer
14 from vllm.utils import Counter
15
16 +from ...core.utils import create_seq_group
17 +
18
19 @pytest.mark.parametrize("seq_output_len", [128])
20 @pytest.mark.parametrize("num_new_tokens", [1, 12])
tests/entrypoints/__init__.py
tests/entrypoints/test_openai_server.py
+2 -45
@@ -1,10 +1,6 @@
1 # imports for guided decoding tests
2 import json
3 -import os
3 import re
5 -import subprocess
6 -import sys
7 -import time
4
5 import jsonschema
6 import openai # use the official client for correctness check
@@ -12,7 +8,6 @@ import pytest
8 # using Ray for overall ease of process management, parallel requests,
9 # and debugging.
10 import ray
15 -import requests
11 import torch
12 # downloading lora to test lora requests
13 from huggingface_hub import snapshot_download
@@ -20,7 +15,8 @@ from openai import BadRequestError
15
16 from vllm.transformers_utils.tokenizer import get_tokenizer
17
23 -MAX_SERVER_START_WAIT_S = 600 # wait for server to start for 60 seconds
18 +from ..utils import ServerRunner
19 +
20 # any model with a chat template should work here
21 MODEL_NAME = "HuggingFaceH4/zephyr-7b-beta"
22 EMBEDDING_MODEL_NAME = "intfloat/e5-mistral-7b-instruct"
@@ -78,45 +74,6 @@ TEST_CHOICE = [
74 pytestmark = pytest.mark.asyncio
75
76
81 -@ray.remote(num_gpus=1)
82 -class ServerRunner:
83 -
84 - def __init__(self, args):
85 - env = os.environ.copy()
86 - env["PYTHONUNBUFFERED"] = "1"
87 - self.proc = subprocess.Popen(
88 - ["python3", "-m", "vllm.entrypoints.openai.api_server"] + args,
89 - env=env,
90 - stdout=sys.stdout,
91 - stderr=sys.stderr,
92 - )
93 - self._wait_for_server()
94 -
95 - def ready(self):
96 - return True
97 -
98 - def _wait_for_server(self):
99 - # run health check
100 - start = time.time()
101 - while True:
102 - try:
103 - if requests.get(
104 - "http://localhost:8000/health").status_code == 200:
105 - break
106 - except Exception as err:
107 - if self.proc.poll() is not None:
108 - raise RuntimeError("Server exited unexpectedly.") from err
109 -
110 - time.sleep(0.5)
111 - if time.time() - start > MAX_SERVER_START_WAIT_S:
112 - raise RuntimeError(
113 - "Server failed to start in time.") from err
114 -
115 - def __del__(self):
116 - if hasattr(self, "proc"):
117 - self.proc.terminate()
118 -
119 -
77 @pytest.fixture(scope="session")
78 def zephyr_lora_files():
79 return snapshot_download(repo_id=LORA_NAME)
tests/kernels/__init__.py
tests/kernels/test_activation.py
+2 -1
@@ -2,11 +2,12 @@ from typing import Type
2
3 import pytest
4 import torch
5 -from allclose_default import get_default_atol, get_default_rtol
5
6 from vllm.model_executor.layers.activation import (FastGELU, GeluAndMul,
7 NewGELU, SiluAndMul)
8
9 +from .allclose_default import get_default_atol, get_default_rtol
10 +
11 DTYPES = [torch.half, torch.bfloat16, torch.float]
12 NUM_TOKENS = [7, 83, 2048] # Arbitrary values for testing
13 D = [512, 4096, 5120, 13824] # Arbitrary values for testing
tests/kernels/test_attention.py
+2 -1
@@ -3,13 +3,14 @@ from typing import List, Optional, Tuple
3
4 import pytest
5 import torch
6 -from allclose_default import get_default_atol, get_default_rtol
6 from xformers import ops as xops
7 from xformers.ops.fmha.attn_bias import BlockDiagonalCausalMask
8
9 from vllm import _custom_ops as ops
10 from vllm.utils import get_max_shared_memory_bytes, is_hip
11
12 +from .allclose_default import get_default_atol, get_default_rtol
13 +
14 FLOAT32_BYTES = torch.finfo(torch.float).bits // 8
15 # This will change depending on the compute capability.
16 # - 512 as a buffer
tests/kernels/test_pos_encoding.py
+2 -1
@@ -3,10 +3,11 @@ from typing import List, Optional
3
4 import pytest
5 import torch
6 -from allclose_default import get_default_atol, get_default_rtol
6
7 from vllm.model_executor.layers.rotary_embedding import get_rope
8
9 +from .allclose_default import get_default_atol, get_default_rtol
10 +
11 IS_NEOX_STYLE = [True, False]
12 DTYPES = [torch.half, torch.bfloat16, torch.float]
13 HEAD_SIZES = [64, 80, 96, 112, 128, 256]
tests/metrics/__init__.py
tests/model_executor/__init__.py
tests/models/__init__.py
tests/models/test_gptq_marlin.py
+2 -1
@@ -13,9 +13,10 @@ import os
13 import pytest
14 import torch
15
16 -from tests.models.utils import check_logprobs_close
16 from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
17
18 +from .utils import check_logprobs_close
19 +
20 os.environ["TOKENIZERS_PARALLELISM"] = "true"
21
22 MAX_MODEL_LEN = 1024
tests/models/test_marlin.py
+2 -1
@@ -15,9 +15,10 @@ from dataclasses import dataclass
15 import pytest
16 import torch
17
18 -from tests.models.utils import check_logprobs_close
18 from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
19
20 +from .utils import check_logprobs_close
21 +
22 capability = torch.cuda.get_device_capability()
23 capability = capability[0] * 10 + capability[1]
24 marlin_not_supported = (capability <
tests/models/test_mistral.py
+1 -1
@@ -4,7 +4,7 @@ Run `pytest tests/models/test_mistral.py`.
4 """
5 import pytest
6
7 -from tests.models.utils import check_logprobs_close
7 +from .utils import check_logprobs_close
8
9 MODELS = [
10 "mistralai/Mistral-7B-Instruct-v0.1",
tests/prefix_caching/__init__.py
tests/quantization/__init__.py
tests/samplers/__init__.py
tests/samplers/test_logprobs.py
+2 -1
@@ -1,9 +1,10 @@
1 import pytest
2 import torch
3
4 -from tests.conftest import VllmRunner
4 from vllm import SamplingParams
5
6 +from ..conftest import VllmRunner
7 +
8 MODELS = ["facebook/opt-125m"]
9
10
tests/spec_decode/e2e/conftest.py
+2 -1
@@ -9,7 +9,6 @@ import torch
9 from pynvml import (nvmlDeviceGetHandleByIndex, nvmlDeviceGetMemoryInfo,
10 nvmlInit)
11
12 -from tests.conftest import cleanup
12 from vllm import LLM
13 from vllm.engine.arg_utils import AsyncEngineArgs
14 from vllm.engine.async_llm_engine import AsyncLLMEngine
@@ -21,6 +20,8 @@ from vllm.sequence import Logprob, MultiModalData
20 from vllm.usage.usage_lib import UsageContext
21 from vllm.utils import Counter, random_uuid
22
23 +from ...conftest import cleanup
24 +
25
26 class AsyncLLM:
27 """AsyncLLM
tests/tensorizer_loader/test_tensorizer.py
+2 -1
@@ -9,12 +9,13 @@ import pytest
9 import ray
10 import torch
11
12 -from tests.entrypoints.test_openai_server import ServerRunner
12 from vllm import SamplingParams
13 from vllm.model_executor.model_loader.tensorizer import (
14 EncryptionParams, TensorizerConfig, TensorSerializer,
15 is_vllm_serialized_tensorizer, load_with_tensorizer, open_stream)
16
17 +from ..utils import ServerRunner
18 +
19 prompts = [
20 "Hello, my name is",
21 "The president of the United States is",
tests/test_sequence.py
+2 -1
@@ -1,9 +1,10 @@
1 import pytest
2
3 -from tests.core.utils import create_dummy_prompt
3 from vllm.sequence import (CompletionSequenceGroupOutput, SamplerOutput,
4 SequenceData, SequenceOutput)
5
6 +from .core.utils import create_dummy_prompt
7 +
8
9 @pytest.fixture
10 def sample_outputs():
tests/utils.py new
+89
@@ -0,0 +1,89 @@
1 +import os
2 +import subprocess
3 +import sys
4 +import time
5 +
6 +import ray
7 +import requests
8 +
9 +from vllm.distributed import (ensure_model_parallel_initialized,
10 + init_distributed_environment)
11 +from vllm.utils import get_open_port
12 +
13 +# Path to root of repository so that utilities can be imported by ray workers
14 +VLLM_PATH = os.path.abspath(os.path.join(__file__, os.pardir, os.pardir))
15 +
16 +
17 +@ray.remote(num_gpus=1)
18 +class ServerRunner:
19 + MAX_SERVER_START_WAIT_S = 600 # wait for server to start for 60 seconds
20 +
21 + def __init__(self, args):
22 + env = os.environ.copy()
23 + env["PYTHONUNBUFFERED"] = "1"
24 + self.proc = subprocess.Popen(
25 + ["python3", "-m", "vllm.entrypoints.openai.api_server"] + args,
26 + env=env,
27 + stdout=sys.stdout,
28 + stderr=sys.stderr,
29 + )
30 + self._wait_for_server()
31 +
32 + def ready(self):
33 + return True
34 +
35 + def _wait_for_server(self):
36 + # run health check
37 + start = time.time()
38 + while True:
39 + try:
40 + if requests.get(
41 + "http://localhost:8000/health").status_code == 200:
42 + break
43 + except Exception as err:
44 + if self.proc.poll() is not None:
45 + raise RuntimeError("Server exited unexpectedly.") from err
46 +
47 + time.sleep(0.5)
48 + if time.time() - start > self.MAX_SERVER_START_WAIT_S:
49 + raise RuntimeError(
50 + "Server failed to start in time.") from err
51 +
52 + def __del__(self):
53 + if hasattr(self, "proc"):
54 + self.proc.terminate()
55 +
56 +
57 +def init_test_distributed_environment(
58 + tp_size: int,
59 + pp_size: int,
60 + rank: int,
61 + distributed_init_port: str,
62 + local_rank: int = -1,
63 +) -> None:
64 + distributed_init_method = f"tcp://localhost:{distributed_init_port}"
65 + init_distributed_environment(
66 + world_size=pp_size * tp_size,
67 + rank=rank,
68 + distributed_init_method=distributed_init_method,
69 + local_rank=local_rank)
70 + ensure_model_parallel_initialized(tp_size, pp_size)
71 +
72 +
73 +def multi_process_tensor_parallel(
74 + tp_size: int,
75 + pp_size: int,
76 + test_target,
77 +) -> None:
78 + # Using ray helps debugging the error when it failed
79 + # as compared to multiprocessing.
80 + ray.init(runtime_env={"working_dir": VLLM_PATH})
81 +
82 + distributed_init_port = get_open_port()
83 + refs = []
84 + for rank in range(tp_size * pp_size):
85 + refs.append(
86 + test_target.remote(tp_size, pp_size, rank, distributed_init_port))
87 + ray.get(refs)
88 +
89 + ray.shutdown()
vllm/test_utils.py deleted
-40
@@ -1,40 +0,0 @@
1 -import ray
2 -
3 -from vllm.distributed import (ensure_model_parallel_initialized,
4 - init_distributed_environment)
5 -from vllm.utils import get_open_port
6 -
7 -
8 -def init_test_distributed_environment(
9 - tp_size: int,
10 - pp_size: int,
11 - rank: int,
12 - distributed_init_port: str,
13 - local_rank: int = -1,
14 -) -> None:
15 - distributed_init_method = f"tcp://localhost:{distributed_init_port}"
16 - init_distributed_environment(
17 - world_size=pp_size * tp_size,
18 - rank=rank,
19 - distributed_init_method=distributed_init_method,
20 - local_rank=local_rank)
21 - ensure_model_parallel_initialized(tp_size, pp_size)
22 -
23 -
24 -def multi_process_tensor_parallel(
25 - tp_size: int,
26 - pp_size: int,
27 - test_target,
28 -) -> None:
29 - # Using ray helps debugging the error when it failed
30 - # as compared to multiprocessing.
31 - ray.init()
32 -
33 - distributed_init_port = get_open_port()
34 - refs = []
35 - for rank in range(tp_size * pp_size):
36 - refs.append(
37 - test_target.remote(tp_size, pp_size, rank, distributed_init_port))
38 - ray.get(refs)
39 -
40 - ray.shutdown()