@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / cafb8e06c5
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 00:30 UTC

[CI/BUILD] enable intel queue for longer CPU tests (#4113)

Yuan committed Jun 4, 2024 at 01:39 UTC cafb8e06c5ffa359ac7fa4b53795e6eaa1a200c7
11 files changed +136 -88
.buildkite/run-cpu-test.sh
+12 -2
@@ -10,5 +10,15 @@ remove_docker_container() { docker rm -f cpu-test || true; }
10 trap remove_docker_container EXIT
11 remove_docker_container
12
13 -# Run the image and launch offline inference
14 -docker run --network host --env VLLM_CPU_KVCACHE_SPACE=1 --name cpu-test cpu-test python3 vllm/examples/offline_inference.py
13 +# Run the image
14 +docker run -itd -v ~/.cache/huggingface:/root/.cache/huggingface --network host -e HF_TOKEN --env VLLM_CPU_KVCACHE_SPACE=4 --name cpu-test cpu-test
15 +
16 +# offline inference
17 +docker exec cpu-test bash -c "python3 examples/offline_inference.py"
18 +
19 +# Run basic model test
20 +docker exec cpu-test bash -c "cd tests;
21 + pip install pytest Pillow protobuf
22 + bash ../.buildkite/download-images.sh
23 + cd ../
24 + pytest -v -s tests/models --ignore=tests/models/test_llava.py --ignore=tests/models/test_embedding.py --ignore=tests/models/test_registry.py"
.buildkite/test-template.j2
+2
@@ -40,6 +40,8 @@ steps:
40
41 - label: "Intel Test"
42 depends_on: ~
43 + agents:
44 + queue: intel
45 command: bash .buildkite/run-cpu-test.sh
46
47 {% for step in steps %}
Dockerfile.cpu
+5 -1
@@ -1,6 +1,6 @@
1 # This vLLM Dockerfile is used to construct image that can build and run vLLM on x86 CPU platform.
2
3 -FROM ubuntu:22.04
3 +FROM ubuntu:22.04 AS cpu-test-1
4
5 RUN apt-get update -y \
6 && apt-get install -y git wget vim numactl gcc-12 g++-12 python3 python3-pip \
@@ -9,6 +9,8 @@ RUN apt-get update -y \
9 RUN pip install --upgrade pip \
10 && pip install wheel packaging ninja setuptools>=49.4.0 numpy
11
12 +FROM cpu-test-1 AS build
13 +
14 COPY ./ /workspace/vllm
15
16 WORKDIR /workspace/vllm
@@ -19,4 +21,6 @@ RUN VLLM_TARGET_DEVICE=cpu python3 setup.py install
21
22 WORKDIR /workspace/
23
24 +RUN ln -s /workspace/vllm/tests && ln -s /workspace/vllm/examples && ln -s /workspace/vllm/benchmarks
25 +
26 CMD ["/bin/bash"]
csrc/cpu/pos_encoding.cpp
+51 -50
@@ -21,73 +21,74 @@ void rotary_embedding_impl(
21 constexpr int VEC_ELEM_NUM = scalar_vec_t::get_elem_num();
22
23 const int embed_dim = rot_dim / 2;
24 - TORCH_CHECK(embed_dim % VEC_ELEM_NUM == 0);
24 + bool flag = (embed_dim % VEC_ELEM_NUM == 0);
25 + const int loop_upper = flag ? embed_dim : embed_dim - VEC_ELEM_NUM;
26
26 -#pragma omp parallel for
27 - for (int token_idx = 0; token_idx < num_tokens; ++token_idx) {
28 - int64_t pos = positions[token_idx];
29 - const scalar_t* cache_ptr = cos_sin_cache + pos * rot_dim;
27 + auto compute_loop = [&](const int64_t token_head, const scalar_t* cache_ptr,
28 + scalar_t* qk) {
29 + int j = 0;
30 + for (; j < loop_upper; j += VEC_ELEM_NUM) {
31 + const int rot_offset = j;
32 + const int x_index = rot_offset;
33 + const int y_index = embed_dim + rot_offset;
34
31 - for (int i = 0; i < num_heads; ++i) {
32 - const int head_idx = i;
33 - const int64_t token_head =
34 - token_idx * query_stride + head_idx * head_size;
35 - for (int j = 0; j < embed_dim; j += VEC_ELEM_NUM) {
36 - const int rot_offset = j;
37 - const int x_index = rot_offset;
38 - const int y_index = embed_dim + rot_offset;
35 + const int64_t out_x = token_head + x_index;
36 + const int64_t out_y = token_head + y_index;
37
40 - const int64_t out_x = token_head + x_index;
41 - const int64_t out_y = token_head + y_index;
38 + const scalar_vec_t cos(cache_ptr + x_index);
39 + const scalar_vec_t sin(cache_ptr + y_index);
40
43 - const scalar_vec_t cos(cache_ptr + x_index);
44 - const scalar_vec_t sin(cache_ptr + y_index);
41 + const scalar_vec_t q_x(qk + out_x);
42 + const scalar_vec_t q_y(qk + out_y);
43
46 - const scalar_vec_t q_x(query + out_x);
47 - const scalar_vec_t q_y(query + out_y);
44 + vec_op::FP32Vec8 fp32_cos(cos);
45 + vec_op::FP32Vec8 fp32_sin(sin);
46
49 - vec_op::FP32Vec8 fp32_cos(cos);
50 - vec_op::FP32Vec8 fp32_sin(sin);
47 + vec_op::FP32Vec8 fp32_q_x(q_x);
48 + vec_op::FP32Vec8 fp32_q_y(q_y);
49
52 - vec_op::FP32Vec8 fp32_q_x(q_x);
53 - vec_op::FP32Vec8 fp32_q_y(q_y);
50 + auto out1 = fp32_q_x * fp32_cos - fp32_q_y * fp32_sin;
51 + scalar_vec_t(out1).save(qk + out_x);
52
55 - auto out1 = fp32_q_x * fp32_cos - fp32_q_y * fp32_sin;
56 - scalar_vec_t(out1).save(query + out_x);
57 -
58 - auto out2 = fp32_q_y * fp32_cos + fp32_q_x * fp32_sin;
59 - scalar_vec_t(out2).save(query + out_y);
60 - }
53 + auto out2 = fp32_q_y * fp32_cos + fp32_q_x * fp32_sin;
54 + scalar_vec_t(out2).save(qk + out_y);
55 }
62 -
63 - for (int i = 0; i < num_kv_heads; ++i) {
64 - const int head_idx = i;
65 - const int64_t token_head = token_idx * key_stride + head_idx * head_size;
66 - for (int j = 0; j < embed_dim; j += VEC_ELEM_NUM) {
67 - const int rot_offset = j;
68 - const int x_index = rot_offset;
69 - const int y_index = embed_dim + rot_offset;
56 + if (!flag) {
57 + for (; j < embed_dim; ++j) {
58 + const int x_index = j;
59 + const int y_index = embed_dim + j;
60
61 const int64_t out_x = token_head + x_index;
62 const int64_t out_y = token_head + y_index;
63
74 - const scalar_vec_t cos(cache_ptr + x_index);
75 - const scalar_vec_t sin(cache_ptr + y_index);
64 + const float fp32_cos = cache_ptr[x_index];
65 + const float fp32_sin = cache_ptr[y_index];
66
77 - const scalar_vec_t k_x(key + out_x);
78 - const scalar_vec_t k_y(key + out_y);
67 + const float fp32_q_x = qk[out_x];
68 + const float fp32_q_y = qk[out_y];
69
80 - vec_op::FP32Vec8 fp32_cos(cos);
81 - vec_op::FP32Vec8 fp32_sin(sin);
70 + qk[out_x] = fp32_q_x * fp32_cos - fp32_q_y * fp32_sin;
71 + qk[out_y] = fp32_q_y * fp32_cos + fp32_q_x * fp32_sin;
72 + }
73 + }
74 + };
75
83 - vec_op::FP32Vec8 fp32_k_x(k_x);
84 - vec_op::FP32Vec8 fp32_k_y(k_y);
76 +#pragma omp parallel for
77 + for (int token_idx = 0; token_idx < num_tokens; ++token_idx) {
78 + int64_t pos = positions[token_idx];
79 + const scalar_t* cache_ptr = cos_sin_cache + pos * rot_dim;
80
86 - auto out1 = fp32_k_x * fp32_cos - fp32_k_y * fp32_sin;
87 - scalar_vec_t(out1).save(key + out_x);
88 - auto out2 = fp32_k_y * fp32_cos + fp32_k_x * fp32_sin;
89 - scalar_vec_t(out2).save(key + out_y);
90 - }
81 + for (int i = 0; i < num_heads; ++i) {
82 + const int head_idx = i;
83 + const int64_t token_head =
84 + token_idx * query_stride + head_idx * head_size;
85 + compute_loop(token_head, cache_ptr, query);
86 + }
87 +
88 + for (int i = 0; i < num_kv_heads; ++i) {
89 + const int head_idx = i;
90 + const int64_t token_head = token_idx * key_stride + head_idx * head_size;
91 + compute_loop(token_head, cache_ptr, key);
92 }
93 }
94 }
tests/conftest.py
+23 -13
@@ -18,6 +18,7 @@ from vllm.logger import init_logger
18 from vllm.multimodal import MultiModalData
19 from vllm.multimodal.image import ImageFeatureData, ImagePixelData
20 from vllm.sequence import SampleLogprobs
21 +from vllm.utils import is_cpu
22
23 logger = init_logger(__name__)
24
@@ -58,7 +59,8 @@ def cleanup():
59 with contextlib.suppress(AssertionError):
60 torch.distributed.destroy_process_group()
61 gc.collect()
61 - torch.cuda.empty_cache()
62 + if not is_cpu():
63 + torch.cuda.empty_cache()
64
65
66 @pytest.fixture()
@@ -151,6 +153,12 @@ _EMBEDDING_MODELS = [
153
154 class HfRunner:
155
156 + def wrap_device(self, input: any):
157 + if not is_cpu():
158 + return input.to("cuda")
159 + else:
160 + return input.to("cpu")
161 +
162 def __init__(
163 self,
164 model_name: str,
@@ -164,16 +172,18 @@ class HfRunner:
172 if model_name in _EMBEDDING_MODELS:
173 # Lazy init required for AMD CI
174 from sentence_transformers import SentenceTransformer
167 - self.model = SentenceTransformer(
168 - model_name,
169 - device="cpu",
170 - ).to(dtype=torch_dtype).cuda()
175 + self.model = self.wrap_device(
176 + SentenceTransformer(
177 + model_name,
178 + device="cpu",
179 + ).to(dtype=torch_dtype))
180 else:
172 - self.model = AutoModelForCausalLM.from_pretrained(
173 - model_name,
174 - torch_dtype=torch_dtype,
175 - trust_remote_code=True,
176 - ).cuda()
181 + self.model = self.wrap_device(
182 + AutoModelForCausalLM.from_pretrained(
183 + model_name,
184 + torch_dtype=torch_dtype,
185 + trust_remote_code=True,
186 + ))
187
188 self.tokenizer = AutoTokenizer.from_pretrained(
189 model_name,
@@ -214,7 +224,7 @@ class HfRunner:
224 inputs = self.processor(**processor_kwargs)
225
226 output_ids = self.model.generate(
217 - **inputs.to("cuda"),
227 + **self.wrap_device(inputs),
228 use_cache=True,
229 **kwargs,
230 )
@@ -271,7 +281,7 @@ class HfRunner:
281 for prompt in prompts:
282 input_ids = self.tokenizer(prompt, return_tensors="pt").input_ids
283 output = self.model.generate(
274 - input_ids.cuda(),
284 + self.wrap_device(input_ids),
285 use_cache=True,
286 do_sample=False,
287 max_new_tokens=max_tokens,
@@ -306,7 +316,7 @@ class HfRunner:
316 for prompt in prompts:
317 input_ids = self.tokenizer(prompt, return_tensors="pt").input_ids
318 output = self.model.generate(
309 - input_ids.cuda(),
319 + self.wrap_device(input_ids),
320 use_cache=True,
321 do_sample=False,
322 max_new_tokens=max_tokens,
tests/models/test_aqlm.py
+7 -4
@@ -8,10 +8,13 @@ import torch
8
9 from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
10
11 -capability = torch.cuda.get_device_capability()
12 -capability = capability[0] * 10 + capability[1]
13 -aqlm_not_supported = (capability <
14 - QUANTIZATION_METHODS["aqlm"].get_min_capability())
11 +aqlm_not_supported = True
12 +
13 +if torch.cuda.is_available():
14 + capability = torch.cuda.get_device_capability()
15 + capability = capability[0] * 10 + capability[1]
16 + aqlm_not_supported = (capability <
17 + QUANTIZATION_METHODS["aqlm"].get_min_capability())
18
19 # In this test we hardcode prompts and generations for the model so we don't
20 # need to require the AQLM package as a dependency
tests/models/test_big_models.py
+8 -2
@@ -5,6 +5,7 @@ This tests bigger models and use half precision.
5 Run `pytest tests/models/test_big_models.py`.
6 """
7 import pytest
8 +import torch
9
10 MODELS = [
11 "meta-llama/Llama-2-7b-hf",
@@ -16,9 +17,14 @@ MODELS = [
17 # "Qwen/Qwen1.5-0.5B" # Broken,
18 ]
19
20 +#TODO: remove this after CPU float16 support ready
21 +target_dtype = "float"
22 +if torch.cuda.is_available():
23 + target_dtype = "half"
24 +
25
26 @pytest.mark.parametrize("model", MODELS)
21 -@pytest.mark.parametrize("dtype", ["half"])
27 +@pytest.mark.parametrize("dtype", [target_dtype])
28 @pytest.mark.parametrize("max_tokens", [32])
29 def test_models(
30 hf_runner,
@@ -46,7 +52,7 @@ def test_models(
52
53
54 @pytest.mark.parametrize("model", MODELS)
49 -@pytest.mark.parametrize("dtype", ["half"])
55 +@pytest.mark.parametrize("dtype", [target_dtype])
56 def test_model_print(
57 vllm_runner,
58 model: str,
tests/models/test_fp8.py
+7 -4
@@ -67,10 +67,13 @@ EXPECTED_STRS_MAP = {
67 },
68 }
69
70 -capability = torch.cuda.get_device_capability()
71 -capability = capability[0] * 10 + capability[1]
72 -fp8_not_supported = (capability <
73 - QUANTIZATION_METHODS["fp8"].get_min_capability())
70 +fp8_not_supported = True
71 +
72 +if torch.cuda.is_available():
73 + capability = torch.cuda.get_device_capability()
74 + capability = capability[0] * 10 + capability[1]
75 + fp8_not_supported = (capability <
76 + QUANTIZATION_METHODS["fp8"].get_min_capability())
77
78
79 @pytest.mark.skipif(fp8_not_supported,
tests/models/test_gptq_marlin.py
+7 -4
@@ -22,10 +22,13 @@ os.environ["TOKENIZERS_PARALLELISM"] = "true"
22
23 MAX_MODEL_LEN = 1024
24
25 -capability = torch.cuda.get_device_capability()
26 -capability = capability[0] * 10 + capability[1]
27 -gptq_marlin_not_supported = (
28 - capability < QUANTIZATION_METHODS["gptq_marlin"].get_min_capability())
25 +gptq_marlin_not_supported = True
26 +
27 +if torch.cuda.is_available():
28 + capability = torch.cuda.get_device_capability()
29 + capability = capability[0] * 10 + capability[1]
30 + gptq_marlin_not_supported = (
31 + capability < QUANTIZATION_METHODS["gptq_marlin"].get_min_capability())
32
33 MODELS = [
34 # act_order==False, group_size=channelwise
tests/models/test_gptq_marlin_24.py
+7 -4
@@ -14,10 +14,13 @@ import torch
14 from tests.models.utils import check_logprobs_close
15 from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
16
17 -capability = torch.cuda.get_device_capability()
18 -capability = capability[0] * 10 + capability[1]
19 -marlin_not_supported = (capability <
20 - QUANTIZATION_METHODS["marlin"].get_min_capability())
17 +marlin_not_supported = True
18 +
19 +if torch.cuda.is_available():
20 + capability = torch.cuda.get_device_capability()
21 + capability = capability[0] * 10 + capability[1]
22 + marlin_not_supported = (
23 + capability < QUANTIZATION_METHODS["marlin"].get_min_capability())
24
25
26 @dataclass
tests/models/test_marlin.py
+7 -4
@@ -19,10 +19,13 @@ from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
19
20 from .utils import check_logprobs_close
21
22 -capability = torch.cuda.get_device_capability()
23 -capability = capability[0] * 10 + capability[1]
24 -marlin_not_supported = (capability <
25 - QUANTIZATION_METHODS["marlin"].get_min_capability())
22 +marlin_not_supported = True
23 +
24 +if torch.cuda.is_available():
25 + capability = torch.cuda.get_device_capability()
26 + capability = capability[0] * 10 + capability[1]
27 + marlin_not_supported = (
28 + capability < QUANTIZATION_METHODS["marlin"].get_min_capability())
29
30
31 @dataclass