A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 00:30 UTC
[CI/BUILD] enable intel queue for longer CPU tests (#4113)
Yuan committed
Jun 4, 2024 at 01:39 UTC
cafb8e06c5ffa359ac7fa4b53795e6eaa1a200c7
11 files changed
+136
-88
.buildkite/run-cpu-test.sh
+12
-2
@@ -10,5 +10,15 @@ remove_docker_container() { docker rm -f cpu-test || true; }
10
trap remove_docker_container EXIT
11
remove_docker_container
12
13
-# Run the image and launch offline inference
14
-docker run --network host --env VLLM_CPU_KVCACHE_SPACE=1 --name cpu-test cpu-test python3 vllm/examples/offline_inference.py
13
+# Run the image
14
+docker run -itd -v ~/.cache/huggingface:/root/.cache/huggingface --network host -e HF_TOKEN --env VLLM_CPU_KVCACHE_SPACE=4 --name cpu-test cpu-test
15
+
16
+# offline inference
17
+docker exec cpu-test bash -c "python3 examples/offline_inference.py"
18
+
19
+# Run basic model test
20
+docker exec cpu-test bash -c "cd tests;
21
+ pip install pytest Pillow protobuf
22
+ bash ../.buildkite/download-images.sh
23
+ cd ../
24
+ pytest -v -s tests/models --ignore=tests/models/test_llava.py --ignore=tests/models/test_embedding.py --ignore=tests/models/test_registry.py"
.buildkite/test-template.j2
+2
@@ -40,6 +40,8 @@ steps:
40
41
- label: "Intel Test"
42
depends_on: ~
43
+ agents:
44
+ queue: intel
45
command: bash .buildkite/run-cpu-test.sh
46
47
{% for step in steps %}
Dockerfile.cpu
+5
-1
@@ -1,6 +1,6 @@
1
# This vLLM Dockerfile is used to construct image that can build and run vLLM on x86 CPU platform.
2
3
-FROM ubuntu:22.04
3
+FROM ubuntu:22.04 AS cpu-test-1
4
5
RUN apt-get update -y \
6
&& apt-get install -y git wget vim numactl gcc-12 g++-12 python3 python3-pip \
@@ -9,6 +9,8 @@ RUN apt-get update -y \
9
RUN pip install --upgrade pip \
10
&& pip install wheel packaging ninja setuptools>=49.4.0 numpy
11
12
+FROM cpu-test-1 AS build
13
+
14
COPY ./ /workspace/vllm
15
16
WORKDIR /workspace/vllm
@@ -19,4 +21,6 @@ RUN VLLM_TARGET_DEVICE=cpu python3 setup.py install
21
22
WORKDIR /workspace/
23
24
+RUN ln -s /workspace/vllm/tests && ln -s /workspace/vllm/examples && ln -s /workspace/vllm/benchmarks
25
+
26
CMD ["/bin/bash"]
csrc/cpu/pos_encoding.cpp
+51
-50
@@ -21,73 +21,74 @@ void rotary_embedding_impl(
21
constexpr int VEC_ELEM_NUM = scalar_vec_t::get_elem_num();
22
23
const int embed_dim = rot_dim / 2;
24
- TORCH_CHECK(embed_dim % VEC_ELEM_NUM == 0);
24
+ bool flag = (embed_dim % VEC_ELEM_NUM == 0);
25
+ const int loop_upper = flag ? embed_dim : embed_dim - VEC_ELEM_NUM;
26
26
-#pragma omp parallel for
27
- for (int token_idx = 0; token_idx < num_tokens; ++token_idx) {
28
- int64_t pos = positions[token_idx];
29
- const scalar_t* cache_ptr = cos_sin_cache + pos * rot_dim;
27
+ auto compute_loop = [&](const int64_t token_head, const scalar_t* cache_ptr,
28
+ scalar_t* qk) {
29
+ int j = 0;
30
+ for (; j < loop_upper; j += VEC_ELEM_NUM) {
31
+ const int rot_offset = j;
32
+ const int x_index = rot_offset;
33
+ const int y_index = embed_dim + rot_offset;
34
31
- for (int i = 0; i < num_heads; ++i) {
32
- const int head_idx = i;
33
- const int64_t token_head =
34
- token_idx * query_stride + head_idx * head_size;
35
- for (int j = 0; j < embed_dim; j += VEC_ELEM_NUM) {
36
- const int rot_offset = j;
37
- const int x_index = rot_offset;
38
- const int y_index = embed_dim + rot_offset;
35
+ const int64_t out_x = token_head + x_index;
36
+ const int64_t out_y = token_head + y_index;
37
40
- const int64_t out_x = token_head + x_index;
41
- const int64_t out_y = token_head + y_index;
38
+ const scalar_vec_t cos(cache_ptr + x_index);
39
+ const scalar_vec_t sin(cache_ptr + y_index);
40
43
- const scalar_vec_t cos(cache_ptr + x_index);
44
- const scalar_vec_t sin(cache_ptr + y_index);
41
+ const scalar_vec_t q_x(qk + out_x);
42
+ const scalar_vec_t q_y(qk + out_y);
43
46
- const scalar_vec_t q_x(query + out_x);
47
- const scalar_vec_t q_y(query + out_y);
44
+ vec_op::FP32Vec8 fp32_cos(cos);
45
+ vec_op::FP32Vec8 fp32_sin(sin);
46
49
- vec_op::FP32Vec8 fp32_cos(cos);
50
- vec_op::FP32Vec8 fp32_sin(sin);
47
+ vec_op::FP32Vec8 fp32_q_x(q_x);
48
+ vec_op::FP32Vec8 fp32_q_y(q_y);
49
52
- vec_op::FP32Vec8 fp32_q_x(q_x);
53
- vec_op::FP32Vec8 fp32_q_y(q_y);
50
+ auto out1 = fp32_q_x * fp32_cos - fp32_q_y * fp32_sin;
51
+ scalar_vec_t(out1).save(qk + out_x);
52
55
- auto out1 = fp32_q_x * fp32_cos - fp32_q_y * fp32_sin;
56
- scalar_vec_t(out1).save(query + out_x);
57
-
58
- auto out2 = fp32_q_y * fp32_cos + fp32_q_x * fp32_sin;
59
- scalar_vec_t(out2).save(query + out_y);
60
- }
53
+ auto out2 = fp32_q_y * fp32_cos + fp32_q_x * fp32_sin;
54
+ scalar_vec_t(out2).save(qk + out_y);
55
}
62
-
63
- for (int i = 0; i < num_kv_heads; ++i) {
64
- const int head_idx = i;
65
- const int64_t token_head = token_idx * key_stride + head_idx * head_size;
66
- for (int j = 0; j < embed_dim; j += VEC_ELEM_NUM) {
67
- const int rot_offset = j;
68
- const int x_index = rot_offset;
69
- const int y_index = embed_dim + rot_offset;
56
+ if (!flag) {
57
+ for (; j < embed_dim; ++j) {
58
+ const int x_index = j;
59
+ const int y_index = embed_dim + j;
60
61
const int64_t out_x = token_head + x_index;
62
const int64_t out_y = token_head + y_index;
63
74
- const scalar_vec_t cos(cache_ptr + x_index);
75
- const scalar_vec_t sin(cache_ptr + y_index);
64
+ const float fp32_cos = cache_ptr[x_index];
65
+ const float fp32_sin = cache_ptr[y_index];
66
77
- const scalar_vec_t k_x(key + out_x);
78
- const scalar_vec_t k_y(key + out_y);
67
+ const float fp32_q_x = qk[out_x];
68
+ const float fp32_q_y = qk[out_y];
69
80
- vec_op::FP32Vec8 fp32_cos(cos);
81
- vec_op::FP32Vec8 fp32_sin(sin);
70
+ qk[out_x] = fp32_q_x * fp32_cos - fp32_q_y * fp32_sin;
71
+ qk[out_y] = fp32_q_y * fp32_cos + fp32_q_x * fp32_sin;
72
+ }
73
+ }
74
+ };
75
83
- vec_op::FP32Vec8 fp32_k_x(k_x);
84
- vec_op::FP32Vec8 fp32_k_y(k_y);
76
+#pragma omp parallel for
77
+ for (int token_idx = 0; token_idx < num_tokens; ++token_idx) {
78
+ int64_t pos = positions[token_idx];
79
+ const scalar_t* cache_ptr = cos_sin_cache + pos * rot_dim;
80
86
- auto out1 = fp32_k_x * fp32_cos - fp32_k_y * fp32_sin;
87
- scalar_vec_t(out1).save(key + out_x);
88
- auto out2 = fp32_k_y * fp32_cos + fp32_k_x * fp32_sin;
89
- scalar_vec_t(out2).save(key + out_y);
90
- }
81
+ for (int i = 0; i < num_heads; ++i) {
82
+ const int head_idx = i;
83
+ const int64_t token_head =
84
+ token_idx * query_stride + head_idx * head_size;
85
+ compute_loop(token_head, cache_ptr, query);
86
+ }
87
+
88
+ for (int i = 0; i < num_kv_heads; ++i) {
89
+ const int head_idx = i;
90
+ const int64_t token_head = token_idx * key_stride + head_idx * head_size;
91
+ compute_loop(token_head, cache_ptr, key);
92
}
93
}
94
}
tests/conftest.py
+23
-13
@@ -18,6 +18,7 @@ from vllm.logger import init_logger
18
from vllm.multimodal import MultiModalData
19
from vllm.multimodal.image import ImageFeatureData, ImagePixelData
20
from vllm.sequence import SampleLogprobs
21
+from vllm.utils import is_cpu
22
23
logger = init_logger(__name__)
24
@@ -58,7 +59,8 @@ def cleanup():
59
with contextlib.suppress(AssertionError):
60
torch.distributed.destroy_process_group()
61
gc.collect()
61
- torch.cuda.empty_cache()
62
+ if not is_cpu():
63
+ torch.cuda.empty_cache()
64
65
66
@pytest.fixture()
@@ -151,6 +153,12 @@ _EMBEDDING_MODELS = [
153
154
class HfRunner:
155
156
+ def wrap_device(self, input: any):
157
+ if not is_cpu():
158
+ return input.to("cuda")
159
+ else:
160
+ return input.to("cpu")
161
+
162
def __init__(
163
self,
164
model_name: str,
@@ -164,16 +172,18 @@ class HfRunner:
172
if model_name in _EMBEDDING_MODELS:
173
# Lazy init required for AMD CI
174
from sentence_transformers import SentenceTransformer
167
- self.model = SentenceTransformer(
168
- model_name,
169
- device="cpu",
170
- ).to(dtype=torch_dtype).cuda()
175
+ self.model = self.wrap_device(
176
+ SentenceTransformer(
177
+ model_name,
178
+ device="cpu",
179
+ ).to(dtype=torch_dtype))
180
else:
172
- self.model = AutoModelForCausalLM.from_pretrained(
173
- model_name,
174
- torch_dtype=torch_dtype,
175
- trust_remote_code=True,
176
- ).cuda()
181
+ self.model = self.wrap_device(
182
+ AutoModelForCausalLM.from_pretrained(
183
+ model_name,
184
+ torch_dtype=torch_dtype,
185
+ trust_remote_code=True,
186
+ ))
187
188
self.tokenizer = AutoTokenizer.from_pretrained(
189
model_name,
@@ -214,7 +224,7 @@ class HfRunner:
224
inputs = self.processor(**processor_kwargs)
225
226
output_ids = self.model.generate(
217
- **inputs.to("cuda"),
227
+ **self.wrap_device(inputs),
228
use_cache=True,
229
**kwargs,
230
)
@@ -271,7 +281,7 @@ class HfRunner:
281
for prompt in prompts:
282
input_ids = self.tokenizer(prompt, return_tensors="pt").input_ids
283
output = self.model.generate(
274
- input_ids.cuda(),
284
+ self.wrap_device(input_ids),
285
use_cache=True,
286
do_sample=False,
287
max_new_tokens=max_tokens,
@@ -306,7 +316,7 @@ class HfRunner:
316
for prompt in prompts:
317
input_ids = self.tokenizer(prompt, return_tensors="pt").input_ids
318
output = self.model.generate(
309
- input_ids.cuda(),
319
+ self.wrap_device(input_ids),
320
use_cache=True,
321
do_sample=False,
322
max_new_tokens=max_tokens,
tests/models/test_aqlm.py
+7
-4
@@ -8,10 +8,13 @@ import torch
8
9
from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
10
11
-capability = torch.cuda.get_device_capability()
12
-capability = capability[0] * 10 + capability[1]
13
-aqlm_not_supported = (capability <
14
- QUANTIZATION_METHODS["aqlm"].get_min_capability())
11
+aqlm_not_supported = True
12
+
13
+if torch.cuda.is_available():
14
+ capability = torch.cuda.get_device_capability()
15
+ capability = capability[0] * 10 + capability[1]
16
+ aqlm_not_supported = (capability <
17
+ QUANTIZATION_METHODS["aqlm"].get_min_capability())
18
19
# In this test we hardcode prompts and generations for the model so we don't
20
# need to require the AQLM package as a dependency
tests/models/test_big_models.py
+8
-2
@@ -5,6 +5,7 @@ This tests bigger models and use half precision.
5
Run `pytest tests/models/test_big_models.py`.
6
"""
7
import pytest
8
+import torch
9
10
MODELS = [
11
"meta-llama/Llama-2-7b-hf",
@@ -16,9 +17,14 @@ MODELS = [
17
# "Qwen/Qwen1.5-0.5B" # Broken,
18
]
19
20
+#TODO: remove this after CPU float16 support ready
21
+target_dtype = "float"
22
+if torch.cuda.is_available():
23
+ target_dtype = "half"
24
+
25
26
@pytest.mark.parametrize("model", MODELS)
21
-@pytest.mark.parametrize("dtype", ["half"])
27
+@pytest.mark.parametrize("dtype", [target_dtype])
28
@pytest.mark.parametrize("max_tokens", [32])
29
def test_models(
30
hf_runner,
@@ -46,7 +52,7 @@ def test_models(
52
53
54
@pytest.mark.parametrize("model", MODELS)
49
-@pytest.mark.parametrize("dtype", ["half"])
55
+@pytest.mark.parametrize("dtype", [target_dtype])
56
def test_model_print(
57
vllm_runner,
58
model: str,
tests/models/test_fp8.py
+7
-4
@@ -67,10 +67,13 @@ EXPECTED_STRS_MAP = {
67
},
68
}
69
70
-capability = torch.cuda.get_device_capability()
71
-capability = capability[0] * 10 + capability[1]
72
-fp8_not_supported = (capability <
73
- QUANTIZATION_METHODS["fp8"].get_min_capability())
70
+fp8_not_supported = True
71
+
72
+if torch.cuda.is_available():
73
+ capability = torch.cuda.get_device_capability()
74
+ capability = capability[0] * 10 + capability[1]
75
+ fp8_not_supported = (capability <
76
+ QUANTIZATION_METHODS["fp8"].get_min_capability())
77
78
79
@pytest.mark.skipif(fp8_not_supported,
tests/models/test_gptq_marlin.py
+7
-4
@@ -22,10 +22,13 @@ os.environ["TOKENIZERS_PARALLELISM"] = "true"
22
23
MAX_MODEL_LEN = 1024
24
25
-capability = torch.cuda.get_device_capability()
26
-capability = capability[0] * 10 + capability[1]
27
-gptq_marlin_not_supported = (
28
- capability < QUANTIZATION_METHODS["gptq_marlin"].get_min_capability())
25
+gptq_marlin_not_supported = True
26
+
27
+if torch.cuda.is_available():
28
+ capability = torch.cuda.get_device_capability()
29
+ capability = capability[0] * 10 + capability[1]
30
+ gptq_marlin_not_supported = (
31
+ capability < QUANTIZATION_METHODS["gptq_marlin"].get_min_capability())
32
33
MODELS = [
34
# act_order==False, group_size=channelwise
tests/models/test_gptq_marlin_24.py
+7
-4
@@ -14,10 +14,13 @@ import torch
14
from tests.models.utils import check_logprobs_close
15
from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
16
17
-capability = torch.cuda.get_device_capability()
18
-capability = capability[0] * 10 + capability[1]
19
-marlin_not_supported = (capability <
20
- QUANTIZATION_METHODS["marlin"].get_min_capability())
17
+marlin_not_supported = True
18
+
19
+if torch.cuda.is_available():
20
+ capability = torch.cuda.get_device_capability()
21
+ capability = capability[0] * 10 + capability[1]
22
+ marlin_not_supported = (
23
+ capability < QUANTIZATION_METHODS["marlin"].get_min_capability())
24
25
26
@dataclass
tests/models/test_marlin.py
+7
-4
@@ -19,10 +19,13 @@ from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
19
20
from .utils import check_logprobs_close
21
22
-capability = torch.cuda.get_device_capability()
23
-capability = capability[0] * 10 + capability[1]
24
-marlin_not_supported = (capability <
25
- QUANTIZATION_METHODS["marlin"].get_min_capability())
22
+marlin_not_supported = True
23
+
24
+if torch.cuda.is_available():
25
+ capability = torch.cuda.get_device_capability()
26
+ capability = capability[0] * 10 + capability[1]
27
+ marlin_not_supported = (
28
+ capability < QUANTIZATION_METHODS["marlin"].get_min_capability())
29
30
31
@dataclass