@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / a4b3e0c1e9
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 08:15 UTC

[Hardware][CPU] Update torch 2.5 (#9911)

Signed-off-by: jiang1.li <jiang1.li@intel.com>

Li, Jiang committed Nov 7, 2024 at 12:43 UTC a4b3e0c1e999d214c6355b16a1c68250e6c030e2
12 files changed +76 -48
.buildkite/run-cpu-test.sh
+1 -1
@@ -46,7 +46,7 @@ docker exec cpu-test bash -c "
46 docker exec cpu-test bash -c "
47 export VLLM_CPU_KVCACHE_SPACE=10
48 export VLLM_CPU_OMP_THREADS_BIND=48-92
49 - python3 -m vllm.entrypoints.openai.api_server --model facebook/opt-125m &
49 + python3 -m vllm.entrypoints.openai.api_server --model facebook/opt-125m --dtype half &
50 timeout 600 bash -c 'until curl localhost:8000/v1/models; do sleep 1; done' || exit 1
51 python3 benchmarks/benchmark_serving.py \
52 --backend vllm \
Dockerfile.cpu
+1 -1
@@ -22,7 +22,7 @@ ENV LD_PRELOAD="/usr/lib/x86_64-linux-gnu/libtcmalloc_minimal.so.4:/usr/local/li
22
23 RUN echo 'ulimit -c 0' >> ~/.bashrc
24
25 -RUN pip install intel_extension_for_pytorch==2.4.0
25 +RUN pip install intel_extension_for_pytorch==2.5.0
26
27 WORKDIR /workspace
28
cmake/cpu_extension.cmake
+1
@@ -18,6 +18,7 @@ include_directories("${CMAKE_SOURCE_DIR}/csrc")
18 #
19 list(APPEND CXX_COMPILE_FLAGS
20 "-fopenmp"
21 + "-mf16c"
22 "-DVLLM_CPU_EXTENSION")
23
24 execute_process(COMMAND cat /proc/cpuinfo
csrc/cpu/attention.cpp
+10
@@ -22,6 +22,16 @@ struct KernelVecType<float> {
22 using v_load_vec_type = vec_op::FP32Vec16;
23 };
24
25 +template <>
26 +struct KernelVecType<c10::Half> {
27 + using q_load_vec_type = vec_op::FP16Vec8;
28 + using q_vec_type = vec_op::FP32Vec16;
29 + using k_load_vec_type = vec_op::FP16Vec16;
30 + using k_vec_type = vec_op::FP32Vec16;
31 + using qk_acc_vec_type = vec_op::FP32Vec16;
32 + using v_load_vec_type = vec_op::FP16Vec16;
33 +};
34 +
35 #ifdef __AVX512BF16__
36 template <>
37 struct KernelVecType<c10::BFloat16> {
csrc/cpu/cpu_types_x86.hpp
+45 -33
@@ -11,10 +11,10 @@ static_assert(false, "AVX2 must be supported for the current implementation.");
11
12 namespace vec_op {
13
14 -// FIXME: FP16 is not fully supported in Torch-CPU
14 #define VLLM_DISPATCH_CASE_FLOATING_TYPES(...) \
15 AT_DISPATCH_CASE(at::ScalarType::Float, __VA_ARGS__) \
17 - AT_DISPATCH_CASE(at::ScalarType::BFloat16, __VA_ARGS__)
16 + AT_DISPATCH_CASE(at::ScalarType::BFloat16, __VA_ARGS__) \
17 + AT_DISPATCH_CASE(at::ScalarType::Half, __VA_ARGS__)
18
19 #define VLLM_DISPATCH_FLOATING_TYPES(TYPE, NAME, ...) \
20 AT_DISPATCH_SWITCH(TYPE, NAME, VLLM_DISPATCH_CASE_FLOATING_TYPES(__VA_ARGS__))
@@ -50,37 +50,37 @@ template <typename T> struct Vec {
50 struct FP32Vec8;
51 struct FP32Vec16;
52
53 -#ifdef __AVX512FP16__
53 struct FP16Vec8 : public Vec<FP16Vec8> {
54 constexpr static int VEC_ELEM_NUM = 8;
55
57 - __m128h reg;
56 + __m128i reg;
57
59 - explicit FP16Vec8(_Float16 v) : reg(_mm_set1_ph(v)) {}
58 + explicit FP16Vec8(const void *ptr)
59 + : reg((__m128i)_mm_loadu_si128((__m128i *)ptr)) {}
60
61 - explicit FP16Vec8(const void *ptr) : reg(_mm_loadu_ph(ptr)) {}
61 + explicit FP16Vec8(const FP32Vec8 &);
62
63 - explicit FP16Vec8(__m128h data) : reg(data) {}
63 + void save(void *ptr) const { *reinterpret_cast<__m128i *>(ptr) = reg; }
64 +};
65
65 - FP16Vec8 operator*(const FP16Vec8 &b) const {
66 - return FP16Vec8(_mm_mul_ph(reg, b.reg));
67 - }
66 +struct FP16Vec16 : public Vec<FP16Vec16> {
67 + constexpr static int VEC_ELEM_NUM = 16;
68
69 - FP16Vec8 operator+(const FP16Vec8 &b) const {
70 - return FP16Vec8(_mm_add_ph(reg, b.reg));
71 - }
69 + __m256i reg;
70
73 - FP16Vec8 operator-(const FP16Vec8 &b) const {
74 - return FP16Vec8(_mm_sub_ph(reg, b.reg));
75 - }
71 + explicit FP16Vec16(const void *ptr)
72 + : reg((__m256i)_mm256_loadu_si256((__m256i *)ptr)) {}
73
77 - FP16Vec8 operator/(const FP16Vec8 &b) const {
78 - return FP16Vec8(_mm_div_ph(reg, b.reg));
79 - }
74 + explicit FP16Vec16(const FP32Vec16 &);
75
81 - void save(void *ptr) const { _mm_storeu_ph(ptr, reg); }
76 + void save(void *ptr) const { *reinterpret_cast<__m256i *>(ptr) = reg; }
77 +
78 + void save(void* ptr, const int elem_num) const {
79 + constexpr uint32_t M = 0xFFFFFFFF;
80 + __mmask16 mask = _cvtu32_mask16(M >> (32 - elem_num));
81 + _mm256_mask_storeu_epi16(ptr, mask, reg);
82 + }
83 };
83 -#endif
84
85 struct BF16Vec8 : public Vec<BF16Vec8> {
86 constexpr static int VEC_ELEM_NUM = 8;
@@ -202,9 +202,7 @@ struct FP32Vec8 : public Vec<FP32Vec8> {
202
203 explicit FP32Vec8(const FP32Vec8 &data) : reg(data.reg) {}
204
205 -#ifdef __AVX512FP16__
206 - explicit FP32Vec8(__m128h v) : reg(_mm256_cvtph_ps(_mm_castph_si128(v))) {}
207 -#endif
205 + explicit FP32Vec8(const FP16Vec8 &v) : reg(_mm256_cvtph_ps(v.reg)) {}
206
207 explicit FP32Vec8(const BF16Vec8 &v)
208 : reg(_mm256_castsi256_ps(
@@ -323,6 +321,10 @@ struct FP32Vec16 : public Vec<FP32Vec16> {
321 : reg(_mm512_castsi512_ps(
322 _mm512_bslli_epi128(_mm512_cvtepu16_epi32(v.reg), 2))) {}
323
324 + explicit FP32Vec16(const FP16Vec16 &v) : reg(_mm512_cvtph_ps(v.reg)) {}
325 +
326 + explicit FP32Vec16(const FP16Vec8 &v) : FP32Vec16(FP32Vec8(v)) {}
327 +
328 explicit FP32Vec16(const BF16Vec8 &v) : FP32Vec16(FP32Vec8(v)) {}
329
330 explicit FP32Vec16(const INT32Vec16 &v)
@@ -534,24 +536,34 @@ template <typename T> using vec_t = typename VecType<T>::vec_type;
536
537 template <> struct VecType<float> { using vec_type = FP32Vec8; };
538
537 -#ifdef __AVX512FP16__
538 -template <> struct VecType<c10::Half> { using vec_type = FP16Vec16; };
539 -#endif
539 +template <> struct VecType<c10::Half> { using vec_type = FP16Vec8; };
540
541 template <> struct VecType<c10::BFloat16> { using vec_type = BF16Vec8; };
542
543 template <typename T> void storeFP32(float v, T *ptr) { *ptr = v; }
544
545 -#ifdef __AVX512FP16__
546 -template <> inline void storeFP32<c10::Half>(float v, c10::Half *ptr) {
547 - *reinterpret_cast<_Float16 *>(ptr) = v;
548 -}
549 -#endif
550 -
545 inline void fma(FP32Vec16 &acc, FP32Vec16 &a, FP32Vec16 &b) {
546 acc = acc + a * b;
547 }
548
549 +template <> inline void storeFP32<c10::Half>(float v, c10::Half *ptr) {
550 + *reinterpret_cast<unsigned short *>(ptr) =
551 + _cvtss_sh(v, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC);
552 +}
553 +
554 +inline FP16Vec8::FP16Vec8(const FP32Vec8 &v)
555 + : reg(_mm256_cvtps_ph(v.reg,
556 + _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC)) {}
557 +
558 +#ifdef __AVX512F__
559 +inline FP16Vec16::FP16Vec16(const FP32Vec16 &v)
560 + : reg(_mm512_cvtps_ph(v.reg,
561 + _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC)) {}
562 +#else
563 +inline FP16Vec16::FP16Vec16(const FP32Vec16 &v)
564 + : reg(_mm256_insertf128_si256(_mm256_castsi128_si256(FP16Vec8(FP32Vec8(v.reg_low)).reg), FP16Vec8(FP32Vec8(v.reg_low)).reg, 1)) {}
565 +#endif
566 +
567 #ifdef __AVX512BF16__
568 template <> inline void storeFP32<c10::BFloat16>(float v, c10::BFloat16 *ptr) {
569 *reinterpret_cast<__bfloat16 *>(ptr) = _mm_cvtness_sbh(v);
csrc/cpu/dnnl_helper.hpp
+6
@@ -2,6 +2,7 @@
2 #define DNNL_HELPER_HPP
3
4 #include <c10/util/BFloat16.h>
5 +#include <c10/util/Half.h>
6
7 #include "oneapi/dnnl/dnnl.hpp"
8
@@ -32,6 +33,11 @@ struct DNNLType<c10::BFloat16> {
33 static constexpr dnnl::memory::data_type type = dnnl::memory::data_type::bf16;
34 };
35
36 +template <>
37 +struct DNNLType<c10::Half> {
38 + static constexpr dnnl::memory::data_type type = dnnl::memory::data_type::f16;
39 +};
40 +
41 template <typename T>
42 constexpr inline dnnl::memory::data_type get_dnnl_type() {
43 return DNNLType<std::decay_t<T>>::type;
csrc/cpu/quant.cpp
+7
@@ -23,6 +23,13 @@ struct KernelVecType<c10::BFloat16> {
23 using cvt_vec_type = vec_op::FP32Vec16;
24 };
25
26 +template <>
27 +struct KernelVecType<c10::Half> {
28 + using load_vec_type = vec_op::FP16Vec16;
29 + using azp_adj_load_vec_type = vec_op::INT32Vec16;
30 + using cvt_vec_type = vec_op::FP32Vec16;
31 +};
32 +
33 #ifdef __AVX512F__
34 template <bool AZP, typename scalar_t>
35 void static_scaled_int8_quant_impl(const scalar_t* input, int8_t* output,
docs/source/getting_started/cpu-installation.rst
+2 -4
@@ -3,13 +3,13 @@
3 Installation with CPU
4 ========================
5
6 -vLLM initially supports basic model inferencing and serving on x86 CPU platform, with data types FP32 and BF16. vLLM CPU backend supports the following vLLM features:
6 +vLLM initially supports basic model inferencing and serving on x86 CPU platform, with data types FP32, FP16 and BF16. vLLM CPU backend supports the following vLLM features:
7
8 - Tensor Parallel (``-tp = N``)
9 - Quantization (``INT8 W8A8, AWQ``)
10
11 .. note::
12 - FP16 data type and more advanced features on `chunked-prefill`, `prefix-caching` and `FP8 KV cache` are under development and will be available soon.
12 + More advanced features on `chunked-prefill`, `prefix-caching` and `FP8 KV cache` are under development and will be available soon.
13
14 Table of contents:
15
@@ -72,8 +72,6 @@ Build from source
72 $ VLLM_TARGET_DEVICE=cpu python setup.py install
73
74 .. note::
75 - - BF16 is the default data type in the current CPU backend (that means the backend will cast FP16 to BF16), and is compatible will all CPUs with AVX512 ISA support.
76 -
75 - AVX512_BF16 is an extension ISA provides native BF16 data type conversion and vector product instructions, will brings some performance improvement compared with pure AVX512. The CPU backend build script will check the host CPU flags to determine whether to enable AVX512_BF16.
76
77 - If you want to force enable AVX512_BF16 for the cross-compilation, please set environment variable VLLM_CPU_AVX512BF16=1 before the building.
requirements-cpu.txt
+1 -1
@@ -2,5 +2,5 @@
2 -r requirements-common.txt
3
4 # Dependencies for x86_64 CPUs
5 -torch == 2.4.0+cpu; platform_machine != "ppc64le"
5 +torch == 2.5.1+cpu; platform_machine != "ppc64le"
6 torchvision; platform_machine != "ppc64le" # required for the image processor of phi3v, this must be updated alongside torch
tests/models/decoder_only/language/test_models.py
+1 -2
@@ -32,8 +32,7 @@ if not current_platform.is_cpu():
32 "openbmb/MiniCPM3-4B",
33 ]
34
35 -# TODO: remove this after CPU float16 support ready
36 -target_dtype = "float" if current_platform.is_cpu() else "half"
35 +target_dtype = "half"
36
37
38 @pytest.mark.parametrize("model", MODELS)
vllm/executor/cpu_executor.py
-5
@@ -2,8 +2,6 @@ import os
2 from functools import partial
3 from typing import Any, Awaitable, List, Optional, Set, Tuple, Union
4
5 -import torch
6 -
5 import vllm.envs as envs
6 from vllm.config import (CacheConfig, ModelConfig, ParallelConfig,
7 SchedulerConfig)
@@ -316,9 +314,6 @@ class CPUExecutorAsync(CPUExecutor, ExecutorAsyncBase):
314
315
316 def _verify_and_get_model_config(config: ModelConfig) -> ModelConfig:
319 - if config.dtype == torch.float16:
320 - logger.warning("float16 is not supported on CPU, casting to bfloat16.")
321 - config.dtype = torch.bfloat16
317 # Reminder: Please update docs/source/serving/compatibility_matrix.rst
318 # If the feature combo become valid
319 if not config.enforce_eager:
vllm/model_executor/layers/quantization/ipex_quant.py
+1 -1
@@ -54,7 +54,7 @@ class IPEXConfig(QuantizationConfig):
54
55 @classmethod
56 def get_supported_act_dtypes(cls) -> List[torch.dtype]:
57 - return [torch.bfloat16]
57 + return [torch.bfloat16, torch.float16]
58
59 @classmethod
60 def get_min_capability(cls) -> int: