A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 02:00 UTC
Move linting to `pre-commit` (#11975)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Harry Mellor committed
Jan 20, 2025 at 06:58 UTC
3ea7b94523f748faf464293ca5fdc4c94e3a3a89
26 files changed
+725
-1287
.buildkite/nightly-benchmarks/scripts/nightly-annotate.sh
+1
-1
@@ -43,7 +43,7 @@ main() {
43
44
45
46
- # The figures should be genereated by a separate process outside the CI/CD pipeline
46
+ # The figures should be generated by a separate process outside the CI/CD pipeline
47
48
# # generate figures
49
# python3 -m pip install tabulate pandas matplotlib
.github/workflows/actionlint.yml
deleted
-40
@@ -1,40 +0,0 @@
1
-name: Lint GitHub Actions workflows
2
-on:
3
- push:
4
- branches:
5
- - "main"
6
- paths:
7
- - '.github/workflows/*.ya?ml'
8
- - '.github/workflows/actionlint.*'
9
- - '.github/workflows/matchers/actionlint.json'
10
- pull_request:
11
- branches:
12
- - "main"
13
- paths:
14
- - '.github/workflows/*.ya?ml'
15
- - '.github/workflows/actionlint.*'
16
- - '.github/workflows/matchers/actionlint.json'
17
-
18
-env:
19
- LC_ALL: en_US.UTF-8
20
-
21
-defaults:
22
- run:
23
- shell: bash
24
-
25
-permissions:
26
- contents: read
27
-
28
-jobs:
29
- actionlint:
30
- runs-on: ubuntu-latest
31
- steps:
32
- - name: "Checkout"
33
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
34
- with:
35
- fetch-depth: 0
36
-
37
- - name: "Run actionlint"
38
- run: |
39
- echo "::add-matcher::.github/workflows/matchers/actionlint.json"
40
- tools/actionlint.sh -color
.github/workflows/clang-format.yml
deleted
-53
@@ -1,53 +0,0 @@
1
-name: clang-format
2
-
3
-on:
4
- # Trigger the workflow on push or pull request,
5
- # but only for the main branch
6
- push:
7
- branches:
8
- - main
9
- paths:
10
- - '**/*.h'
11
- - '**/*.cpp'
12
- - '**/*.cu'
13
- - '**/*.cuh'
14
- - '.github/workflows/clang-format.yml'
15
- pull_request:
16
- branches:
17
- - main
18
- paths:
19
- - '**/*.h'
20
- - '**/*.cpp'
21
- - '**/*.cu'
22
- - '**/*.cuh'
23
- - '.github/workflows/clang-format.yml'
24
-
25
-jobs:
26
- clang-format:
27
- runs-on: ubuntu-latest
28
- strategy:
29
- matrix:
30
- python-version: ["3.11"]
31
- steps:
32
- - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
33
- - name: Set up Python ${{ matrix.python-version }}
34
- uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0
35
- with:
36
- python-version: ${{ matrix.python-version }}
37
- - name: Install dependencies
38
- run: |
39
- python -m pip install --upgrade pip
40
- pip install clang-format==18.1.5
41
- - name: Running clang-format
42
- run: |
43
- EXCLUDES=(
44
- 'csrc/moe/topk_softmax_kernels.cu'
45
- 'csrc/quantization/gguf/ggml-common.h'
46
- 'csrc/quantization/gguf/dequantize.cuh'
47
- 'csrc/quantization/gguf/vecdotq.cuh'
48
- 'csrc/quantization/gguf/mmq.cuh'
49
- 'csrc/quantization/gguf/mmvq.cuh'
50
- )
51
- find csrc/ \( -name '*.h' -o -name '*.cpp' -o -name '*.cu' -o -name '*.cuh' \) -print \
52
- | grep -vFf <(printf "%s\n" "${EXCLUDES[@]}") \
53
- | xargs clang-format --dry-run --Werror
.github/workflows/codespell.yml
deleted
-45
@@ -1,45 +0,0 @@
1
-name: codespell
2
-
3
-on:
4
- # Trigger the workflow on push or pull request,
5
- # but only for the main branch
6
- push:
7
- branches:
8
- - main
9
- paths:
10
- - "**/*.py"
11
- - "**/*.md"
12
- - "**/*.rst"
13
- - pyproject.toml
14
- - requirements-lint.txt
15
- - .github/workflows/codespell.yml
16
- pull_request:
17
- branches:
18
- - main
19
- paths:
20
- - "**/*.py"
21
- - "**/*.md"
22
- - "**/*.rst"
23
- - pyproject.toml
24
- - requirements-lint.txt
25
- - .github/workflows/codespell.yml
26
-
27
-jobs:
28
- codespell:
29
- runs-on: ubuntu-latest
30
- strategy:
31
- matrix:
32
- python-version: ["3.12"]
33
- steps:
34
- - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
35
- - name: Set up Python ${{ matrix.python-version }}
36
- uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0
37
- with:
38
- python-version: ${{ matrix.python-version }}
39
- - name: Install dependencies
40
- run: |
41
- python -m pip install --upgrade pip
42
- pip install -r requirements-lint.txt
43
- - name: Spelling check with codespell
44
- run: |
45
- codespell --toml pyproject.toml
.github/workflows/doc-lint.yml
deleted
-32
@@ -1,32 +0,0 @@
1
-name: Lint documentation
2
-
3
-on:
4
- push:
5
- branches:
6
- - main
7
- paths:
8
- - "docs/**"
9
- pull_request:
10
- branches:
11
- - main
12
- paths:
13
- - "docs/**"
14
-
15
-jobs:
16
- doc-lint:
17
- runs-on: ubuntu-latest
18
- strategy:
19
- matrix:
20
- python-version: ["3.12"]
21
- steps:
22
- - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
23
- - name: Set up Python ${{ matrix.python-version }}
24
- uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0
25
- with:
26
- python-version: ${{ matrix.python-version }}
27
- - name: Install dependencies
28
- run: |
29
- python -m pip install --upgrade pip
30
- pip install -r requirements-lint.txt
31
- - name: Linting docs
32
- run: tools/doc-lint.sh
.github/workflows/dummy.yml
new
+20
@@ -0,0 +1,20 @@
1
+name: dummy-checks
2
+
3
+on:
4
+ pull_request:
5
+
6
+jobs:
7
+ mypy:
8
+ runs-on: ubuntu-latest
9
+ strategy:
10
+ matrix:
11
+ python-version: ["3.12"]
12
+ steps:
13
+ - run: echo "This is a dummy step that always passes"
14
+ ruff:
15
+ runs-on: ubuntu-latest
16
+ strategy:
17
+ matrix:
18
+ python-version: ["3.12"]
19
+ steps:
20
+ - run: echo "This is a dummy step that always passes"
.github/workflows/matchers/ruff.json
deleted
-17
@@ -1,17 +0,0 @@
1
-{
2
- "problemMatcher": [
3
- {
4
- "owner": "ruff",
5
- "pattern": [
6
- {
7
- "regexp": "^(.+?):(\\d+):(\\d+): (\\w+): (.+)$",
8
- "file": 1,
9
- "line": 2,
10
- "column": 3,
11
- "code": 4,
12
- "message": 5
13
- }
14
- ]
15
- }
16
- ]
17
- }
.github/workflows/mypy.yaml
deleted
-51
@@ -1,51 +0,0 @@
1
-name: mypy
2
-
3
-on:
4
- # Trigger the workflow on push or pull request,
5
- # but only for the main branch
6
- push:
7
- branches:
8
- - main
9
- paths:
10
- - '**/*.py'
11
- - '.github/workflows/mypy.yaml'
12
- - 'tools/mypy.sh'
13
- - 'pyproject.toml'
14
- pull_request:
15
- branches:
16
- - main
17
- # This workflow is only relevant when one of the following files changes.
18
- # However, we have github configured to expect and require this workflow
19
- # to run and pass before github with auto-merge a pull request. Until github
20
- # allows more flexible auto-merge policy, we can just run this on every PR.
21
- # It doesn't take that long to run, anyway.
22
- #paths:
23
- # - '**/*.py'
24
- # - '.github/workflows/mypy.yaml'
25
- # - 'tools/mypy.sh'
26
- # - 'pyproject.toml'
27
-
28
-jobs:
29
- mypy:
30
- runs-on: ubuntu-latest
31
- strategy:
32
- matrix:
33
- python-version: ["3.9", "3.10", "3.11", "3.12"]
34
- steps:
35
- - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
36
- - name: Set up Python ${{ matrix.python-version }}
37
- uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0
38
- with:
39
- python-version: ${{ matrix.python-version }}
40
- - name: Install dependencies
41
- run: |
42
- python -m pip install --upgrade pip
43
- pip install mypy==1.11.1
44
- pip install types-setuptools
45
- pip install types-PyYAML
46
- pip install types-requests
47
- pip install types-setuptools
48
- - name: Mypy
49
- run: |
50
- echo "::add-matcher::.github/workflows/matchers/mypy.json"
51
- tools/mypy.sh 1 ${{ matrix.python-version }}
.github/workflows/png-lint.yml
deleted
-37
@@ -1,37 +0,0 @@
1
-name: Lint PNG exports from excalidraw
2
-on:
3
- push:
4
- branches:
5
- - "main"
6
- paths:
7
- - '*.excalidraw.png'
8
- - '.github/workflows/png-lint.yml'
9
- pull_request:
10
- branches:
11
- - "main"
12
- paths:
13
- - '*.excalidraw.png'
14
- - '.github/workflows/png-lint.yml'
15
-
16
-env:
17
- LC_ALL: en_US.UTF-8
18
-
19
-defaults:
20
- run:
21
- shell: bash
22
-
23
-permissions:
24
- contents: read
25
-
26
-jobs:
27
- actionlint:
28
- runs-on: ubuntu-latest
29
- steps:
30
- - name: "Checkout"
31
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
32
- with:
33
- fetch-depth: 0
34
-
35
- - name: "Run png-lint.sh to check excalidraw exported images"
36
- run: |
37
- tools/png-lint.sh
.github/workflows/pre-commit.yml
new
+17
@@ -0,0 +1,17 @@
1
+name: pre-commit
2
+
3
+on:
4
+ pull_request:
5
+ push:
6
+ branches: [main]
7
+
8
+jobs:
9
+ pre-commit:
10
+ runs-on: ubuntu-latest
11
+ steps:
12
+ - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
13
+ - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0
14
+ with:
15
+ python-version: "3.12"
16
+ - run: echo "::add-matcher::.github/workflows/matchers/actionlint.json"
17
+ - uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd # v3.0.1
.github/workflows/ruff.yml
deleted
-52
@@ -1,52 +0,0 @@
1
-name: ruff
2
-
3
-on:
4
- # Trigger the workflow on push or pull request,
5
- # but only for the main branch
6
- push:
7
- branches:
8
- - main
9
- paths:
10
- - "**/*.py"
11
- - pyproject.toml
12
- - requirements-lint.txt
13
- - .github/workflows/matchers/ruff.json
14
- - .github/workflows/ruff.yml
15
- pull_request:
16
- branches:
17
- - main
18
- # This workflow is only relevant when one of the following files changes.
19
- # However, we have github configured to expect and require this workflow
20
- # to run and pass before github with auto-merge a pull request. Until github
21
- # allows more flexible auto-merge policy, we can just run this on every PR.
22
- # It doesn't take that long to run, anyway.
23
- #paths:
24
- # - "**/*.py"
25
- # - pyproject.toml
26
- # - requirements-lint.txt
27
- # - .github/workflows/matchers/ruff.json
28
- # - .github/workflows/ruff.yml
29
-
30
-jobs:
31
- ruff:
32
- runs-on: ubuntu-latest
33
- strategy:
34
- matrix:
35
- python-version: ["3.12"]
36
- steps:
37
- - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
38
- - name: Set up Python ${{ matrix.python-version }}
39
- uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0
40
- with:
41
- python-version: ${{ matrix.python-version }}
42
- - name: Install dependencies
43
- run: |
44
- python -m pip install --upgrade pip
45
- pip install -r requirements-lint.txt
46
- - name: Analysing the code with ruff
47
- run: |
48
- echo "::add-matcher::.github/workflows/matchers/ruff.json"
49
- ruff check --output-format github .
50
- - name: Run isort
51
- run: |
52
- isort . --check-only
.github/workflows/shellcheck.yml
deleted
-37
@@ -1,37 +0,0 @@
1
-name: Lint shell scripts
2
-on:
3
- push:
4
- branches:
5
- - "main"
6
- paths:
7
- - '**/*.sh'
8
- - '.github/workflows/shellcheck.yml'
9
- pull_request:
10
- branches:
11
- - "main"
12
- paths:
13
- - '**/*.sh'
14
- - '.github/workflows/shellcheck.yml'
15
-
16
-env:
17
- LC_ALL: en_US.UTF-8
18
-
19
-defaults:
20
- run:
21
- shell: bash
22
-
23
-permissions:
24
- contents: read
25
-
26
-jobs:
27
- shellcheck:
28
- runs-on: ubuntu-latest
29
- steps:
30
- - name: "Checkout"
31
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
32
- with:
33
- fetch-depth: 0
34
-
35
- - name: "Check shell scripts"
36
- run: |
37
- tools/shellcheck.sh
.github/workflows/yapf.yml
deleted
-38
@@ -1,38 +0,0 @@
1
-name: yapf
2
-
3
-on:
4
- # Trigger the workflow on push or pull request,
5
- # but only for the main branch
6
- push:
7
- branches:
8
- - main
9
- paths:
10
- - "**/*.py"
11
- - .github/workflows/yapf.yml
12
- pull_request:
13
- branches:
14
- - main
15
- paths:
16
- - "**/*.py"
17
- - .github/workflows/yapf.yml
18
-
19
-jobs:
20
- yapf:
21
- runs-on: ubuntu-latest
22
- strategy:
23
- matrix:
24
- python-version: ["3.12"]
25
- steps:
26
- - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
27
- - name: Set up Python ${{ matrix.python-version }}
28
- uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0
29
- with:
30
- python-version: ${{ matrix.python-version }}
31
- - name: Install dependencies
32
- run: |
33
- python -m pip install --upgrade pip
34
- pip install yapf==0.32.0
35
- pip install toml==0.10.2
36
- - name: Running yapf
37
- run: |
38
- yapf --diff --recursive .
.pre-commit-config.yaml
new
+73
@@ -0,0 +1,73 @@
1
+repos:
2
+- repo: https://github.com/google/yapf
3
+ rev: v0.32.0
4
+ hooks:
5
+ - id: yapf
6
+ args: [--in-place, --verbose]
7
+ additional_dependencies: [toml] # TODO: Remove when yapf is upgraded
8
+- repo: https://github.com/astral-sh/ruff-pre-commit
9
+ rev: v0.6.5
10
+ hooks:
11
+ - id: ruff
12
+ args: [--output-format, github]
13
+- repo: https://github.com/codespell-project/codespell
14
+ rev: v2.3.0
15
+ hooks:
16
+ - id: codespell
17
+ exclude: 'benchmarks/sonnet.txt|(build|tests/(lora/data|models/fixtures|prompts))/.*'
18
+- repo: https://github.com/PyCQA/isort
19
+ rev: 5.13.2
20
+ hooks:
21
+ - id: isort
22
+- repo: https://github.com/pre-commit/mirrors-clang-format
23
+ rev: v18.1.5
24
+ hooks:
25
+ - id: clang-format
26
+ exclude: 'csrc/(moe/topk_softmax_kernels.cu|quantization/gguf/(ggml-common.h|dequantize.cuh|vecdotq.cuh|mmq.cuh|mmvq.cuh))'
27
+ types_or: [c++, cuda]
28
+ args: [--style=file, --verbose]
29
+- repo: https://github.com/jackdewinter/pymarkdown
30
+ rev: v0.9.27
31
+ hooks:
32
+ - id: pymarkdown
33
+ files: docs/.*
34
+- repo: local
35
+ hooks:
36
+ - id: mypy-3.9 # TODO: Use https://github.com/pre-commit/mirrors-mypy when mypy setup is less awkward
37
+ name: Run mypy for Python 3.9
38
+ entry: tools/mypy.sh 1 "3.9"
39
+ language: python
40
+ types: [python]
41
+ additional_dependencies: &mypy_deps [mypy==1.11.1, types-setuptools, types-PyYAML, types-requests]
42
+ - id: mypy-3.10 # TODO: Use https://github.com/pre-commit/mirrors-mypy when mypy setup is less awkward
43
+ name: Run mypy for Python 3.10
44
+ entry: tools/mypy.sh 1 "3.10"
45
+ language: python
46
+ types: [python]
47
+ additional_dependencies: *mypy_deps
48
+ - id: mypy-3.11 # TODO: Use https://github.com/pre-commit/mirrors-mypy when mypy setup is less awkward
49
+ name: Run mypy for Python 3.11
50
+ entry: tools/mypy.sh 1 "3.11"
51
+ language: python
52
+ types: [python]
53
+ additional_dependencies: *mypy_deps
54
+ - id: mypy-3.12 # TODO: Use https://github.com/pre-commit/mirrors-mypy when mypy setup is less awkward
55
+ name: Run mypy for Python 3.12
56
+ entry: tools/mypy.sh 1 "3.12"
57
+ language: python
58
+ types: [python]
59
+ additional_dependencies: *mypy_deps
60
+ - id: shellcheck
61
+ name: Lint shell scripts
62
+ entry: tools/shellcheck.sh
63
+ language: script
64
+ types: [shell]
65
+ - id: png-lint
66
+ name: Lint PNG exports from excalidraw
67
+ entry: tools/png-lint.sh
68
+ language: script
69
+ types: [png]
70
+- repo: https://github.com/rhysd/actionlint
71
+ rev: v1.7.6
72
+ hooks:
73
+ - id: actionlint
csrc/core/scalar_type.hpp
+1
-1
@@ -32,7 +32,7 @@ class ScalarType {
32
signed_(signed_),
33
bias(bias),
34
finite_values_only(finite_values_only),
35
- nan_repr(nan_repr){};
35
+ nan_repr(nan_repr) {};
36
37
static constexpr ScalarType int_(uint8_t size_bits, int32_t bias = 0) {
38
return ScalarType(0, size_bits - 1, true, bias);
csrc/cpu/cpu_types.hpp
+3
-3
@@ -2,13 +2,13 @@
2
#define CPU_TYPES_HPP
3
4
#if defined(__x86_64__)
5
- //x86 implementation
5
+ // x86 implementation
6
#include "cpu_types_x86.hpp"
7
#elif defined(__POWER9_VECTOR__)
8
- //ppc implementation
8
+ // ppc implementation
9
#include "cpu_types_vsx.hpp"
10
#elif defined(__aarch64__)
11
- //arm implementation
11
+ // arm implementation
12
#include "cpu_types_arm.hpp"
13
#else
14
#warning "unsupported vLLM cpu implementation"
csrc/cpu/cpu_types_arm.hpp
+284
-265
@@ -1,48 +1,50 @@
1
#include <arm_neon.h>
2
-#include <torch/all.h>
2
+#include <torch/all.h>
3
#include <cmath>
4
5
namespace vec_op {
6
7
#ifdef ARM_BF16_SUPPORT
8
- #define VLLM_DISPATCH_CASE_FLOATING_TYPES(...) \
9
- AT_DISPATCH_CASE(at::ScalarType::Float, __VA_ARGS__) \
10
- AT_DISPATCH_CASE(at::ScalarType::Half, __VA_ARGS__) \
11
- AT_DISPATCH_CASE(at::ScalarType::BFloat16, __VA_ARGS__)
8
+ #define VLLM_DISPATCH_CASE_FLOATING_TYPES(...) \
9
+ AT_DISPATCH_CASE(at::ScalarType::Float, __VA_ARGS__) \
10
+ AT_DISPATCH_CASE(at::ScalarType::Half, __VA_ARGS__) \
11
+ AT_DISPATCH_CASE(at::ScalarType::BFloat16, __VA_ARGS__)
12
#else
13
- #define VLLM_DISPATCH_CASE_FLOATING_TYPES(...) \
14
- AT_DISPATCH_CASE(at::ScalarType::Float, __VA_ARGS__) \
13
+ #define VLLM_DISPATCH_CASE_FLOATING_TYPES(...) \
14
+ AT_DISPATCH_CASE(at::ScalarType::Float, __VA_ARGS__) \
15
AT_DISPATCH_CASE(at::ScalarType::Half, __VA_ARGS__)
16
#endif
17
18
-#define VLLM_DISPATCH_FLOATING_TYPES(TYPE, NAME, ...) \
18
+#define VLLM_DISPATCH_FLOATING_TYPES(TYPE, NAME, ...) \
19
AT_DISPATCH_SWITCH(TYPE, NAME, VLLM_DISPATCH_CASE_FLOATING_TYPES(__VA_ARGS__))
20
21
#ifndef CPU_OP_GUARD
22
-#define CPU_KERNEL_GUARD_IN(NAME)
23
-#define CPU_KERNEL_GUARD_OUT(NAME)
22
+ #define CPU_KERNEL_GUARD_IN(NAME)
23
+ #define CPU_KERNEL_GUARD_OUT(NAME)
24
#else
25
-#define CPU_KERNEL_GUARD_IN(NAME) \
26
- std::cout << #NAME << " invoked." << std::endl;
27
-#define CPU_KERNEL_GUARD_OUT(NAME) std::cout << #NAME << " exit." << std::endl;
25
+ #define CPU_KERNEL_GUARD_IN(NAME) \
26
+ std::cout << #NAME << " invoked." << std::endl;
27
+ #define CPU_KERNEL_GUARD_OUT(NAME) \
28
+ std::cout << #NAME << " exit." << std::endl;
29
#endif
30
31
#define FORCE_INLINE __attribute__((always_inline)) inline
32
33
namespace {
33
- template <typename T, T... indexes, typename F>
34
- constexpr void unroll_loop_item(std::integer_sequence<T, indexes...>, F &&f) {
35
- (f(std::integral_constant<T, indexes>{}), ...);
36
- };
37
-};
34
+template <typename T, T... indexes, typename F>
35
+constexpr void unroll_loop_item(std::integer_sequence<T, indexes...>, F&& f) {
36
+ (f(std::integral_constant<T, indexes>{}), ...);
37
+};
38
+}; // namespace
39
40
template <typename T, T count, typename F,
41
typename = std::enable_if_t<std::is_invocable_v<F, T>>>
41
-constexpr void unroll_loop(F &&f) {
42
+constexpr void unroll_loop(F&& f) {
43
unroll_loop_item(std::make_integer_sequence<T, count>{}, std::forward<F>(f));
44
}
45
45
-template <typename T> struct Vec {
46
+template <typename T>
47
+struct Vec {
48
constexpr static int get_elem_num() { return T::VEC_ELEM_NUM; };
49
};
50
@@ -54,127 +56,124 @@ struct FP16Vec8 : public Vec<FP16Vec8> {
56
57
float16x8_t reg;
58
57
- explicit FP16Vec8(const void *ptr)
58
- : reg(vld1q_f16(static_cast<const __fp16 *>(ptr))) {};
59
+ explicit FP16Vec8(const void* ptr)
60
+ : reg(vld1q_f16(static_cast<const __fp16*>(ptr))) {};
61
60
- explicit FP16Vec8(const FP32Vec8 &);
62
+ explicit FP16Vec8(const FP32Vec8&);
63
62
- void save(void *ptr) const {
63
- vst1q_f16(static_cast<__fp16 *>(ptr), reg);
64
- }
64
+ void save(void* ptr) const { vst1q_f16(static_cast<__fp16*>(ptr), reg); }
65
};
66
67
struct FP16Vec16 : public Vec<FP16Vec16> {
68
- constexpr static int VEC_ELEM_NUM = 16;
69
-
70
- float16x8x2_t reg;
71
-
72
- explicit FP16Vec16(const void *ptr) {
73
- reg.val[0] = vld1q_f16(reinterpret_cast<const __fp16*>(ptr));
74
- reg.val[1] = vld1q_f16(reinterpret_cast<const __fp16*>(ptr) + 8);
75
- }
76
-
77
- explicit FP16Vec16(const FP32Vec16& vec);
78
-
79
- void save(void *ptr) const {
80
- vst1q_f16(reinterpret_cast<__fp16*>(ptr), reg.val[0]);
81
- vst1q_f16(reinterpret_cast<__fp16*>(ptr) + 8, reg.val[1]);
68
+ constexpr static int VEC_ELEM_NUM = 16;
69
+
70
+ float16x8x2_t reg;
71
+
72
+ explicit FP16Vec16(const void* ptr) {
73
+ reg.val[0] = vld1q_f16(reinterpret_cast<const __fp16*>(ptr));
74
+ reg.val[1] = vld1q_f16(reinterpret_cast<const __fp16*>(ptr) + 8);
75
+ }
76
+
77
+ explicit FP16Vec16(const FP32Vec16& vec);
78
+
79
+ void save(void* ptr) const {
80
+ vst1q_f16(reinterpret_cast<__fp16*>(ptr), reg.val[0]);
81
+ vst1q_f16(reinterpret_cast<__fp16*>(ptr) + 8, reg.val[1]);
82
+ }
83
+
84
+ void save(void* ptr, const int elem_num) const {
85
+ int full_blocks = elem_num / 8;
86
+ int remainder = elem_num % 8;
87
+
88
+ if (full_blocks > 0) {
89
+ vst1q_f16(reinterpret_cast<__fp16*>(ptr), reg.val[0]);
90
+ if (full_blocks > 1) {
91
+ vst1q_f16(reinterpret_cast<__fp16*>(ptr) + 8, reg.val[1]);
92
+ }
93
}
83
-
84
- void save(void *ptr, const int elem_num) const {
85
- int full_blocks = elem_num / 8;
86
- int remainder = elem_num % 8;
87
-
88
- if (full_blocks > 0) {
89
- vst1q_f16(reinterpret_cast<__fp16*>(ptr), reg.val[0]);
90
- if (full_blocks > 1) {
91
- vst1q_f16(reinterpret_cast<__fp16*>(ptr) + 8, reg.val[1]);
92
- }
93
- }
94
-
95
- // Note: below is the unrolled version of the following code:
96
- //
97
- // for (int i = 0; i < remainder; ++i) {
98
- // reinterpret_cast<__fp16*>(ptr)[full_blocks * 8 + i] =
99
- // vgetq_lane_f16(temp, i);
100
- // }
101
- //
102
- // For macOS build (Clang), the arm/neon intrinsics function
103
- // `vgetq_lane_f16` needs the parameter `i` to be constant at compile
104
- // time.
105
-
106
- if (remainder > 0) {
107
- float16x8_t temp = reg.val[full_blocks];
108
- __fp16* fp16_ptr = reinterpret_cast<__fp16*>(ptr);
109
- switch (remainder)
110
- {
111
- case 1:
112
- fp16_ptr[full_blocks * 8 + 0] = vgetq_lane_f16(temp, 0);
113
- break;
114
- case 2:
115
- fp16_ptr[full_blocks * 8 + 0] = vgetq_lane_f16(temp, 0);
116
- fp16_ptr[full_blocks * 8 + 1] = vgetq_lane_f16(temp, 1);
117
- break;
118
- case 3:
119
- fp16_ptr[full_blocks * 8 + 0] = vgetq_lane_f16(temp, 0);
120
- fp16_ptr[full_blocks * 8 + 1] = vgetq_lane_f16(temp, 1);
121
- fp16_ptr[full_blocks * 8 + 2] = vgetq_lane_f16(temp, 2);
122
- break;
123
- case 4:
124
- fp16_ptr[full_blocks * 8 + 0] = vgetq_lane_f16(temp, 0);
125
- fp16_ptr[full_blocks * 8 + 1] = vgetq_lane_f16(temp, 1);
126
- fp16_ptr[full_blocks * 8 + 2] = vgetq_lane_f16(temp, 2);
127
- fp16_ptr[full_blocks * 8 + 3] = vgetq_lane_f16(temp, 3);
128
- break;
129
- case 5:
130
- fp16_ptr[full_blocks * 8 + 0] = vgetq_lane_f16(temp, 0);
131
- fp16_ptr[full_blocks * 8 + 1] = vgetq_lane_f16(temp, 1);
132
- fp16_ptr[full_blocks * 8 + 2] = vgetq_lane_f16(temp, 2);
133
- fp16_ptr[full_blocks * 8 + 3] = vgetq_lane_f16(temp, 3);
134
- fp16_ptr[full_blocks * 8 + 4] = vgetq_lane_f16(temp, 4);
135
- break;
136
- case 6:
137
- fp16_ptr[full_blocks * 8 + 0] = vgetq_lane_f16(temp, 0);
138
- fp16_ptr[full_blocks * 8 + 1] = vgetq_lane_f16(temp, 1);
139
- fp16_ptr[full_blocks * 8 + 2] = vgetq_lane_f16(temp, 2);
140
- fp16_ptr[full_blocks * 8 + 3] = vgetq_lane_f16(temp, 3);
141
- fp16_ptr[full_blocks * 8 + 4] = vgetq_lane_f16(temp, 4);
142
- fp16_ptr[full_blocks * 8 + 5] = vgetq_lane_f16(temp, 5);
143
- break;
144
- case 7:
145
- fp16_ptr[full_blocks * 8 + 0] = vgetq_lane_f16(temp, 0);
146
- fp16_ptr[full_blocks * 8 + 1] = vgetq_lane_f16(temp, 1);
147
- fp16_ptr[full_blocks * 8 + 2] = vgetq_lane_f16(temp, 2);
148
- fp16_ptr[full_blocks * 8 + 3] = vgetq_lane_f16(temp, 3);
149
- fp16_ptr[full_blocks * 8 + 4] = vgetq_lane_f16(temp, 4);
150
- fp16_ptr[full_blocks * 8 + 5] = vgetq_lane_f16(temp, 5);
151
- fp16_ptr[full_blocks * 8 + 6] = vgetq_lane_f16(temp, 6);
152
- break;
153
-
154
- default:
155
- break;
156
- }
157
- }
94
+
95
+ // Note: below is the unrolled version of the following code:
96
+ //
97
+ // for (int i = 0; i < remainder; ++i) {
98
+ // reinterpret_cast<__fp16*>(ptr)[full_blocks * 8 + i] =
99
+ // vgetq_lane_f16(temp, i);
100
+ // }
101
+ //
102
+ // For macOS build (Clang), the arm/neon intrinsics function
103
+ // `vgetq_lane_f16` needs the parameter `i` to be constant at compile
104
+ // time.
105
+
106
+ if (remainder > 0) {
107
+ float16x8_t temp = reg.val[full_blocks];
108
+ __fp16* fp16_ptr = reinterpret_cast<__fp16*>(ptr);
109
+ switch (remainder) {
110
+ case 1:
111
+ fp16_ptr[full_blocks * 8 + 0] = vgetq_lane_f16(temp, 0);
112
+ break;
113
+ case 2:
114
+ fp16_ptr[full_blocks * 8 + 0] = vgetq_lane_f16(temp, 0);
115
+ fp16_ptr[full_blocks * 8 + 1] = vgetq_lane_f16(temp, 1);
116
+ break;
117
+ case 3:
118
+ fp16_ptr[full_blocks * 8 + 0] = vgetq_lane_f16(temp, 0);
119
+ fp16_ptr[full_blocks * 8 + 1] = vgetq_lane_f16(temp, 1);
120
+ fp16_ptr[full_blocks * 8 + 2] = vgetq_lane_f16(temp, 2);
121
+ break;
122
+ case 4:
123
+ fp16_ptr[full_blocks * 8 + 0] = vgetq_lane_f16(temp, 0);
124
+ fp16_ptr[full_blocks * 8 + 1] = vgetq_lane_f16(temp, 1);
125
+ fp16_ptr[full_blocks * 8 + 2] = vgetq_lane_f16(temp, 2);
126
+ fp16_ptr[full_blocks * 8 + 3] = vgetq_lane_f16(temp, 3);
127
+ break;
128
+ case 5:
129
+ fp16_ptr[full_blocks * 8 + 0] = vgetq_lane_f16(temp, 0);
130
+ fp16_ptr[full_blocks * 8 + 1] = vgetq_lane_f16(temp, 1);
131
+ fp16_ptr[full_blocks * 8 + 2] = vgetq_lane_f16(temp, 2);
132
+ fp16_ptr[full_blocks * 8 + 3] = vgetq_lane_f16(temp, 3);
133
+ fp16_ptr[full_blocks * 8 + 4] = vgetq_lane_f16(temp, 4);
134
+ break;
135
+ case 6:
136
+ fp16_ptr[full_blocks * 8 + 0] = vgetq_lane_f16(temp, 0);
137
+ fp16_ptr[full_blocks * 8 + 1] = vgetq_lane_f16(temp, 1);
138
+ fp16_ptr[full_blocks * 8 + 2] = vgetq_lane_f16(temp, 2);
139
+ fp16_ptr[full_blocks * 8 + 3] = vgetq_lane_f16(temp, 3);
140
+ fp16_ptr[full_blocks * 8 + 4] = vgetq_lane_f16(temp, 4);
141
+ fp16_ptr[full_blocks * 8 + 5] = vgetq_lane_f16(temp, 5);
142
+ break;
143
+ case 7:
144
+ fp16_ptr[full_blocks * 8 + 0] = vgetq_lane_f16(temp, 0);
145
+ fp16_ptr[full_blocks * 8 + 1] = vgetq_lane_f16(temp, 1);
146
+ fp16_ptr[full_blocks * 8 + 2] = vgetq_lane_f16(temp, 2);
147
+ fp16_ptr[full_blocks * 8 + 3] = vgetq_lane_f16(temp, 3);
148
+ fp16_ptr[full_blocks * 8 + 4] = vgetq_lane_f16(temp, 4);
149
+ fp16_ptr[full_blocks * 8 + 5] = vgetq_lane_f16(temp, 5);
150
+ fp16_ptr[full_blocks * 8 + 6] = vgetq_lane_f16(temp, 6);
151
+ break;
152
+
153
+ default:
154
+ break;
155
+ }
156
}
157
+ }
158
};
159
161
-
160
#ifdef ARM_BF16_SUPPORT
161
struct BF16Vec8 : public Vec<BF16Vec8> {
162
constexpr static int VEC_ELEM_NUM = 8;
163
164
bfloat16x8_t reg;
165
168
- explicit BF16Vec8(const void *ptr)
169
- : reg(*reinterpret_cast<const bfloat16x8_t *>(ptr)) {};
166
+ explicit BF16Vec8(const void* ptr)
167
+ : reg(*reinterpret_cast<const bfloat16x8_t*>(ptr)) {};
168
169
explicit BF16Vec8(bfloat16x8_t data) : reg(data) {};
170
173
- explicit BF16Vec8(const FP32Vec8 &);
171
+ explicit BF16Vec8(const FP32Vec8&);
172
175
- explicit BF16Vec8(float32x4x2_t v) : reg(vcvtq_high_bf16_f32(vcvtq_low_bf16_f32(v.val[0]), v.val[1])) {};
173
+ explicit BF16Vec8(float32x4x2_t v)
174
+ : reg(vcvtq_high_bf16_f32(vcvtq_low_bf16_f32(v.val[0]), v.val[1])) {};
175
177
- void save(void *ptr) const { *reinterpret_cast<bfloat16x8_t *>(ptr) = reg; }
176
+ void save(void* ptr) const { *reinterpret_cast<bfloat16x8_t*>(ptr) = reg; }
177
};
178
179
struct BF16Vec16 : public Vec<BF16Vec16> {
@@ -182,19 +181,18 @@ struct BF16Vec16 : public Vec<BF16Vec16> {
181
182
bfloat16x8x2_t reg;
183
185
- explicit BF16Vec16(const void *ptr)
186
- : reg(*reinterpret_cast<const bfloat16x8x2_t *>(ptr)) {};
184
+ explicit BF16Vec16(const void* ptr)
185
+ : reg(*reinterpret_cast<const bfloat16x8x2_t*>(ptr)) {};
186
187
explicit BF16Vec16(bfloat16x8x2_t data) : reg(data) {};
188
190
- explicit BF16Vec16(const FP32Vec16 &);
189
+ explicit BF16Vec16(const FP32Vec16&);
190
192
- explicit BF16Vec16(float32x4x4_t v) : reg({
193
- vcvtq_high_bf16_f32(vcvtq_low_bf16_f32(v.val[0]), v.val[1]),
194
- vcvtq_high_bf16_f32(vcvtq_low_bf16_f32(v.val[2]), v.val[3])
195
- }){};
191
+ explicit BF16Vec16(float32x4x4_t v)
192
+ : reg({vcvtq_high_bf16_f32(vcvtq_low_bf16_f32(v.val[0]), v.val[1]),
193
+ vcvtq_high_bf16_f32(vcvtq_low_bf16_f32(v.val[2]), v.val[3])}) {};
194
197
- void save(void *ptr) const { *reinterpret_cast<bfloat16x8x2_t *>(ptr) = reg; };
195
+ void save(void* ptr) const { *reinterpret_cast<bfloat16x8x2_t*>(ptr) = reg; };
196
};
197
198
struct BF16Vec32 : public Vec<BF16Vec32> {
@@ -202,19 +200,15 @@ struct BF16Vec32 : public Vec<BF16Vec32> {
200
201
bfloat16x8x4_t reg;
202
205
- explicit BF16Vec32(const void *ptr)
206
- : reg(*reinterpret_cast<const bfloat16x8x4_t *>(ptr)) {};
203
+ explicit BF16Vec32(const void* ptr)
204
+ : reg(*reinterpret_cast<const bfloat16x8x4_t*>(ptr)) {};
205
206
explicit BF16Vec32(bfloat16x8x4_t data) : reg(data) {};
207
210
- explicit BF16Vec32(const BF16Vec8 &vec8_data) : reg({
211
- vec8_data.reg,
212
- vec8_data.reg,
213
- vec8_data.reg,
214
- vec8_data.reg
215
- }) {};
208
+ explicit BF16Vec32(const BF16Vec8& vec8_data)
209
+ : reg({vec8_data.reg, vec8_data.reg, vec8_data.reg, vec8_data.reg}) {};
210
217
- void save(void *ptr) const { *reinterpret_cast<bfloat16x8x4_t *>(ptr) = reg; };
211
+ void save(void* ptr) const { *reinterpret_cast<bfloat16x8x4_t*>(ptr) = reg; };
212
};
213
#endif
214
@@ -232,11 +226,11 @@ struct FP32Vec4 : public Vec<FP32Vec4> {
226
227
explicit FP32Vec4() : reg(vdupq_n_f32(0.0f)) {};
228
235
- explicit FP32Vec4(const float *ptr) : reg(vld1q_f32(ptr)) {};
229
+ explicit FP32Vec4(const float* ptr) : reg(vld1q_f32(ptr)) {};
230
231
explicit FP32Vec4(float32x4_t data) : reg(data) {};
232
239
- explicit FP32Vec4(const FP32Vec4 &data) : reg(data.reg) {};
233
+ explicit FP32Vec4(const FP32Vec4& data) : reg(data.reg) {};
234
};
235
236
struct FP32Vec8 : public Vec<FP32Vec8> {
@@ -252,32 +246,37 @@ struct FP32Vec8 : public Vec<FP32Vec8> {
246
247
explicit FP32Vec8() : reg({vmovq_n_f32(0.0), vmovq_n_f32(0.0)}) {};
248
255
- explicit FP32Vec8(const float *ptr) : reg({vld1q_f32(ptr), vld1q_f32(ptr + 4)}) {};
249
+ explicit FP32Vec8(const float* ptr)
250
+ : reg({vld1q_f32(ptr), vld1q_f32(ptr + 4)}) {};
251
252
explicit FP32Vec8(float32x4x2_t data) : reg(data) {};
253
259
- explicit FP32Vec8(const FP32Vec8 &data) : reg(data.reg) {};
254
+ explicit FP32Vec8(const FP32Vec8& data) : reg(data.reg) {};
255
261
- explicit FP32Vec8(const FP16Vec8 &v) {
262
- reg.val[0] = vcvt_f32_f16(vget_low_f16(v.reg));
263
- reg.val[1] = vcvt_f32_f16(vget_high_f16(v.reg));
264
- };
256
+ explicit FP32Vec8(const FP16Vec8& v) {
257
+ reg.val[0] = vcvt_f32_f16(vget_low_f16(v.reg));
258
+ reg.val[1] = vcvt_f32_f16(vget_high_f16(v.reg));
259
+ };
260
266
- explicit FP32Vec8(float16x8_t v) : reg({vcvt_f32_f16(vget_low_f16(v)), vcvt_f32_f16(vget_high_f16(v))}) {};
261
+ explicit FP32Vec8(float16x8_t v)
262
+ : reg({vcvt_f32_f16(vget_low_f16(v)), vcvt_f32_f16(vget_high_f16(v))}) {};
263
268
- #ifdef ARM_BF16_SUPPORT
264
+#ifdef ARM_BF16_SUPPORT
265
270
- explicit FP32Vec8(bfloat16x8_t v) : reg({vcvtq_low_f32_bf16(v), vcvtq_high_f32_bf16(v)}) {};
266
+ explicit FP32Vec8(bfloat16x8_t v)
267
+ : reg({vcvtq_low_f32_bf16(v), vcvtq_high_f32_bf16(v)}) {};
268
272
- explicit FP32Vec8(const BF16Vec8 &v) : reg({vcvtq_low_f32_bf16(v.reg), vcvtq_high_f32_bf16(v.reg)}) {};
269
+ explicit FP32Vec8(const BF16Vec8& v)
270
+ : reg({vcvtq_low_f32_bf16(v.reg), vcvtq_high_f32_bf16(v.reg)}) {};
271
274
- #endif
272
+#endif
273
274
float reduce_sum() const {
275
AliasReg ar;
276
ar.reg = reg;
277
float answer = 0;
280
- unroll_loop<int, VEC_ELEM_NUM>([&answer, &ar](int i) { answer += ar.values[i]; });
278
+ unroll_loop<int, VEC_ELEM_NUM>(
279
+ [&answer, &ar](int i) { answer += ar.values[i]; });
280
281
return answer;
282
}
@@ -324,10 +323,14 @@ struct FP32Vec8 : public Vec<FP32Vec8> {
323
AliasReg ar;
324
ar.reg = reg;
325
327
- float32x2_t er_vec0 = {static_cast<float32_t>(erf(ar.values[0])), static_cast<float32_t>(erf(ar.values[1]))};
328
- float32x2_t er_vec1 = {static_cast<float32_t>(erf(ar.values[2])), static_cast<float32_t>(erf(ar.values[3]))};
329
- float32x2_t er_vec2 = {static_cast<float32_t>(erf(ar.values[4])), static_cast<float32_t>(erf(ar.values[5]))};
330
- float32x2_t er_vec3 = {static_cast<float32_t>(erf(ar.values[6])), static_cast<float32_t>(erf(ar.values[7]))};
326
+ float32x2_t er_vec0 = {static_cast<float32_t>(erf(ar.values[0])),
327
+ static_cast<float32_t>(erf(ar.values[1]))};
328
+ float32x2_t er_vec1 = {static_cast<float32_t>(erf(ar.values[2])),
329
+ static_cast<float32_t>(erf(ar.values[3]))};
330
+ float32x2_t er_vec2 = {static_cast<float32_t>(erf(ar.values[4])),
331
+ static_cast<float32_t>(erf(ar.values[5]))};
332
+ float32x2_t er_vec3 = {static_cast<float32_t>(erf(ar.values[6])),
333
+ static_cast<float32_t>(erf(ar.values[7]))};
334
335
float32x4_t result0 = vcombine_f32(er_vec0, er_vec1);
336
float32x4_t result1 = vcombine_f32(er_vec2, er_vec3);
@@ -337,25 +340,29 @@ struct FP32Vec8 : public Vec<FP32Vec8> {
340
result.val[1] = result1;
341
342
return FP32Vec8(result);
340
- }
343
+ }
344
342
- FP32Vec8 operator*(const FP32Vec8 &b) const {
343
- return FP32Vec8(float32x4x2_t({vmulq_f32(reg.val[0], b.reg.val[0]), vmulq_f32(reg.val[1], b.reg.val[1])}));
345
+ FP32Vec8 operator*(const FP32Vec8& b) const {
346
+ return FP32Vec8(float32x4x2_t({vmulq_f32(reg.val[0], b.reg.val[0]),
347
+ vmulq_f32(reg.val[1], b.reg.val[1])}));
348
}
349
346
- FP32Vec8 operator+(const FP32Vec8 &b) const {
347
- return FP32Vec8(float32x4x2_t({vaddq_f32(reg.val[0], b.reg.val[0]), vaddq_f32(reg.val[1], b.reg.val[1])}));
350
+ FP32Vec8 operator+(const FP32Vec8& b) const {
351
+ return FP32Vec8(float32x4x2_t({vaddq_f32(reg.val[0], b.reg.val[0]),
352
+ vaddq_f32(reg.val[1], b.reg.val[1])}));
353
}
354
350
- FP32Vec8 operator-(const FP32Vec8 &b) const {
351
- return FP32Vec8(float32x4x2_t({vsubq_f32(reg.val[0], b.reg.val[0]), vsubq_f32(reg.val[1], b.reg.val[1])}));
355
+ FP32Vec8 operator-(const FP32Vec8& b) const {
356
+ return FP32Vec8(float32x4x2_t({vsubq_f32(reg.val[0], b.reg.val[0]),
357
+ vsubq_f32(reg.val[1], b.reg.val[1])}));
358
}
359
354
- FP32Vec8 operator/(const FP32Vec8 &b) const {
355
- return FP32Vec8(float32x4x2_t({vdivq_f32(reg.val[0], b.reg.val[0]), vdivq_f32(reg.val[1], b.reg.val[1])}));
360
+ FP32Vec8 operator/(const FP32Vec8& b) const {
361
+ return FP32Vec8(float32x4x2_t({vdivq_f32(reg.val[0], b.reg.val[0]),
362
+ vdivq_f32(reg.val[1], b.reg.val[1])}));
363
}
364
358
- void save(float *ptr) const {
365
+ void save(float* ptr) const {
366
vst1q_f32(ptr, reg.val[0]);
367
vst1q_f32(ptr + 4, reg.val[1]);
368
}
@@ -370,103 +377,100 @@ struct FP32Vec16 : public Vec<FP32Vec16> {
377
378
float32x4x4_t reg;
379
373
- explicit FP32Vec16(float v) : reg({vmovq_n_f32(v), vmovq_n_f32(v), vmovq_n_f32(v), vmovq_n_f32(v)}) {}
380
+ explicit FP32Vec16(float v)
381
+ : reg({vmovq_n_f32(v), vmovq_n_f32(v), vmovq_n_f32(v), vmovq_n_f32(v)}) {}
382
375
- explicit FP32Vec16() : reg({vmovq_n_f32(0.0), vmovq_n_f32(0.0), vmovq_n_f32(0.0), vmovq_n_f32(0.0)}) {}
383
+ explicit FP32Vec16()
384
+ : reg({vmovq_n_f32(0.0), vmovq_n_f32(0.0), vmovq_n_f32(0.0),
385
+ vmovq_n_f32(0.0)}) {}
386
377
- explicit FP32Vec16(const float *ptr) : reg({vld1q_f32(ptr), vld1q_f32(ptr + 4), vld1q_f32(ptr + 8), vld1q_f32(ptr + 12)}) {}
387
+ explicit FP32Vec16(const float* ptr)
388
+ : reg({vld1q_f32(ptr), vld1q_f32(ptr + 4), vld1q_f32(ptr + 8),
389
+ vld1q_f32(ptr + 12)}) {}
390
391
explicit FP32Vec16(float32x4x4_t data) : reg(data) {}
392
381
- explicit FP32Vec16(const FP32Vec8 &data) {
382
- reg.val[0] = data.reg.val[0];
383
- reg.val[1] = data.reg.val[1];
384
- reg.val[2] = data.reg.val[0];
385
- reg.val[3] = data.reg.val[1];
393
+ explicit FP32Vec16(const FP32Vec8& data) {
394
+ reg.val[0] = data.reg.val[0];
395
+ reg.val[1] = data.reg.val[1];
396
+ reg.val[2] = data.reg.val[0];
397
+ reg.val[3] = data.reg.val[1];
398
}
399
388
- explicit FP32Vec16(const FP32Vec16 &data) : reg(data.reg) {}
400
+ explicit FP32Vec16(const FP32Vec16& data) : reg(data.reg) {}
401
390
- explicit FP32Vec16(const FP16Vec8 &v) : FP32Vec16(FP32Vec8(v.reg)) {}
402
+ explicit FP32Vec16(const FP16Vec8& v) : FP32Vec16(FP32Vec8(v.reg)) {}
403
392
- #ifdef ARM_BF16_SUPPORT
393
- explicit FP32Vec16(bfloat16x8x2_t v) : reg({
394
- vcvtq_low_f32_bf16(v.val[0]),
395
- vcvtq_high_f32_bf16(v.val[0]),
396
- vcvtq_low_f32_bf16(v.val[1]),
397
- vcvtq_high_f32_bf16(v.val[1])
398
- }) {};
399
- #endif
404
+#ifdef ARM_BF16_SUPPORT
405
+ explicit FP32Vec16(bfloat16x8x2_t v)
406
+ : reg({vcvtq_low_f32_bf16(v.val[0]), vcvtq_high_f32_bf16(v.val[0]),
407
+ vcvtq_low_f32_bf16(v.val[1]), vcvtq_high_f32_bf16(v.val[1])}) {};
408
+#endif
409
401
- explicit FP32Vec16(const FP32Vec4 &data) {
410
+ explicit FP32Vec16(const FP32Vec4& data) {
411
reg.val[0] = data.reg;
412
reg.val[1] = data.reg;
413
reg.val[2] = data.reg;
414
reg.val[3] = data.reg;
415
};
416
408
- #ifdef ARM_BF16_SUPPORT
409
- explicit FP32Vec16(const BF16Vec16 &v) : reg({
410
- vcvtq_low_f32_bf16(v.reg.val[0]),
411
- vcvtq_high_f32_bf16(v.reg.val[0]),
412
- vcvtq_low_f32_bf16(v.reg.val[1]),
413
- vcvtq_high_f32_bf16(v.reg.val[1])
414
- }) {};
415
-
416
- explicit FP32Vec16(const BF16Vec8 &v) : FP32Vec16(FP32Vec8(v)) {};
417
- #endif
418
-
419
- explicit FP32Vec16(const FP16Vec16 &v) {
420
- reg.val[0] = vcvt_f32_f16(vget_low_f16(v.reg.val[0]));
421
- reg.val[1] = vcvt_f32_f16(vget_high_f16(v.reg.val[0]));
422
- reg.val[2] = vcvt_f32_f16(vget_low_f16(v.reg.val[1]));
423
- reg.val[3] = vcvt_f32_f16(vget_high_f16(v.reg.val[1]));
417
+#ifdef ARM_BF16_SUPPORT
418
+ explicit FP32Vec16(const BF16Vec16& v)
419
+ : reg({vcvtq_low_f32_bf16(v.reg.val[0]),
420
+ vcvtq_high_f32_bf16(v.reg.val[0]),
421
+ vcvtq_low_f32_bf16(v.reg.val[1]),
422
+ vcvtq_high_f32_bf16(v.reg.val[1])}) {};
423
+
424
+ explicit FP32Vec16(const BF16Vec8& v) : FP32Vec16(FP32Vec8(v)) {};
425
+#endif
426
+
427
+ explicit FP32Vec16(const FP16Vec16& v) {
428
+ reg.val[0] = vcvt_f32_f16(vget_low_f16(v.reg.val[0]));
429
+ reg.val[1] = vcvt_f32_f16(vget_high_f16(v.reg.val[0]));
430
+ reg.val[2] = vcvt_f32_f16(vget_low_f16(v.reg.val[1]));
431
+ reg.val[3] = vcvt_f32_f16(vget_high_f16(v.reg.val[1]));
432
};
433
426
- FP32Vec16 operator+(const FP32Vec16 &b) const {
427
- return FP32Vec16(float32x4x4_t({
428
- vaddq_f32(reg.val[0], b.reg.val[0]),
429
- vaddq_f32(reg.val[1], b.reg.val[1]),
430
- vaddq_f32(reg.val[2], b.reg.val[2]),
431
- vaddq_f32(reg.val[3], b.reg.val[3])}));
434
+ FP32Vec16 operator+(const FP32Vec16& b) const {
435
+ return FP32Vec16(float32x4x4_t({vaddq_f32(reg.val[0], b.reg.val[0]),
436
+ vaddq_f32(reg.val[1], b.reg.val[1]),
437
+ vaddq_f32(reg.val[2], b.reg.val[2]),
438
+ vaddq_f32(reg.val[3], b.reg.val[3])}));
439
};
440
434
- FP32Vec16 operator*(const FP32Vec16 &b) const {
435
- return FP32Vec16(float32x4x4_t({
436
- vmulq_f32(reg.val[0], b.reg.val[0]),
437
- vmulq_f32(reg.val[1], b.reg.val[1]),
438
- vmulq_f32(reg.val[2], b.reg.val[2]),
439
- vmulq_f32(reg.val[3], b.reg.val[3])}));
441
+ FP32Vec16 operator*(const FP32Vec16& b) const {
442
+ return FP32Vec16(float32x4x4_t({vmulq_f32(reg.val[0], b.reg.val[0]),
443
+ vmulq_f32(reg.val[1], b.reg.val[1]),
444
+ vmulq_f32(reg.val[2], b.reg.val[2]),
445
+ vmulq_f32(reg.val[3], b.reg.val[3])}));
446
};
447
442
- FP32Vec16 operator-(const FP32Vec16 &b) const {
443
- return FP32Vec16(float32x4x4_t({
444
- vsubq_f32(reg.val[0], b.reg.val[0]),
445
- vsubq_f32(reg.val[1], b.reg.val[1]),
446
- vsubq_f32(reg.val[2], b.reg.val[2]),
447
- vsubq_f32(reg.val[3], b.reg.val[3])
448
- }));
448
+ FP32Vec16 operator-(const FP32Vec16& b) const {
449
+ return FP32Vec16(float32x4x4_t({vsubq_f32(reg.val[0], b.reg.val[0]),
450
+ vsubq_f32(reg.val[1], b.reg.val[1]),
451
+ vsubq_f32(reg.val[2], b.reg.val[2]),
452
+ vsubq_f32(reg.val[3], b.reg.val[3])}));
453
};
454
451
- FP32Vec16 operator/(const FP32Vec16 &b) const {
452
- return FP32Vec16(float32x4x4_t({
453
- vdivq_f32(reg.val[0], b.reg.val[0]),
454
- vdivq_f32(reg.val[1], b.reg.val[1]),
455
- vdivq_f32(reg.val[2], b.reg.val[2]),
456
- vdivq_f32(reg.val[3], b.reg.val[3])
457
- }));
455
+ FP32Vec16 operator/(const FP32Vec16& b) const {
456
+ return FP32Vec16(float32x4x4_t({vdivq_f32(reg.val[0], b.reg.val[0]),
457
+ vdivq_f32(reg.val[1], b.reg.val[1]),
458
+ vdivq_f32(reg.val[2], b.reg.val[2]),
459
+ vdivq_f32(reg.val[3], b.reg.val[3])}));
460
};
461
462
float reduce_sum() const {
463
AliasReg ar;
464
ar.reg = reg;
465
float answer = 0;
464
- unroll_loop<int, VEC_ELEM_NUM>([&answer, &ar](int i) { answer += ar.values[i]; });
466
+ unroll_loop<int, VEC_ELEM_NUM>(
467
+ [&answer, &ar](int i) { answer += ar.values[i]; });
468
469
return answer;
470
};
471
469
- template <int group_size> float reduce_sub_sum(int idx) {
472
+ template <int group_size>
473
+ float reduce_sub_sum(int idx) {
474
static_assert(VEC_ELEM_NUM % group_size == 0);
475
476
AliasReg ar;
@@ -479,7 +483,7 @@ struct FP32Vec16 : public Vec<FP32Vec16> {
483
return answer;
484
};
485
482
- void save(float *ptr) const {
486
+ void save(float* ptr) const {
487
vst1q_f32(ptr, reg.val[0]);
488
vst1q_f32(ptr + 4, reg.val[1]);
489
vst1q_f32(ptr + 8, reg.val[2]);
@@ -487,43 +491,59 @@ struct FP32Vec16 : public Vec<FP32Vec16> {
491
};
492
};
493
490
-template <typename T> struct VecType { using vec_type = void; };
494
+template <typename T>
495
+struct VecType {
496
+ using vec_type = void;
497
+};
498
492
-template <typename T> using vec_t = typename VecType<T>::vec_type;
499
+template <typename T>
500
+using vec_t = typename VecType<T>::vec_type;
501
494
-template <> struct VecType<float> { using vec_type = FP32Vec8; };
502
+template <>
503
+struct VecType<float> {
504
+ using vec_type = FP32Vec8;
505
+};
506
496
-template <> struct VecType<c10::Half> { using vec_type = FP16Vec8; };
507
+template <>
508
+struct VecType<c10::Half> {
509
+ using vec_type = FP16Vec8;
510
+};
511
512
#ifdef ARM_BF16_SUPPORT
499
-template <> struct VecType<c10::BFloat16> { using vec_type = BF16Vec8; };
513
+template <>
514
+struct VecType<c10::BFloat16> {
515
+ using vec_type = BF16Vec8;
516
+};
517
#endif
518
502
-template <typename T> void storeFP32(float v, T *ptr) { *ptr = v; }
519
+template <typename T>
520
+void storeFP32(float v, T* ptr) {
521
+ *ptr = v;
522
+}
523
504
-template <> inline void storeFP32<c10::Half>(float v, c10::Half *ptr) {
505
- *reinterpret_cast<__fp16 *>(ptr) = v;
524
+template <>
525
+inline void storeFP32<c10::Half>(float v, c10::Half* ptr) {
526
+ *reinterpret_cast<__fp16*>(ptr) = v;
527
}
528
508
-inline FP16Vec16::FP16Vec16(const FP32Vec16 &v) {
509
- float16x4_t low_0 = vcvt_f16_f32(v.reg.val[0]);
510
- float16x4_t high_0 = vcvt_f16_f32(v.reg.val[1]);
511
- float16x4_t low_1 = vcvt_f16_f32(v.reg.val[2]);
512
- float16x4_t high_1 = vcvt_f16_f32(v.reg.val[3]);
529
+inline FP16Vec16::FP16Vec16(const FP32Vec16& v) {
530
+ float16x4_t low_0 = vcvt_f16_f32(v.reg.val[0]);
531
+ float16x4_t high_0 = vcvt_f16_f32(v.reg.val[1]);
532
+ float16x4_t low_1 = vcvt_f16_f32(v.reg.val[2]);
533
+ float16x4_t high_1 = vcvt_f16_f32(v.reg.val[3]);
534
514
- reg.val[0] = vcombine_f16(low_0, high_0);
515
- reg.val[1] = vcombine_f16(low_1, high_1);
535
+ reg.val[0] = vcombine_f16(low_0, high_0);
536
+ reg.val[1] = vcombine_f16(low_1, high_1);
537
};
538
518
-inline FP16Vec8 :: FP16Vec8(const FP32Vec8 &v) {
519
- float16x4_t lower_half = vcvt_f16_f32(v.reg.val[0]);
520
- float16x4_t upper_half = vcvt_f16_f32(v.reg.val[1]);
539
+inline FP16Vec8 ::FP16Vec8(const FP32Vec8& v) {
540
+ float16x4_t lower_half = vcvt_f16_f32(v.reg.val[0]);
541
+ float16x4_t upper_half = vcvt_f16_f32(v.reg.val[1]);
542
522
- reg = vcombine_f16(lower_half, upper_half);
543
+ reg = vcombine_f16(lower_half, upper_half);
544
};
545
525
-inline void fma(FP32Vec16 &acc, FP32Vec16 &a, FP32Vec16 &b) {
526
-
546
+inline void fma(FP32Vec16& acc, FP32Vec16& a, FP32Vec16& b) {
547
acc.reg.val[0] = vfmaq_f32(acc.reg.val[0], a.reg.val[0], b.reg.val[0]);
548
acc.reg.val[1] = vfmaq_f32(acc.reg.val[1], a.reg.val[1], b.reg.val[1]);
549
acc.reg.val[2] = vfmaq_f32(acc.reg.val[2], a.reg.val[2], b.reg.val[2]);
@@ -531,8 +551,7 @@ inline void fma(FP32Vec16 &acc, FP32Vec16 &a, FP32Vec16 &b) {
551
};
552
553
#ifdef ARM_BF16_SUPPORT
534
-inline void fma(FP32Vec16 &acc, BF16Vec32 &a, BF16Vec32 &b) {
535
-
554
+inline void fma(FP32Vec16& acc, BF16Vec32& a, BF16Vec32& b) {
555
float32x4_t a0_low = vcvt_f32_bf16(vget_low_bf16(a.reg.val[0]));
556
float32x4_t a0_high = vcvt_f32_bf16(vget_high_bf16(a.reg.val[0]));
557
float32x4_t a1_low = vcvt_f32_bf16(vget_low_bf16(a.reg.val[1]));
@@ -551,22 +570,22 @@ inline void fma(FP32Vec16 &acc, BF16Vec32 &a, BF16Vec32 &b) {
570
#endif
571
572
#ifdef ARM_BF16_SUPPORT
554
-inline BF16Vec8::BF16Vec8(const FP32Vec8 &v) : reg(vcvtq_high_bf16_f32(vcvtq_low_bf16_f32(v.reg.val[0]), v.reg.val[1])) {};
555
-
556
-inline BF16Vec16::BF16Vec16(const FP32Vec16 &v) : reg({
557
- vcvtq_high_bf16_f32(vcvtq_low_bf16_f32(v.reg.val[0]), v.reg.val[1]),
558
- vcvtq_high_bf16_f32(vcvtq_low_bf16_f32(v.reg.val[2]), v.reg.val[3])
559
- }){};
573
+inline BF16Vec8::BF16Vec8(const FP32Vec8& v)
574
+ : reg(vcvtq_high_bf16_f32(vcvtq_low_bf16_f32(v.reg.val[0]), v.reg.val[1])) {
575
+ };
576
+
577
+inline BF16Vec16::BF16Vec16(const FP32Vec16& v)
578
+ : reg({vcvtq_high_bf16_f32(vcvtq_low_bf16_f32(v.reg.val[0]), v.reg.val[1]),
579
+ vcvtq_high_bf16_f32(vcvtq_low_bf16_f32(v.reg.val[2]),
580
+ v.reg.val[3])}) {};
581
#endif
582
562
-inline void prefetch(const void *addr) {
563
- __builtin_prefetch(addr, 0, 1);
564
-};
583
+inline void prefetch(const void* addr) { __builtin_prefetch(addr, 0, 1); };
584
585
#ifdef ARM_BF16_SUPPORT
586
template <>
568
-inline void storeFP32<c10::BFloat16>(float v, c10::BFloat16 *ptr) {
569
- *reinterpret_cast<__bf16 *>(ptr) = vcvth_bf16_f32(v);
587
+inline void storeFP32<c10::BFloat16>(float v, c10::BFloat16* ptr) {
588
+ *reinterpret_cast<__bf16*>(ptr) = vcvth_bf16_f32(v);
589
};
590
#endif
572
-};
\ No newline at end of file
591
+}; // namespace vec_op
\ No newline at end of file
csrc/cpu/cpu_types_vsx.hpp
+143
-111
@@ -9,38 +9,40 @@
9
namespace vec_op {
10
11
// FIXME: FP16 is not fully supported in Torch-CPU
12
-#define VLLM_DISPATCH_CASE_FLOATING_TYPES(...) \
13
- AT_DISPATCH_CASE(at::ScalarType::Float, __VA_ARGS__) \
12
+#define VLLM_DISPATCH_CASE_FLOATING_TYPES(...) \
13
+ AT_DISPATCH_CASE(at::ScalarType::Float, __VA_ARGS__) \
14
AT_DISPATCH_CASE(at::ScalarType::BFloat16, __VA_ARGS__)
15
16
-#define VLLM_DISPATCH_FLOATING_TYPES(TYPE, NAME, ...) \
16
+#define VLLM_DISPATCH_FLOATING_TYPES(TYPE, NAME, ...) \
17
AT_DISPATCH_SWITCH(TYPE, NAME, VLLM_DISPATCH_CASE_FLOATING_TYPES(__VA_ARGS__))
18
19
#ifndef CPU_OP_GUARD
20
-#define CPU_KERNEL_GUARD_IN(NAME)
21
-#define CPU_KERNEL_GUARD_OUT(NAME)
20
+ #define CPU_KERNEL_GUARD_IN(NAME)
21
+ #define CPU_KERNEL_GUARD_OUT(NAME)
22
#else
23
-#define CPU_KERNEL_GUARD_IN(NAME) \
24
- std::cout << #NAME << " invoked." << std::endl;
25
-#define CPU_KERNEL_GUARD_OUT(NAME) std::cout << #NAME << " exit." << std::endl;
23
+ #define CPU_KERNEL_GUARD_IN(NAME) \
24
+ std::cout << #NAME << " invoked." << std::endl;
25
+ #define CPU_KERNEL_GUARD_OUT(NAME) \
26
+ std::cout << #NAME << " exit." << std::endl;
27
#endif
28
29
#define FORCE_INLINE __attribute__((always_inline)) inline
30
31
namespace {
32
template <typename T, T... indexes, typename F>
32
-constexpr void unroll_loop_item(std::integer_sequence<T, indexes...>, F &&f) {
33
+constexpr void unroll_loop_item(std::integer_sequence<T, indexes...>, F&& f) {
34
(f(std::integral_constant<T, indexes>{}), ...);
35
}
35
-}; // namespace
36
+}; // namespace
37
38
template <typename T, T count, typename F,
39
typename = std::enable_if_t<std::is_invocable_v<F, T>>>
39
-constexpr void unroll_loop(F &&f) {
40
+constexpr void unroll_loop(F&& f) {
41
unroll_loop_item(std::make_integer_sequence<T, count>{}, std::forward<F>(f));
42
}
43
43
-template <typename T> struct Vec {
44
+template <typename T>
45
+struct Vec {
46
constexpr static int get_elem_num() { return T::VEC_ELEM_NUM; }
47
};
48
@@ -68,12 +70,14 @@ struct BF16Vec8 : public Vec<BF16Vec8> {
70
71
__vector signed short reg;
72
71
- explicit BF16Vec8(const void *ptr)
72
- : reg((__vector signed short)vec_xl(0, (__vector signed short *)ptr)) {}
73
+ explicit BF16Vec8(const void* ptr)
74
+ : reg((__vector signed short)vec_xl(0, (__vector signed short*)ptr)) {}
75
74
- explicit BF16Vec8(const FP32Vec8 &);
76
+ explicit BF16Vec8(const FP32Vec8&);
77
76
- void save(void *ptr) const { *reinterpret_cast<__vector signed short *>(ptr) = reg; }
78
+ void save(void* ptr) const {
79
+ *reinterpret_cast<__vector signed short*>(ptr) = reg;
80
+ }
81
};
82
83
struct BF16Vec16 : public Vec<BF16Vec16> {
@@ -81,18 +85,18 @@ struct BF16Vec16 : public Vec<BF16Vec16> {
85
86
ss16x8x2_t reg;
87
84
- explicit BF16Vec16(const void *ptr) {
88
+ explicit BF16Vec16(const void* ptr) {
89
// Load 256 bits in two parts
86
- reg.val[0] = (__vector signed short)vec_xl(0, (signed short *)ptr);
87
- reg.val[1] = (__vector signed short)vec_xl(16, (signed short *)ptr);
90
+ reg.val[0] = (__vector signed short)vec_xl(0, (signed short*)ptr);
91
+ reg.val[1] = (__vector signed short)vec_xl(16, (signed short*)ptr);
92
}
93
90
- explicit BF16Vec16(const FP32Vec16 &);
94
+ explicit BF16Vec16(const FP32Vec16&);
95
92
- void save(void *ptr) const {
96
+ void save(void* ptr) const {
97
// Save 256 bits in two parts
94
- vec_xst(reg.val[0], 0, (signed short *)ptr);
95
- vec_xst(reg.val[1], 16, (signed short *)ptr);
98
+ vec_xst(reg.val[0], 0, (signed short*)ptr);
99
+ vec_xst(reg.val[1], 16, (signed short*)ptr);
100
}
101
};
102
@@ -102,19 +106,15 @@ struct BF16Vec32 : public Vec<BF16Vec32> {
106
constexpr static int VEC_ELEM_NUM = 32;
107
108
ss16x8x4_t reg;
105
- explicit BF16Vec32(const void *ptr)
106
- : reg(*reinterpret_cast<const ss16x8x4_t *>(ptr)) {}
109
+ explicit BF16Vec32(const void* ptr)
110
+ : reg(*reinterpret_cast<const ss16x8x4_t*>(ptr)) {}
111
112
explicit BF16Vec32(ss16x8x4_t data) : reg(data) {}
113
110
- explicit BF16Vec32(const BF16Vec8 &vec8_data) : reg({
111
- vec8_data.reg,
112
- vec8_data.reg,
113
- vec8_data.reg,
114
- vec8_data.reg
115
- }) {}
114
+ explicit BF16Vec32(const BF16Vec8& vec8_data)
115
+ : reg({vec8_data.reg, vec8_data.reg, vec8_data.reg, vec8_data.reg}) {}
116
117
- void save(void *ptr) const { *reinterpret_cast<ss16x8x4_t *>(ptr) = reg; }
117
+ void save(void* ptr) const { *reinterpret_cast<ss16x8x4_t*>(ptr) = reg; }
118
};
119
120
struct FP32Vec4 : public Vec<FP32Vec4> {
@@ -130,11 +130,11 @@ struct FP32Vec4 : public Vec<FP32Vec4> {
130
131
explicit FP32Vec4() : reg(vec_splats(0.0f)) {}
132
133
- explicit FP32Vec4(const float *ptr) : reg(vec_xl(0, ptr)) {}
133
+ explicit FP32Vec4(const float* ptr) : reg(vec_xl(0, ptr)) {}
134
135
explicit FP32Vec4(__vector float data) : reg(data) {}
136
137
- explicit FP32Vec4(const FP32Vec4 &data) : reg(data.reg) {}
137
+ explicit FP32Vec4(const FP32Vec4& data) : reg(data.reg) {}
138
};
139
140
struct FP32Vec8 : public Vec<FP32Vec8> {
@@ -156,19 +156,19 @@ struct FP32Vec8 : public Vec<FP32Vec8> {
156
reg.val[1] = vec_splats(0.0f);
157
}
158
159
- explicit FP32Vec8(const float *ptr) {
159
+ explicit FP32Vec8(const float* ptr) {
160
reg.val[0] = vec_xl(0, ptr);
161
reg.val[1] = vec_xl(16, ptr);
162
}
163
164
explicit FP32Vec8(f32x4x2_t data) : reg(data) {}
165
166
- explicit FP32Vec8(const FP32Vec8 &data) {
166
+ explicit FP32Vec8(const FP32Vec8& data) {
167
reg.val[0] = data.reg.val[0];
168
reg.val[1] = data.reg.val[1];
169
}
170
171
- explicit FP32Vec8(const BF16Vec8 &v) {
171
+ explicit FP32Vec8(const BF16Vec8& v) {
172
reg.val[0] = (__vector float)vec_mergeh(zero, v.reg);
173
reg.val[1] = (__vector float)vec_mergel(zero, v.reg);
174
}
@@ -177,7 +177,8 @@ struct FP32Vec8 : public Vec<FP32Vec8> {
177
AliasReg ar;
178
ar.reg = reg;
179
float result = 0;
180
- unroll_loop<int, VEC_ELEM_NUM>([&result, &ar](int i) { result += ar.values[i]; });
180
+ unroll_loop<int, VEC_ELEM_NUM>(
181
+ [&result, &ar](int i) { result += ar.values[i]; });
182
183
return result;
184
}
@@ -230,23 +231,27 @@ struct FP32Vec8 : public Vec<FP32Vec8> {
231
return FP32Vec8(f32x4x2_t({ret.val[0], ret.val[1]}));
232
}
233
233
- FP32Vec8 operator*(const FP32Vec8 &b) const {
234
- return FP32Vec8({vec_mul(reg.val[0], b.reg.val[0]), vec_mul(reg.val[1], b.reg.val[1])});
234
+ FP32Vec8 operator*(const FP32Vec8& b) const {
235
+ return FP32Vec8(
236
+ {vec_mul(reg.val[0], b.reg.val[0]), vec_mul(reg.val[1], b.reg.val[1])});
237
}
238
237
- FP32Vec8 operator+(const FP32Vec8 &b) const {
238
- return FP32Vec8({vec_add(reg.val[0], b.reg.val[0]), vec_add(reg.val[1], b.reg.val[1])});
239
+ FP32Vec8 operator+(const FP32Vec8& b) const {
240
+ return FP32Vec8(
241
+ {vec_add(reg.val[0], b.reg.val[0]), vec_add(reg.val[1], b.reg.val[1])});
242
}
243
241
- FP32Vec8 operator-(const FP32Vec8 &b) const {
242
- return FP32Vec8({vec_sub(reg.val[0], b.reg.val[0]), vec_sub(reg.val[1], b.reg.val[1])});
244
+ FP32Vec8 operator-(const FP32Vec8& b) const {
245
+ return FP32Vec8(
246
+ {vec_sub(reg.val[0], b.reg.val[0]), vec_sub(reg.val[1], b.reg.val[1])});
247
}
248
245
- FP32Vec8 operator/(const FP32Vec8 &b) const {
246
- return FP32Vec8({vec_div(reg.val[0], b.reg.val[0]), vec_div(reg.val[1], b.reg.val[1])});
249
+ FP32Vec8 operator/(const FP32Vec8& b) const {
250
+ return FP32Vec8(
251
+ {vec_div(reg.val[0], b.reg.val[0]), vec_div(reg.val[1], b.reg.val[1])});
252
}
253
249
- void save(float *ptr) const {
254
+ void save(float* ptr) const {
255
vec_xst(reg.val[0], 0, ptr);
256
vec_xst(reg.val[1], 16, ptr);
257
}
@@ -275,7 +280,7 @@ struct FP32Vec16 : public Vec<FP32Vec16> {
280
reg.val[3] = vec_splats(0.0f);
281
}
282
278
- explicit FP32Vec16(const float *ptr) {
283
+ explicit FP32Vec16(const float* ptr) {
284
reg.val[0] = vec_xl(0, ptr);
285
reg.val[1] = vec_xl(16, ptr);
286
reg.val[2] = vec_xl(32, ptr);
@@ -284,78 +289,76 @@ struct FP32Vec16 : public Vec<FP32Vec16> {
289
290
explicit FP32Vec16(f32x4x4_t data) : reg(data) {}
291
287
- explicit FP32Vec16(const FP32Vec16 &data) {
292
+ explicit FP32Vec16(const FP32Vec16& data) {
293
reg.val[0] = data.reg.val[0];
294
reg.val[1] = data.reg.val[1];
295
reg.val[2] = data.reg.val[2];
296
reg.val[3] = data.reg.val[3];
297
}
298
294
- explicit FP32Vec16(const FP32Vec4 &data) {
299
+ explicit FP32Vec16(const FP32Vec4& data) {
300
reg.val[0] = data.reg;
301
reg.val[1] = data.reg;
302
reg.val[2] = data.reg;
303
reg.val[3] = data.reg;
304
}
305
301
- explicit FP32Vec16(const FP32Vec8 &data) {
306
+ explicit FP32Vec16(const FP32Vec8& data) {
307
reg.val[0] = data.reg.val[0];
308
reg.val[1] = data.reg.val[1];
309
reg.val[2] = data.reg.val[0];
310
reg.val[3] = data.reg.val[1];
311
}
312
308
- explicit FP32Vec16(const BF16Vec16 &v) {
313
+ explicit FP32Vec16(const BF16Vec16& v) {
314
reg.val[0] = (__vector float)vec_mergeh(zero, v.reg.val[0]);
315
reg.val[1] = (__vector float)vec_mergel(zero, v.reg.val[0]);
316
reg.val[2] = (__vector float)vec_mergeh(zero, v.reg.val[1]);
317
reg.val[3] = (__vector float)vec_mergel(zero, v.reg.val[1]);
318
}
319
315
- explicit FP32Vec16(const BF16Vec8 &v) : FP32Vec16(FP32Vec8(v)) {}
320
+ explicit FP32Vec16(const BF16Vec8& v) : FP32Vec16(FP32Vec8(v)) {}
321
317
- FP32Vec16 operator*(const FP32Vec16 &b) const {
318
- return FP32Vec16(f32x4x4_t({
319
- vec_mul(reg.val[0], b.reg.val[0]),
320
- vec_mul(reg.val[1], b.reg.val[1]),
321
- vec_mul(reg.val[2], b.reg.val[2]),
322
- vec_mul(reg.val[3], b.reg.val[3])}));
322
+ FP32Vec16 operator*(const FP32Vec16& b) const {
323
+ return FP32Vec16(f32x4x4_t({vec_mul(reg.val[0], b.reg.val[0]),
324
+ vec_mul(reg.val[1], b.reg.val[1]),
325
+ vec_mul(reg.val[2], b.reg.val[2]),
326
+ vec_mul(reg.val[3], b.reg.val[3])}));
327
}
328
325
- FP32Vec16 operator+(const FP32Vec16 &b) const {
326
- return FP32Vec16(f32x4x4_t({
327
- vec_add(reg.val[0], b.reg.val[0]),
328
- vec_add(reg.val[1], b.reg.val[1]),
329
- vec_add(reg.val[2], b.reg.val[2]),
330
- vec_add(reg.val[3], b.reg.val[3])}));
329
+ FP32Vec16 operator+(const FP32Vec16& b) const {
330
+ return FP32Vec16(f32x4x4_t({vec_add(reg.val[0], b.reg.val[0]),
331
+ vec_add(reg.val[1], b.reg.val[1]),
332
+ vec_add(reg.val[2], b.reg.val[2]),
333
+ vec_add(reg.val[3], b.reg.val[3])}));
334
}
335
333
- FP32Vec16 operator-(const FP32Vec16 &b) const {
334
- return FP32Vec16(f32x4x4_t({
335
- vec_sub(reg.val[0], b.reg.val[0]),
336
- vec_sub(reg.val[1], b.reg.val[1]),
337
- vec_sub(reg.val[2], b.reg.val[2]),
338
- vec_sub(reg.val[3], b.reg.val[3])}));
336
+ FP32Vec16 operator-(const FP32Vec16& b) const {
337
+ return FP32Vec16(f32x4x4_t({vec_sub(reg.val[0], b.reg.val[0]),
338
+ vec_sub(reg.val[1], b.reg.val[1]),
339
+ vec_sub(reg.val[2], b.reg.val[2]),
340
+ vec_sub(reg.val[3], b.reg.val[3])}));
341
}
342
341
- FP32Vec16 operator/(const FP32Vec16 &b) const {
342
- return FP32Vec16(f32x4x4_t({
343
- vec_div(reg.val[0], b.reg.val[0]),
344
- vec_div(reg.val[1], b.reg.val[1]),
345
- vec_div(reg.val[2], b.reg.val[2]),
346
- vec_div(reg.val[3], b.reg.val[3])}));
343
+ FP32Vec16 operator/(const FP32Vec16& b) const {
344
+ return FP32Vec16(f32x4x4_t({vec_div(reg.val[0], b.reg.val[0]),
345
+ vec_div(reg.val[1], b.reg.val[1]),
346
+ vec_div(reg.val[2], b.reg.val[2]),
347
+ vec_div(reg.val[3], b.reg.val[3])}));
348
}
349
350
float reduce_sum() const {
351
AliasReg ar;
352
ar.reg = reg;
353
float result = 0;
353
- unroll_loop<int, VEC_ELEM_NUM>([&result, &ar](int i) { result += ar.values[i]; });
354
+ unroll_loop<int, VEC_ELEM_NUM>(
355
+ [&result, &ar](int i) { result += ar.values[i]; });
356
357
return result;
358
}
359
358
- template <int group_size> float reduce_sub_sum(int idx) {
360
+ template <int group_size>
361
+ float reduce_sub_sum(int idx) {
362
static_assert(VEC_ELEM_NUM % group_size == 0);
363
364
AliasReg ar;
@@ -368,7 +371,7 @@ struct FP32Vec16 : public Vec<FP32Vec16> {
371
return result;
372
}
373
371
- void save(float *ptr) const {
374
+ void save(float* ptr) const {
375
vec_xst(reg.val[0], 0, ptr);
376
vec_xst(reg.val[1], 16, ptr);
377
vec_xst(reg.val[2], 32, ptr);
@@ -376,43 +379,62 @@ struct FP32Vec16 : public Vec<FP32Vec16> {
379
}
380
};
381
379
-template <typename T> struct VecType { using vec_type = void; };
382
+template <typename T>
383
+struct VecType {
384
+ using vec_type = void;
385
+};
386
381
-template <typename T> using vec_t = typename VecType<T>::vec_type;
387
+template <typename T>
388
+using vec_t = typename VecType<T>::vec_type;
389
383
-template <> struct VecType<float> { using vec_type = FP32Vec8; };
390
+template <>
391
+struct VecType<float> {
392
+ using vec_type = FP32Vec8;
393
+};
394
385
-template <> struct VecType<c10::BFloat16> { using vec_type = BF16Vec8; };
395
+template <>
396
+struct VecType<c10::BFloat16> {
397
+ using vec_type = BF16Vec8;
398
+};
399
387
-template <typename T> void storeFP32(float v, T *ptr) { *ptr = v; }
400
+template <typename T>
401
+void storeFP32(float v, T* ptr) {
402
+ *ptr = v;
403
+}
404
389
-inline void fma(FP32Vec16 &acc, FP32Vec16 &a, FP32Vec16 &b) {
405
+inline void fma(FP32Vec16& acc, FP32Vec16& a, FP32Vec16& b) {
406
acc = acc + a * b;
407
}
408
393
-template <> inline void storeFP32<c10::BFloat16>(float v, c10::BFloat16 *ptr) {
394
- c10::BFloat16 __attribute__((__may_alias__)) *v_ptr =
395
- reinterpret_cast<c10::BFloat16 *>(&v);
409
+template <>
410
+inline void storeFP32<c10::BFloat16>(float v, c10::BFloat16* ptr) {
411
+ c10::BFloat16 __attribute__((__may_alias__))* v_ptr =
412
+ reinterpret_cast<c10::BFloat16*>(&v);
413
*ptr = *(v_ptr + 1);
414
}
415
416
#ifndef __VEC_CLASS_FP_NAN
400
-#define __VEC_CLASS_FP_NAN (1 << 6)
417
+ #define __VEC_CLASS_FP_NAN (1 << 6)
418
#endif
419
403
-const static __vector unsigned char omask = { 0, 1, 4, 5, 8, 9, 12, 13, 16, 17, 20, 21, 24, 25, 28, 29 };
420
+const static __vector unsigned char omask = {0, 1, 4, 5, 8, 9, 12, 13,
421
+ 16, 17, 20, 21, 24, 25, 28, 29};
422
#ifndef _ARCH_PWR10
405
-const static __vector unsigned int bias = { 0x00007fff, 0x00007fff, 0x00007fff, 0x00007fff };
406
-const static __vector unsigned int nan = { 0x7fc00000, 0x7fc00000, 0x7fc00000, 0x7fc00000 };
407
-const static __vector unsigned int sh16 = { 16, 16, 16, 16 };
408
-const static __vector unsigned int one = { 1, 1, 1, 1 };
423
+const static __vector unsigned int bias = {0x00007fff, 0x00007fff, 0x00007fff,
424
+ 0x00007fff};
425
+const static __vector unsigned int nan = {0x7fc00000, 0x7fc00000, 0x7fc00000,
426
+ 0x7fc00000};
427
+const static __vector unsigned int sh16 = {16, 16, 16, 16};
428
+const static __vector unsigned int one = {1, 1, 1, 1};
429
#endif
430
411
-inline BF16Vec8::BF16Vec8(const FP32Vec8 &v) {
431
+inline BF16Vec8::BF16Vec8(const FP32Vec8& v) {
432
#ifdef _ARCH_PWR10
433
__vector signed short ret[2];
414
- ret[0] = (__vector signed short)__builtin_vsx_xvcvspbf16((__vector unsigned char)v.reg.val[0]);
415
- ret[1] = (__vector signed short)__builtin_vsx_xvcvspbf16((__vector unsigned char)v.reg.val[1]);
434
+ ret[0] = (__vector signed short)__builtin_vsx_xvcvspbf16(
435
+ (__vector unsigned char)v.reg.val[0]);
436
+ ret[1] = (__vector signed short)__builtin_vsx_xvcvspbf16(
437
+ (__vector unsigned char)v.reg.val[1]);
438
reg = vec_perm(ret[0], ret[1], omask);
439
#elif defined(_ARCH_PWR9)
440
__vector unsigned int inp0 = (__vector unsigned int)(v.reg.val[0]);
@@ -425,8 +447,10 @@ inline BF16Vec8::BF16Vec8(const FP32Vec8 &v) {
447
__vector unsigned int rnd1 = vec_add(lsb1, bias);
448
inp0 = vec_add(inp0, rnd0);
449
inp1 = vec_add(inp1, rnd1);
428
- __vector __bool int sel0 = vec_test_data_class(v.reg.val[0], __VEC_CLASS_FP_NAN);
429
- __vector __bool int sel1 = vec_test_data_class(v.reg.val[1], __VEC_CLASS_FP_NAN);
450
+ __vector __bool int sel0 =
451
+ vec_test_data_class(v.reg.val[0], __VEC_CLASS_FP_NAN);
452
+ __vector __bool int sel1 =
453
+ vec_test_data_class(v.reg.val[1], __VEC_CLASS_FP_NAN);
454
inp0 = vec_sel(inp0, nan, sel0);
455
inp1 = vec_sel(inp1, nan, sel1);
456
inp0 = vec_sr(inp0, sh16);
@@ -435,13 +459,17 @@ inline BF16Vec8::BF16Vec8(const FP32Vec8 &v) {
459
#endif
460
}
461
438
-inline BF16Vec16::BF16Vec16(const FP32Vec16 &v) {
462
+inline BF16Vec16::BF16Vec16(const FP32Vec16& v) {
463
#ifdef _ARCH_PWR10
464
__vector signed short ret[4];
441
- ret[0] = (__vector signed short)__builtin_vsx_xvcvspbf16((__vector unsigned char)v.reg.val[0]);
442
- ret[1] = (__vector signed short)__builtin_vsx_xvcvspbf16((__vector unsigned char)v.reg.val[1]);
443
- ret[2] = (__vector signed short)__builtin_vsx_xvcvspbf16((__vector unsigned char)v.reg.val[2]);
444
- ret[3] = (__vector signed short)__builtin_vsx_xvcvspbf16((__vector unsigned char)v.reg.val[3]);
465
+ ret[0] = (__vector signed short)__builtin_vsx_xvcvspbf16(
466
+ (__vector unsigned char)v.reg.val[0]);
467
+ ret[1] = (__vector signed short)__builtin_vsx_xvcvspbf16(
468
+ (__vector unsigned char)v.reg.val[1]);
469
+ ret[2] = (__vector signed short)__builtin_vsx_xvcvspbf16(
470
+ (__vector unsigned char)v.reg.val[2]);
471
+ ret[3] = (__vector signed short)__builtin_vsx_xvcvspbf16(
472
+ (__vector unsigned char)v.reg.val[3]);
473
reg.val[0] = vec_perm(ret[0], ret[1], omask);
474
reg.val[1] = vec_perm(ret[2], ret[3], omask);
475
#elif defined(_ARCH_PWR9)
@@ -465,10 +493,14 @@ inline BF16Vec16::BF16Vec16(const FP32Vec16 &v) {
493
inp1 = vec_add(inp1, rnd1);
494
inp2 = vec_add(inp2, rnd2);
495
inp3 = vec_add(inp3, rnd3);
468
- __vector __bool int sel0 = vec_test_data_class(v.reg.val[0], __VEC_CLASS_FP_NAN);
469
- __vector __bool int sel1 = vec_test_data_class(v.reg.val[1], __VEC_CLASS_FP_NAN);
470
- __vector __bool int sel2 = vec_test_data_class(v.reg.val[2], __VEC_CLASS_FP_NAN);
471
- __vector __bool int sel3 = vec_test_data_class(v.reg.val[3], __VEC_CLASS_FP_NAN);
496
+ __vector __bool int sel0 =
497
+ vec_test_data_class(v.reg.val[0], __VEC_CLASS_FP_NAN);
498
+ __vector __bool int sel1 =
499
+ vec_test_data_class(v.reg.val[1], __VEC_CLASS_FP_NAN);
500
+ __vector __bool int sel2 =
501
+ vec_test_data_class(v.reg.val[2], __VEC_CLASS_FP_NAN);
502
+ __vector __bool int sel3 =
503
+ vec_test_data_class(v.reg.val[3], __VEC_CLASS_FP_NAN);
504
inp0 = vec_sel(inp0, nan, sel0);
505
inp1 = vec_sel(inp1, nan, sel1);
506
inp2 = vec_sel(inp2, nan, sel2);
@@ -482,10 +514,10 @@ inline BF16Vec16::BF16Vec16(const FP32Vec16 &v) {
514
#endif
515
}
516
485
-inline void prefetch(const void *addr) {
517
+inline void prefetch(const void* addr) {
518
__asm__ __volatile__("dcbt 0, %0" : : "r"(addr) : "memory");
519
}
520
489
-}; // namespace vec_op
521
+}; // namespace vec_op
522
523
#endif
csrc/cpu/cpu_types_x86.hpp
+165
-146
@@ -11,39 +11,40 @@ static_assert(false, "AVX2 must be supported for the current implementation.");
11
12
namespace vec_op {
13
14
-#define VLLM_DISPATCH_CASE_FLOATING_TYPES(...) \
15
- AT_DISPATCH_CASE(at::ScalarType::Float, __VA_ARGS__) \
16
- AT_DISPATCH_CASE(at::ScalarType::BFloat16, __VA_ARGS__) \
14
+#define VLLM_DISPATCH_CASE_FLOATING_TYPES(...) \
15
+ AT_DISPATCH_CASE(at::ScalarType::Float, __VA_ARGS__) \
16
+ AT_DISPATCH_CASE(at::ScalarType::BFloat16, __VA_ARGS__) \
17
AT_DISPATCH_CASE(at::ScalarType::Half, __VA_ARGS__)
18
19
-#define VLLM_DISPATCH_FLOATING_TYPES(TYPE, NAME, ...) \
19
+#define VLLM_DISPATCH_FLOATING_TYPES(TYPE, NAME, ...) \
20
AT_DISPATCH_SWITCH(TYPE, NAME, VLLM_DISPATCH_CASE_FLOATING_TYPES(__VA_ARGS__))
21
22
#ifndef CPU_OP_GUARD
23
-#define CPU_KERNEL_GUARD_IN(NAME)
24
-#define CPU_KERNEL_GUARD_OUT(NAME)
23
+ #define CPU_KERNEL_GUARD_IN(NAME)
24
+ #define CPU_KERNEL_GUARD_OUT(NAME)
25
#else
26
-#define CPU_KERNEL_GUARD_IN(NAME) \
27
- RECORD_FUNCTION(#NAME, c10::ArrayRef<c10::IValue>({}));
28
-#define CPU_KERNEL_GUARD_OUT(NAME)
26
+ #define CPU_KERNEL_GUARD_IN(NAME) \
27
+ RECORD_FUNCTION(#NAME, c10::ArrayRef<c10::IValue>({}));
28
+ #define CPU_KERNEL_GUARD_OUT(NAME)
29
#endif
30
31
#define FORCE_INLINE __attribute__((always_inline)) inline
32
33
namespace {
34
template <typename T, T... indexes, typename F>
35
-constexpr void unroll_loop_item(std::integer_sequence<T, indexes...>, F &&f) {
35
+constexpr void unroll_loop_item(std::integer_sequence<T, indexes...>, F&& f) {
36
(f(std::integral_constant<T, indexes>{}), ...);
37
}
38
-}; // namespace
38
+}; // namespace
39
40
template <typename T, T count, typename F,
41
typename = std::enable_if_t<std::is_invocable_v<F, T>>>
42
-constexpr void unroll_loop(F &&f) {
42
+constexpr void unroll_loop(F&& f) {
43
unroll_loop_item(std::make_integer_sequence<T, count>{}, std::forward<F>(f));
44
}
45
46
-template <typename T> struct Vec {
46
+template <typename T>
47
+struct Vec {
48
constexpr static int get_elem_num() { return T::VEC_ELEM_NUM; }
49
};
50
@@ -55,12 +56,12 @@ struct FP16Vec8 : public Vec<FP16Vec8> {
56
57
__m128i reg;
58
58
- explicit FP16Vec8(const void *ptr)
59
- : reg((__m128i)_mm_loadu_si128((__m128i *)ptr)) {}
59
+ explicit FP16Vec8(const void* ptr)
60
+ : reg((__m128i)_mm_loadu_si128((__m128i*)ptr)) {}
61
61
- explicit FP16Vec8(const FP32Vec8 &);
62
+ explicit FP16Vec8(const FP32Vec8&);
63
63
- void save(void *ptr) const { *reinterpret_cast<__m128i *>(ptr) = reg; }
64
+ void save(void* ptr) const { *reinterpret_cast<__m128i*>(ptr) = reg; }
65
};
66
67
struct FP16Vec16 : public Vec<FP16Vec16> {
@@ -68,12 +69,12 @@ struct FP16Vec16 : public Vec<FP16Vec16> {
69
70
__m256i reg;
71
71
- explicit FP16Vec16(const void *ptr)
72
- : reg((__m256i)_mm256_loadu_si256((__m256i *)ptr)) {}
72
+ explicit FP16Vec16(const void* ptr)
73
+ : reg((__m256i)_mm256_loadu_si256((__m256i*)ptr)) {}
74
74
- explicit FP16Vec16(const FP32Vec16 &);
75
+ explicit FP16Vec16(const FP32Vec16&);
76
76
- void save(void *ptr) const { *reinterpret_cast<__m256i *>(ptr) = reg; }
77
+ void save(void* ptr) const { *reinterpret_cast<__m256i*>(ptr) = reg; }
78
79
void save(void* ptr, const int elem_num) const {
80
constexpr uint32_t M = 0xFFFFFFFF;
@@ -87,12 +88,12 @@ struct BF16Vec8 : public Vec<BF16Vec8> {
88
89
__m128i reg;
90
90
- explicit BF16Vec8(const void *ptr)
91
- : reg((__m128i)_mm_loadu_si128((__m128i *)ptr)) {}
91
+ explicit BF16Vec8(const void* ptr)
92
+ : reg((__m128i)_mm_loadu_si128((__m128i*)ptr)) {}
93
93
- explicit BF16Vec8(const FP32Vec8 &);
94
+ explicit BF16Vec8(const FP32Vec8&);
95
95
- void save(void *ptr) const { *reinterpret_cast<__m128i *>(ptr) = reg; }
96
+ void save(void* ptr) const { *reinterpret_cast<__m128i*>(ptr) = reg; }
97
};
98
99
struct BF16Vec16 : public Vec<BF16Vec16> {
@@ -100,12 +101,12 @@ struct BF16Vec16 : public Vec<BF16Vec16> {
101
102
__m256i reg;
103
103
- explicit BF16Vec16(const void *ptr)
104
- : reg((__m256i)_mm256_loadu_si256((__m256i *)ptr)) {}
104
+ explicit BF16Vec16(const void* ptr)
105
+ : reg((__m256i)_mm256_loadu_si256((__m256i*)ptr)) {}
106
106
- explicit BF16Vec16(const FP32Vec16 &);
107
+ explicit BF16Vec16(const FP32Vec16&);
108
108
- void save(void *ptr) const { *reinterpret_cast<__m256i *>(ptr) = reg; }
109
+ void save(void* ptr) const { *reinterpret_cast<__m256i*>(ptr) = reg; }
110
111
void save(void* ptr, const int elem_num) const {
112
constexpr uint32_t M = 0xFFFFFFFF;
@@ -120,11 +121,11 @@ struct BF16Vec32 : public Vec<BF16Vec32> {
121
122
__m512i reg;
123
123
- explicit BF16Vec32(const void *ptr) : reg((__m512i)_mm512_loadu_si512(ptr)) {}
124
+ explicit BF16Vec32(const void* ptr) : reg((__m512i)_mm512_loadu_si512(ptr)) {}
125
126
explicit BF16Vec32(__m512i data) : reg(data) {}
127
127
- explicit BF16Vec32(BF16Vec8 &vec8_data)
128
+ explicit BF16Vec32(BF16Vec8& vec8_data)
129
: reg((__m512i)_mm512_inserti32x4(
130
_mm512_inserti32x4(_mm512_inserti32x4(_mm512_castsi128_si512(
131
(__m128i)vec8_data.reg),
@@ -132,7 +133,7 @@ struct BF16Vec32 : public Vec<BF16Vec32> {
133
(__m128i)vec8_data.reg, 2),
134
(__m128i)vec8_data.reg, 3)) {}
135
135
- void save(void *ptr) const { *reinterpret_cast<__m512i *>(ptr) = reg; }
136
+ void save(void* ptr) const { *reinterpret_cast<__m512i*>(ptr) = reg; }
137
};
138
#else
139
struct BF16Vec32 : public Vec<BF16Vec32> {
@@ -141,24 +142,24 @@ struct BF16Vec32 : public Vec<BF16Vec32> {
142
__m256i reg_low;
143
__m256i reg_high;
144
144
- explicit BF16Vec32(const void *ptr)
145
- : reg_low(_mm256_loadu_si256((__m256i const *)ptr)),
146
- reg_high(_mm256_loadu_si256((__m256i const *)ptr + 1)) {}
145
+ explicit BF16Vec32(const void* ptr)
146
+ : reg_low(_mm256_loadu_si256((__m256i const*)ptr)),
147
+ reg_high(_mm256_loadu_si256((__m256i const*)ptr + 1)) {}
148
148
- explicit BF16Vec32(__m256i low, __m256i high) : reg_low(low),
149
- reg_high(high) {}
149
+ explicit BF16Vec32(__m256i low, __m256i high)
150
+ : reg_low(low), reg_high(high) {}
151
151
- explicit BF16Vec32(BF16Vec8 &vec8_data)
152
+ explicit BF16Vec32(BF16Vec8& vec8_data)
153
: reg_low((__m256i)_mm256_inserti32x4(
153
- _mm256_castsi128_si256((__m128i)vec8_data.reg),
154
- (__m128i)vec8_data.reg, 1)),
154
+ _mm256_castsi128_si256((__m128i)vec8_data.reg),
155
+ (__m128i)vec8_data.reg, 1)),
156
reg_high((__m256i)_mm256_inserti32x4(
156
- _mm256_castsi128_si256((__m128i)vec8_data.reg),
157
- (__m128i)vec8_data.reg, 1)) {}
157
+ _mm256_castsi128_si256((__m128i)vec8_data.reg),
158
+ (__m128i)vec8_data.reg, 1)) {}
159
159
- void save(void *ptr) const {
160
- *reinterpret_cast<__m256i *>(ptr) = reg_low;
161
- *reinterpret_cast<__m256i *>((__m256i *)ptr + 1) = reg_high;
160
+ void save(void* ptr) const {
161
+ *reinterpret_cast<__m256i*>(ptr) = reg_low;
162
+ *reinterpret_cast<__m256i*>((__m256i*)ptr + 1) = reg_high;
163
}
164
};
165
#endif
@@ -176,11 +177,11 @@ struct FP32Vec4 : public Vec<FP32Vec4> {
177
178
explicit FP32Vec4() : reg(_mm_set1_ps(0.0)) {}
179
179
- explicit FP32Vec4(const float *ptr) : reg(_mm_loadu_ps(ptr)) {}
180
+ explicit FP32Vec4(const float* ptr) : reg(_mm_loadu_ps(ptr)) {}
181
182
explicit FP32Vec4(__m128 data) : reg(data) {}
183
183
- explicit FP32Vec4(const FP32Vec4 &data) : reg(data.reg) {}
184
+ explicit FP32Vec4(const FP32Vec4& data) : reg(data.reg) {}
185
};
186
187
struct FP32Vec8 : public Vec<FP32Vec8> {
@@ -196,15 +197,15 @@ struct FP32Vec8 : public Vec<FP32Vec8> {
197
198
explicit FP32Vec8() : reg(_mm256_set1_ps(0.0)) {}
199
199
- explicit FP32Vec8(const float *ptr) : reg(_mm256_loadu_ps(ptr)) {}
200
+ explicit FP32Vec8(const float* ptr) : reg(_mm256_loadu_ps(ptr)) {}
201
202
explicit FP32Vec8(__m256 data) : reg(data) {}
203
203
- explicit FP32Vec8(const FP32Vec8 &data) : reg(data.reg) {}
204
+ explicit FP32Vec8(const FP32Vec8& data) : reg(data.reg) {}
205
205
- explicit FP32Vec8(const FP16Vec8 &v) : reg(_mm256_cvtph_ps(v.reg)) {}
206
+ explicit FP32Vec8(const FP16Vec8& v) : reg(_mm256_cvtph_ps(v.reg)) {}
207
207
- explicit FP32Vec8(const BF16Vec8 &v)
208
+ explicit FP32Vec8(const BF16Vec8& v)
209
: reg(_mm256_castsi256_ps(
210
_mm256_bslli_epi128(_mm256_cvtepu16_epi32(v.reg), 2))) {}
211
@@ -212,7 +213,8 @@ struct FP32Vec8 : public Vec<FP32Vec8> {
213
AliasReg ar;
214
ar.reg = reg;
215
float result = 0;
215
- unroll_loop<int, VEC_ELEM_NUM>([&result, &ar](int i) { result += ar.values[i]; });
216
+ unroll_loop<int, VEC_ELEM_NUM>(
217
+ [&result, &ar](int i) { result += ar.values[i]; });
218
219
return result;
220
}
@@ -244,27 +246,27 @@ struct FP32Vec8 : public Vec<FP32Vec8> {
246
erf(ar.values[1]), erf(ar.values[0])));
247
}
248
247
- FP32Vec8 operator*(const FP32Vec8 &b) const {
249
+ FP32Vec8 operator*(const FP32Vec8& b) const {
250
return FP32Vec8(_mm256_mul_ps(reg, b.reg));
251
}
252
251
- FP32Vec8 operator+(const FP32Vec8 &b) const {
253
+ FP32Vec8 operator+(const FP32Vec8& b) const {
254
return FP32Vec8(_mm256_add_ps(reg, b.reg));
255
}
256
255
- FP32Vec8 operator-(const FP32Vec8 &b) const {
257
+ FP32Vec8 operator-(const FP32Vec8& b) const {
258
return FP32Vec8(_mm256_sub_ps(reg, b.reg));
259
}
260
259
- FP32Vec8 operator/(const FP32Vec8 &b) const {
261
+ FP32Vec8 operator/(const FP32Vec8& b) const {
262
return FP32Vec8(_mm256_div_ps(reg, b.reg));
263
}
264
263
- void save(float *ptr) const { _mm256_storeu_ps(ptr, reg); }
265
+ void save(float* ptr) const { _mm256_storeu_ps(ptr, reg); }
266
};
267
268
#ifdef __AVX512F__
267
-struct INT32Vec16: public Vec<INT32Vec16> {
269
+struct INT32Vec16 : public Vec<INT32Vec16> {
270
constexpr static int VEC_ELEM_NUM = 16;
271
union AliasReg {
272
__m512i reg;
@@ -272,12 +274,11 @@ struct INT32Vec16: public Vec<INT32Vec16> {
274
};
275
276
__m512i reg;
275
-
276
- explicit INT32Vec16(const void* data_ptr) : reg(_mm512_loadu_epi32(data_ptr)) {}
277
278
- void save(int32_t* ptr) const {
279
- _mm512_storeu_epi32(ptr, reg);
280
- }
278
+ explicit INT32Vec16(const void* data_ptr)
279
+ : reg(_mm512_loadu_epi32(data_ptr)) {}
280
+
281
+ void save(int32_t* ptr) const { _mm512_storeu_epi32(ptr, reg); }
282
283
void save(int32_t* ptr, const int elem_num) const {
284
constexpr uint32_t M = 0xFFFFFFFF;
@@ -301,11 +302,11 @@ struct FP32Vec16 : public Vec<FP32Vec16> {
302
303
explicit FP32Vec16() : reg(_mm512_set1_ps(0.0)) {}
304
304
- explicit FP32Vec16(const float *ptr) : reg(_mm512_loadu_ps(ptr)) {}
305
+ explicit FP32Vec16(const float* ptr) : reg(_mm512_loadu_ps(ptr)) {}
306
307
explicit FP32Vec16(__m512 data) : reg(data) {}
308
308
- explicit FP32Vec16(const FP32Vec4 &data)
309
+ explicit FP32Vec16(const FP32Vec4& data)
310
: reg((__m512)_mm512_inserti32x4(
311
_mm512_inserti32x4(
312
_mm512_inserti32x4(_mm512_castsi128_si512((__m128i)data.reg),
@@ -313,36 +314,37 @@ struct FP32Vec16 : public Vec<FP32Vec16> {
314
(__m128i)data.reg, 2),
315
(__m128i)data.reg, 3)) {}
316
316
- explicit FP32Vec16(const FP32Vec8 &data)
317
+ explicit FP32Vec16(const FP32Vec8& data)
318
: reg((__m512)_mm512_inserti32x8(
319
_mm512_castsi256_si512((__m256i)data.reg), (__m256i)data.reg, 1)) {}
320
320
- explicit FP32Vec16(const BF16Vec16 &v)
321
+ explicit FP32Vec16(const BF16Vec16& v)
322
: reg(_mm512_castsi512_ps(
323
_mm512_bslli_epi128(_mm512_cvtepu16_epi32(v.reg), 2))) {}
324
324
- explicit FP32Vec16(const FP16Vec16 &v) : reg(_mm512_cvtph_ps(v.reg)) {}
325
+ explicit FP32Vec16(const FP16Vec16& v) : reg(_mm512_cvtph_ps(v.reg)) {}
326
326
- explicit FP32Vec16(const FP16Vec8 &v) : FP32Vec16(FP32Vec8(v)) {}
327
+ explicit FP32Vec16(const FP16Vec8& v) : FP32Vec16(FP32Vec8(v)) {}
328
328
- explicit FP32Vec16(const BF16Vec8 &v) : FP32Vec16(FP32Vec8(v)) {}
329
+ explicit FP32Vec16(const BF16Vec8& v) : FP32Vec16(FP32Vec8(v)) {}
330
330
- explicit FP32Vec16(const INT32Vec16 &v)
331
- : reg(_mm512_cvt_roundepi32_ps(v.reg, _MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC)) {}
331
+ explicit FP32Vec16(const INT32Vec16& v)
332
+ : reg(_mm512_cvt_roundepi32_ps(
333
+ v.reg, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC)) {}
334
333
- FP32Vec16 operator*(const FP32Vec16 &b) const {
335
+ FP32Vec16 operator*(const FP32Vec16& b) const {
336
return FP32Vec16(_mm512_mul_ps(reg, b.reg));
337
}
338
337
- FP32Vec16 operator+(const FP32Vec16 &b) const {
339
+ FP32Vec16 operator+(const FP32Vec16& b) const {
340
return FP32Vec16(_mm512_add_ps(reg, b.reg));
341
}
342
341
- FP32Vec16 operator-(const FP32Vec16 &b) const {
343
+ FP32Vec16 operator-(const FP32Vec16& b) const {
344
return FP32Vec16(_mm512_sub_ps(reg, b.reg));
345
}
346
345
- FP32Vec16 operator/(const FP32Vec16 &b) const {
347
+ FP32Vec16 operator/(const FP32Vec16& b) const {
348
return FP32Vec16(_mm512_div_ps(reg, b.reg));
349
}
350
@@ -370,9 +372,7 @@ struct FP32Vec16 : public Vec<FP32Vec16> {
372
return FP32Vec16(_mm512_mask_min_ps(reg, mask, reg, b.reg));
373
}
374
373
- FP32Vec16 abs() const {
374
- return FP32Vec16(_mm512_abs_ps(reg));
375
- }
375
+ FP32Vec16 abs() const { return FP32Vec16(_mm512_abs_ps(reg)); }
376
377
float reduce_sum() const { return _mm512_reduce_add_ps(reg); }
378
@@ -380,14 +380,15 @@ struct FP32Vec16 : public Vec<FP32Vec16> {
380
381
float reduce_min() const { return _mm512_reduce_min_ps(reg); }
382
383
- template <int group_size> float reduce_sub_sum(int idx) {
383
+ template <int group_size>
384
+ float reduce_sub_sum(int idx) {
385
static_assert(VEC_ELEM_NUM % group_size == 0);
386
constexpr uint32_t base_mask = (0xFFFF >> (16 - group_size));
387
__mmask16 mask = _cvtu32_mask16(base_mask << (idx * group_size));
388
return _mm512_mask_reduce_add_ps(mask, reg);
389
}
390
390
- void save(float *ptr) const { _mm512_storeu_ps(ptr, reg); }
391
+ void save(float* ptr) const { _mm512_storeu_ps(ptr, reg); }
392
393
void save(float* ptr, const int elem_num) const {
394
constexpr uint32_t M = 0xFFFFFFFF;
@@ -407,32 +408,30 @@ struct FP32Vec16 : public Vec<FP32Vec16> {
408
__m256 reg_low;
409
__m256 reg_high;
410
410
- explicit FP32Vec16(float v) : reg_low(_mm256_set1_ps(v)),
411
- reg_high(_mm256_set1_ps(v)) {}
411
+ explicit FP32Vec16(float v)
412
+ : reg_low(_mm256_set1_ps(v)), reg_high(_mm256_set1_ps(v)) {}
413
413
- explicit FP32Vec16() : reg_low(_mm256_set1_ps(0.0)),
414
- reg_high(_mm256_set1_ps(0.0)) {}
414
+ explicit FP32Vec16()
415
+ : reg_low(_mm256_set1_ps(0.0)), reg_high(_mm256_set1_ps(0.0)) {}
416
416
- explicit FP32Vec16(const float *ptr) : reg_low(_mm256_loadu_ps(ptr)),
417
- reg_high(_mm256_loadu_ps(ptr + 8)) {}
417
+ explicit FP32Vec16(const float* ptr)
418
+ : reg_low(_mm256_loadu_ps(ptr)), reg_high(_mm256_loadu_ps(ptr + 8)) {}
419
420
explicit FP32Vec16(__m256 low, __m256 high) : reg_low(low), reg_high(high) {}
421
421
- explicit FP32Vec16(const FP32Vec16 &data) : reg_low(data.reg_low),
422
- reg_high(data.reg_high) {}
422
+ explicit FP32Vec16(const FP32Vec16& data)
423
+ : reg_low(data.reg_low), reg_high(data.reg_high) {}
424
424
- explicit FP32Vec16(const FP32Vec4 &data)
425
+ explicit FP32Vec16(const FP32Vec4& data)
426
: reg_low((__m256)_mm256_inserti128_si256(
426
- _mm256_castsi128_si256((__m128i)data.reg),
427
- (__m128i)data.reg, 1)),
427
+ _mm256_castsi128_si256((__m128i)data.reg), (__m128i)data.reg, 1)),
428
reg_high((__m256)_mm256_inserti128_si256(
429
- _mm256_castsi128_si256((__m128i)data.reg),
430
- (__m128i)data.reg, 1)) {}
429
+ _mm256_castsi128_si256((__m128i)data.reg), (__m128i)data.reg, 1)) {}
430
432
- explicit FP32Vec16(const FP32Vec8 &data)
431
+ explicit FP32Vec16(const FP32Vec8& data)
432
: reg_low(data.reg), reg_high(data.reg) {}
433
435
- explicit FP32Vec16(const FP16Vec16 &v) {
434
+ explicit FP32Vec16(const FP16Vec16& v) {
435
__m128i low = _mm256_extractf128_si256(v.reg, 0);
436
__m128i high = _mm256_extractf128_si256(v.reg, 1);
437
@@ -440,9 +439,9 @@ struct FP32Vec16 : public Vec<FP32Vec16> {
439
reg_high = _mm256_cvtph_ps(high);
440
}
441
443
- explicit FP32Vec16(const FP16Vec8 &v) : FP32Vec16(FP32Vec8(v)) {}
442
+ explicit FP32Vec16(const FP16Vec8& v) : FP32Vec16(FP32Vec8(v)) {}
443
445
- explicit FP32Vec16(const BF16Vec16 &v) {
444
+ explicit FP32Vec16(const BF16Vec16& v) {
445
__m128i low = _mm256_extractf128_si256(v.reg, 0);
446
__m128i high = _mm256_extractf128_si256(v.reg, 1);
447
@@ -456,24 +455,24 @@ struct FP32Vec16 : public Vec<FP32Vec16> {
455
reg_high = _mm256_castsi256_ps(v_high_shifted);
456
}
457
459
- explicit FP32Vec16(const BF16Vec8 &v) : FP32Vec16(FP32Vec8(v)) {}
458
+ explicit FP32Vec16(const BF16Vec8& v) : FP32Vec16(FP32Vec8(v)) {}
459
461
- FP32Vec16 operator*(const FP32Vec16 &b) const {
460
+ FP32Vec16 operator*(const FP32Vec16& b) const {
461
return FP32Vec16(_mm256_mul_ps(reg_low, b.reg_low),
462
_mm256_mul_ps(reg_high, b.reg_high));
463
}
464
466
- FP32Vec16 operator+(const FP32Vec16 &b) const {
465
+ FP32Vec16 operator+(const FP32Vec16& b) const {
466
return FP32Vec16(_mm256_add_ps(reg_low, b.reg_low),
467
_mm256_add_ps(reg_high, b.reg_high));
468
}
469
471
- FP32Vec16 operator-(const FP32Vec16 &b) const {
470
+ FP32Vec16 operator-(const FP32Vec16& b) const {
471
return FP32Vec16(_mm256_sub_ps(reg_low, b.reg_low),
472
_mm256_sub_ps(reg_high, b.reg_high));
473
}
474
476
- FP32Vec16 operator/(const FP32Vec16 &b) const {
475
+ FP32Vec16 operator/(const FP32Vec16& b) const {
476
return FP32Vec16(_mm256_div_ps(reg_low, b.reg_low),
477
_mm256_div_ps(reg_high, b.reg_high));
478
}
@@ -484,7 +483,8 @@ struct FP32Vec16 : public Vec<FP32Vec16> {
483
return low.reduce_sum() + high.reduce_sum();
484
}
485
487
- template <int group_size> float reduce_sub_sum(int idx) {
486
+ template <int group_size>
487
+ float reduce_sub_sum(int idx) {
488
float sum = 0.0;
489
static_assert(VEC_ELEM_NUM % group_size == 0);
490
constexpr uint32_t base_mask = (0xFFFF >> (16 - group_size));
@@ -507,7 +507,7 @@ struct FP32Vec16 : public Vec<FP32Vec16> {
507
return sum;
508
}
509
510
- void save(float *ptr) const {
510
+ void save(float* ptr) const {
511
_mm256_storeu_ps(ptr, reg_low);
512
_mm256_storeu_ps(ptr + 8, reg_high);
513
}
@@ -515,7 +515,7 @@ struct FP32Vec16 : public Vec<FP32Vec16> {
515
#endif
516
517
#ifdef __AVX512F__
518
-struct INT8Vec16: public Vec<INT8Vec16> {
518
+struct INT8Vec16 : public Vec<INT8Vec16> {
519
constexpr static int VEC_ELEM_NUM = 16;
520
union AliasReg {
521
__m128i reg;
@@ -523,14 +523,12 @@ struct INT8Vec16: public Vec<INT8Vec16> {
523
};
524
525
__m128i reg;
526
-
527
- explicit INT8Vec16(const FP32Vec16& vec) : reg(
528
- _mm512_cvtepi32_epi8(_mm512_cvt_roundps_epi32(vec.reg, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC))
529
- ) {}
526
531
- void save(int8_t* ptr) const {
532
- _mm_storeu_epi8(ptr, reg);
533
- }
527
+ explicit INT8Vec16(const FP32Vec16& vec)
528
+ : reg(_mm512_cvtepi32_epi8(_mm512_cvt_roundps_epi32(
529
+ vec.reg, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC))) {}
530
+
531
+ void save(int8_t* ptr) const { _mm_storeu_epi8(ptr, reg); }
532
533
void save(int8_t* ptr, const int elem_num) const {
534
constexpr uint32_t M = 0xFFFFFFFF;
@@ -540,71 +538,92 @@ struct INT8Vec16: public Vec<INT8Vec16> {
538
};
539
#endif
540
543
-template <typename T> struct VecType { using vec_type = void; };
541
+template <typename T>
542
+struct VecType {
543
+ using vec_type = void;
544
+};
545
545
-template <typename T> using vec_t = typename VecType<T>::vec_type;
546
+template <typename T>
547
+using vec_t = typename VecType<T>::vec_type;
548
547
-template <> struct VecType<float> { using vec_type = FP32Vec8; };
549
+template <>
550
+struct VecType<float> {
551
+ using vec_type = FP32Vec8;
552
+};
553
549
-template <> struct VecType<c10::Half> { using vec_type = FP16Vec8; };
554
+template <>
555
+struct VecType<c10::Half> {
556
+ using vec_type = FP16Vec8;
557
+};
558
551
-template <> struct VecType<c10::BFloat16> { using vec_type = BF16Vec8; };
559
+template <>
560
+struct VecType<c10::BFloat16> {
561
+ using vec_type = BF16Vec8;
562
+};
563
553
-template <typename T> void storeFP32(float v, T *ptr) { *ptr = v; }
564
+template <typename T>
565
+void storeFP32(float v, T* ptr) {
566
+ *ptr = v;
567
+}
568
555
-inline void fma(FP32Vec16 &acc, FP32Vec16 &a, FP32Vec16 &b) {
569
+inline void fma(FP32Vec16& acc, FP32Vec16& a, FP32Vec16& b) {
570
acc = acc + a * b;
571
}
572
559
-template <> inline void storeFP32<c10::Half>(float v, c10::Half *ptr) {
560
- *reinterpret_cast<unsigned short *>(ptr) =
573
+template <>
574
+inline void storeFP32<c10::Half>(float v, c10::Half* ptr) {
575
+ *reinterpret_cast<unsigned short*>(ptr) =
576
_cvtss_sh(v, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC);
577
}
578
564
-inline FP16Vec8::FP16Vec8(const FP32Vec8 &v)
579
+inline FP16Vec8::FP16Vec8(const FP32Vec8& v)
580
: reg(_mm256_cvtps_ph(v.reg,
581
_MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC)) {}
582
583
#ifdef __AVX512F__
569
-inline FP16Vec16::FP16Vec16(const FP32Vec16 &v)
584
+inline FP16Vec16::FP16Vec16(const FP32Vec16& v)
585
: reg(_mm512_cvtps_ph(v.reg,
586
_MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC)) {}
587
#else
573
-inline FP16Vec16::FP16Vec16(const FP32Vec16 &v)
574
- : reg(_mm256_insertf128_si256(_mm256_castsi128_si256(FP16Vec8(FP32Vec8(v.reg_low)).reg), FP16Vec8(FP32Vec8(v.reg_low)).reg, 1)) {}
588
+inline FP16Vec16::FP16Vec16(const FP32Vec16& v)
589
+ : reg(_mm256_insertf128_si256(
590
+ _mm256_castsi128_si256(FP16Vec8(FP32Vec8(v.reg_low)).reg),
591
+ FP16Vec8(FP32Vec8(v.reg_low)).reg, 1)) {}
592
#endif
593
594
#ifdef __AVX512BF16__
578
-template <> inline void storeFP32<c10::BFloat16>(float v, c10::BFloat16 *ptr) {
579
- *reinterpret_cast<__bfloat16 *>(ptr) = _mm_cvtness_sbh(v);
595
+template <>
596
+inline void storeFP32<c10::BFloat16>(float v, c10::BFloat16* ptr) {
597
+ *reinterpret_cast<__bfloat16*>(ptr) = _mm_cvtness_sbh(v);
598
}
599
582
-inline BF16Vec8::BF16Vec8(const FP32Vec8 &v)
600
+inline BF16Vec8::BF16Vec8(const FP32Vec8& v)
601
: reg((__m128i)_mm256_cvtneps_pbh(v.reg)) {}
602
585
-inline BF16Vec16::BF16Vec16(const FP32Vec16 &v)
603
+inline BF16Vec16::BF16Vec16(const FP32Vec16& v)
604
: reg((__m256i)_mm512_cvtneps_pbh(v.reg)) {}
605
588
-inline void fma(FP32Vec16 &acc, BF16Vec32 &a, BF16Vec32 &b) {
606
+inline void fma(FP32Vec16& acc, BF16Vec32& a, BF16Vec32& b) {
607
acc.reg = _mm512_dpbf16_ps(acc.reg, (__m512bh)a.reg, (__m512bh)b.reg);
608
}
609
#else
592
-template <> inline void storeFP32<c10::BFloat16>(float v, c10::BFloat16 *ptr) {
593
- c10::BFloat16 __attribute__((__may_alias__)) *v_ptr =
594
- reinterpret_cast<c10::BFloat16 *>(&v);
610
+template <>
611
+inline void storeFP32<c10::BFloat16>(float v, c10::BFloat16* ptr) {
612
+ c10::BFloat16 __attribute__((__may_alias__))* v_ptr =
613
+ reinterpret_cast<c10::BFloat16*>(&v);
614
*ptr = *(v_ptr + 1);
615
}
616
598
-#ifdef __AVX512F__
599
-inline BF16Vec8::BF16Vec8(const FP32Vec8 &v)
617
+ #ifdef __AVX512F__
618
+inline BF16Vec8::BF16Vec8(const FP32Vec8& v)
619
: reg(_mm256_cvtepi32_epi16(
620
_mm256_bsrli_epi128(_mm256_castps_si256(v.reg), 2))) {}
621
603
-inline BF16Vec16::BF16Vec16(const FP32Vec16 &v)
622
+inline BF16Vec16::BF16Vec16(const FP32Vec16& v)
623
: reg(_mm512_cvtepi32_epi16(
624
_mm512_bsrli_epi128(_mm512_castps_si512(v.reg), 2))) {}
606
-#else
607
-namespace{
625
+ #else
626
+namespace {
627
__m128i FP32Vec8_to_BF16Vec8_avx2(__m256 a) {
628
__m256i ai = _mm256_castps_si256(a);
629
ai = _mm256_srli_epi32(ai, 16);
@@ -612,21 +631,21 @@ __m128i FP32Vec8_to_BF16Vec8_avx2(__m256 a) {
631
ai = _mm256_permute4x64_epi64(ai, 0b00111001);
632
return _mm256_extracti128_si256(ai, 0);
633
}
615
-}
634
+} // namespace
635
617
-inline BF16Vec8::BF16Vec8(const FP32Vec8 &v)
636
+inline BF16Vec8::BF16Vec8(const FP32Vec8& v)
637
: reg(FP32Vec8_to_BF16Vec8_avx2(v.reg)) {}
638
620
-inline BF16Vec16::BF16Vec16(const FP32Vec16 &v) {
639
+inline BF16Vec16::BF16Vec16(const FP32Vec16& v) {
640
BF16Vec8 low = BF16Vec8(FP32Vec8(v.reg_low));
641
BF16Vec8 high = BF16Vec8(FP32Vec8(v.reg_high));
642
reg = _mm256_insertf128_si256(_mm256_castsi128_si256(low.reg), high.reg, 1);
643
}
625
-#endif // __AVX512F__
626
-#endif // __AVX512BF16__
644
+ #endif // __AVX512F__
645
+#endif // __AVX512BF16__
646
628
-inline void prefetch(const void *addr) { _mm_prefetch(addr, _MM_HINT_T1); }
647
+inline void prefetch(const void* addr) { _mm_prefetch(addr, _MM_HINT_T1); }
648
630
-}; // namespace vec_op
649
+}; // namespace vec_op
650
651
#endif
csrc/cutlass_extensions/common.hpp
+1
-2
@@ -27,8 +27,7 @@
27
inline int get_cuda_max_shared_memory_per_block_opt_in(int const device) {
28
int max_shared_mem_per_block_opt_in = 0;
29
cudaDeviceGetAttribute(&max_shared_mem_per_block_opt_in,
30
- cudaDevAttrMaxSharedMemoryPerBlockOptin,
31
- device);
30
+ cudaDevAttrMaxSharedMemoryPerBlockOptin, device);
31
return max_shared_mem_per_block_opt_in;
32
}
33
docs/source/contributing/overview.md
+8
-5
@@ -25,10 +25,12 @@ Check out the [building from source](#build-from-source) documentation for detai
25
```bash
26
pip install -r requirements-dev.txt
27
28
-# linting and formatting
29
-bash format.sh
30
-# Static type checking
31
-mypy
28
+# Linting, formatting and static type checking
29
+pre-commit install
30
+
31
+# You can manually run pre-commit with
32
+pre-commit run --all-files
33
+
34
# Unit tests
35
pytest tests/
36
```
@@ -88,7 +90,8 @@ If the PR spans more than one category, please include all relevant prefixes.
90
The PR needs to meet the following code quality standards:
91
92
- We adhere to [Google Python style guide](https://google.github.io/styleguide/pyguide.html) and [Google C++ style guide](https://google.github.io/styleguide/cppguide.html).
91
-- Pass all linter checks. Please use <gh-file:format.sh> to format your code.
93
+- Pass all linter checks. Please use `pre-commit` to format your code. See
94
+ <https://pre-commit.com/#usage> if `pre-commit` is new to you.
95
- The code needs to be well-documented to ensure future contributors can easily
96
understand the code.
97
- Include sufficient tests to ensure the project stays correct and robust. This
format.sh
deleted
-321
@@ -1,321 +0,0 @@
1
-#!/usr/bin/env bash
2
-# YAPF formatter, adapted from ray and skypilot.
3
-#
4
-# Usage:
5
-# # Do work and commit your work.
6
-
7
-# # Format files that differ from origin/main.
8
-# bash format.sh
9
-
10
-# # Commit changed files with message 'Run yapf and ruff'
11
-#
12
-#
13
-# YAPF + Clang formatter (if installed). This script formats all changed files from the last mergebase.
14
-# You are encouraged to run this locally before pushing changes for review.
15
-
16
-# Cause the script to exit if a single command fails
17
-set -eo pipefail
18
-
19
-# this stops git rev-parse from failing if we run this from the .git directory
20
-builtin cd "$(dirname "${BASH_SOURCE:-$0}")"
21
-ROOT="$(git rev-parse --show-toplevel)"
22
-builtin cd "$ROOT" || exit 1
23
-
24
-check_command() {
25
- if ! command -v "$1" &> /dev/null; then
26
- echo "❓❓$1 is not installed, please run \`pip install -r requirements-lint.txt\`"
27
- exit 1
28
- fi
29
-}
30
-
31
-check_command yapf
32
-check_command ruff
33
-check_command mypy
34
-check_command codespell
35
-check_command isort
36
-check_command clang-format
37
-
38
-YAPF_VERSION=$(yapf --version | awk '{print $2}')
39
-RUFF_VERSION=$(ruff --version | awk '{print $2}')
40
-MYPY_VERSION=$(mypy --version | awk '{print $2}')
41
-CODESPELL_VERSION=$(codespell --version)
42
-ISORT_VERSION=$(isort --vn)
43
-CLANGFORMAT_VERSION=$(clang-format --version | awk '{print $3}')
44
-PYMARKDOWNLNT_VERSION=$(pymarkdownlnt version | awk '{print $1}')
45
-
46
-# # params: tool name, tool version, required version
47
-tool_version_check() {
48
- expected=$(grep "$1" requirements-lint.txt | cut -d'=' -f3)
49
- if [[ "$2" != "$expected" ]]; then
50
- echo "❓❓Wrong $1 version installed: $expected is required, not $2."
51
- exit 1
52
- fi
53
-}
54
-
55
-tool_version_check "yapf" "$YAPF_VERSION"
56
-tool_version_check "ruff" "$RUFF_VERSION"
57
-tool_version_check "mypy" "$MYPY_VERSION"
58
-tool_version_check "isort" "$ISORT_VERSION"
59
-tool_version_check "codespell" "$CODESPELL_VERSION"
60
-tool_version_check "clang-format" "$CLANGFORMAT_VERSION"
61
-tool_version_check "pymarkdownlnt" "$PYMARKDOWNLNT_VERSION"
62
-
63
-YAPF_FLAGS=(
64
- '--recursive'
65
- '--parallel'
66
-)
67
-
68
-YAPF_EXCLUDES=(
69
- '--exclude' 'build/**'
70
-)
71
-
72
-# Format specified files
73
-format() {
74
- yapf --in-place "${YAPF_FLAGS[@]}" "$@"
75
-}
76
-
77
-# Format files that differ from main branch. Ignores dirs that are not slated
78
-# for autoformat yet.
79
-format_changed() {
80
- # The `if` guard ensures that the list of filenames is not empty, which
81
- # could cause yapf to receive 0 positional arguments, making it hang
82
- # waiting for STDIN.
83
- #
84
- # `diff-filter=ACM` and $MERGEBASE is to ensure we only format files that
85
- # exist on both branches.
86
- MERGEBASE="$(git merge-base origin/main HEAD)"
87
-
88
- if ! git diff --diff-filter=ACM --quiet --exit-code "$MERGEBASE" -- '*.py' '*.pyi' &>/dev/null; then
89
- git diff --name-only --diff-filter=ACM "$MERGEBASE" -- '*.py' '*.pyi' | xargs -P 5 \
90
- yapf --in-place "${YAPF_EXCLUDES[@]}" "${YAPF_FLAGS[@]}"
91
- fi
92
-
93
-}
94
-
95
-# Format all files
96
-format_all() {
97
- yapf --in-place "${YAPF_FLAGS[@]}" "${YAPF_EXCLUDES[@]}" .
98
-}
99
-
100
-## This flag formats individual files. --files *must* be the first command line
101
-## arg to use this option.
102
-if [[ "$1" == '--files' ]]; then
103
- format "${@:2}"
104
- # If `--all` is passed, then any further arguments are ignored and the
105
- # entire python directory is formatted.
106
-elif [[ "$1" == '--all' ]]; then
107
- format_all
108
-else
109
- # Format only the files that changed in last commit.
110
- format_changed
111
-fi
112
-echo 'vLLM yapf: Done'
113
-
114
-# Run mypy
115
-echo 'vLLM mypy:'
116
-tools/mypy.sh
117
-echo 'vLLM mypy: Done'
118
-
119
-
120
-# If git diff returns a file that is in the skip list, the file may be checked anyway:
121
-# https://github.com/codespell-project/codespell/issues/1915
122
-# Avoiding the "./" prefix and using "/**" globs for directories appears to solve the problem
123
-CODESPELL_EXCLUDES=(
124
- '--skip' 'tests/prompts/**,./benchmarks/sonnet.txt,*tests/lora/data/**,build/**'
125
-)
126
-
127
-# check spelling of specified files
128
-spell_check() {
129
- codespell "$@"
130
-}
131
-
132
-spell_check_all(){
133
- codespell --toml pyproject.toml "${CODESPELL_EXCLUDES[@]}"
134
-}
135
-
136
-# Spelling check of files that differ from main branch.
137
-spell_check_changed() {
138
- # The `if` guard ensures that the list of filenames is not empty, which
139
- # could cause ruff to receive 0 positional arguments, making it hang
140
- # waiting for STDIN.
141
- #
142
- # `diff-filter=ACM` and $MERGEBASE is to ensure we only lint files that
143
- # exist on both branches.
144
- MERGEBASE="$(git merge-base origin/main HEAD)"
145
- if ! git diff --diff-filter=ACM --quiet --exit-code "$MERGEBASE" -- '*.py' '*.pyi' &>/dev/null; then
146
- git diff --name-only --diff-filter=ACM "$MERGEBASE" -- '*.py' '*.pyi' | xargs \
147
- codespell "${CODESPELL_EXCLUDES[@]}"
148
- fi
149
-}
150
-
151
-# Run Codespell
152
-## This flag runs spell check of individual files. --files *must* be the first command line
153
-## arg to use this option.
154
-if [[ "$1" == '--files' ]]; then
155
- spell_check "${@:2}"
156
- # If `--all` is passed, then any further arguments are ignored and the
157
- # entire python directory is linted.
158
-elif [[ "$1" == '--all' ]]; then
159
- spell_check_all
160
-else
161
- # Check spelling only of the files that changed in last commit.
162
- spell_check_changed
163
-fi
164
-echo 'vLLM codespell: Done'
165
-
166
-
167
-# Lint specified files
168
-lint() {
169
- ruff check "$@"
170
-}
171
-
172
-# Lint files that differ from main branch. Ignores dirs that are not slated
173
-# for autolint yet.
174
-lint_changed() {
175
- # The `if` guard ensures that the list of filenames is not empty, which
176
- # could cause ruff to receive 0 positional arguments, making it hang
177
- # waiting for STDIN.
178
- #
179
- # `diff-filter=ACM` and $MERGEBASE is to ensure we only lint files that
180
- # exist on both branches.
181
- MERGEBASE="$(git merge-base origin/main HEAD)"
182
-
183
- if ! git diff --diff-filter=ACM --quiet --exit-code "$MERGEBASE" -- '*.py' '*.pyi' &>/dev/null; then
184
- git diff --name-only --diff-filter=ACM "$MERGEBASE" -- '*.py' '*.pyi' | xargs \
185
- ruff check
186
- fi
187
-
188
-}
189
-
190
-# Run Ruff
191
-### This flag lints individual files. --files *must* be the first command line
192
-### arg to use this option.
193
-if [[ "$1" == '--files' ]]; then
194
- lint "${@:2}"
195
- # If `--all` is passed, then any further arguments are ignored and the
196
- # entire python directory is linted.
197
-elif [[ "$1" == '--all' ]]; then
198
- lint vllm tests
199
-else
200
- # Format only the files that changed in last commit.
201
- lint_changed
202
-fi
203
-echo 'vLLM ruff: Done'
204
-
205
-# check spelling of specified files
206
-isort_check() {
207
- isort "$@"
208
-}
209
-
210
-isort_check_all(){
211
- isort .
212
-}
213
-
214
-# Spelling check of files that differ from main branch.
215
-isort_check_changed() {
216
- # The `if` guard ensures that the list of filenames is not empty, which
217
- # could cause ruff to receive 0 positional arguments, making it hang
218
- # waiting for STDIN.
219
- #
220
- # `diff-filter=ACM` and $MERGEBASE is to ensure we only lint files that
221
- # exist on both branches.
222
- MERGEBASE="$(git merge-base origin/main HEAD)"
223
-
224
- if ! git diff --diff-filter=ACM --quiet --exit-code "$MERGEBASE" -- '*.py' '*.pyi' &>/dev/null; then
225
- git diff --name-only --diff-filter=ACM "$MERGEBASE" -- '*.py' '*.pyi' | xargs \
226
- isort
227
- fi
228
-}
229
-
230
-# Run Isort
231
-# This flag runs spell check of individual files. --files *must* be the first command line
232
-# arg to use this option.
233
-if [[ "$1" == '--files' ]]; then
234
- isort_check "${@:2}"
235
- # If `--all` is passed, then any further arguments are ignored and the
236
- # entire python directory is linted.
237
-elif [[ "$1" == '--all' ]]; then
238
- isort_check_all
239
-else
240
- # Check spelling only of the files that changed in last commit.
241
- isort_check_changed
242
-fi
243
-echo 'vLLM isort: Done'
244
-
245
-# Clang-format section
246
-# Exclude some files for formatting because they are vendored
247
-# NOTE: Keep up to date with .github/workflows/clang-format.yml
248
-CLANG_FORMAT_EXCLUDES=(
249
- 'csrc/moe/topk_softmax_kernels.cu'
250
- 'csrc/quantization/gguf/ggml-common.h'
251
- 'csrc/quantization/gguf/dequantize.cuh'
252
- 'csrc/quantization/gguf/vecdotq.cuh'
253
- 'csrc/quantization/gguf/mmq.cuh'
254
- 'csrc/quantization/gguf/mmvq.cuh'
255
-)
256
-
257
-# Format specified files with clang-format
258
-clang_format() {
259
- clang-format -i "$@"
260
-}
261
-
262
-# Format files that differ from main branch with clang-format.
263
-clang_format_changed() {
264
- # The `if` guard ensures that the list of filenames is not empty, which
265
- # could cause clang-format to receive 0 positional arguments, making it hang
266
- # waiting for STDIN.
267
- #
268
- # `diff-filter=ACM` and $MERGEBASE is to ensure we only format files that
269
- # exist on both branches.
270
- MERGEBASE="$(git merge-base origin/main HEAD)"
271
-
272
- # Get the list of changed files, excluding the specified ones
273
- changed_files=$(git diff --name-only --diff-filter=ACM "$MERGEBASE" -- '*.h' '*.cpp' '*.cu' '*.cuh' | (grep -vFf <(printf "%s\n" "${CLANG_FORMAT_EXCLUDES[@]}") || echo -e))
274
- if [ -n "$changed_files" ]; then
275
- echo "$changed_files" | xargs -P 5 clang-format -i
276
- fi
277
-}
278
-
279
-# Format all files with clang-format
280
-clang_format_all() {
281
- find csrc/ \( -name '*.h' -o -name '*.cpp' -o -name '*.cu' -o -name '*.cuh' \) -print \
282
- | grep -vFf <(printf "%s\n" "${CLANG_FORMAT_EXCLUDES[@]}") \
283
- | xargs clang-format -i
284
-}
285
-
286
-# Run clang-format
287
-if [[ "$1" == '--files' ]]; then
288
- clang_format "${@:2}"
289
-elif [[ "$1" == '--all' ]]; then
290
- clang_format_all
291
-else
292
- clang_format_changed
293
-fi
294
-echo 'vLLM clang-format: Done'
295
-
296
-echo 'vLLM actionlint:'
297
-tools/actionlint.sh -color
298
-echo 'vLLM actionlint: Done'
299
-
300
-echo 'vLLM shellcheck:'
301
-tools/shellcheck.sh
302
-echo 'vLLM shellcheck: Done'
303
-
304
-echo 'excalidraw png check:'
305
-tools/png-lint.sh
306
-echo 'excalidraw png check: Done'
307
-
308
-if ! git diff --quiet &>/dev/null; then
309
- echo
310
- echo "🔍🔍There are files changed by the format checker or by you that are not added and committed:"
311
- git --no-pager diff --name-only
312
- echo "🔍🔍Format checker passed, but please add, commit and push all the files above to include changes made by the format checker."
313
-
314
- exit 1
315
-else
316
- echo "✨🎉 Format check passed! Congratulations! 🎉✨"
317
-fi
318
-
319
-echo 'vLLM doc-lint:'
320
-tools/doc-lint.sh
321
-echo 'vLLM doc-lint: Done'
pyproject.toml
+8
@@ -15,6 +15,11 @@ build-backend = "setuptools.build_meta"
15
[tool.setuptools_scm]
16
# version_file = "vllm/_version.py" # currently handled by `setup.py:get_version()`
17
18
+[tool.yapfignore]
19
+ignore_patterns = [
20
+ "build/**",
21
+]
22
+
23
[tool.ruff]
24
# Allow lines to be as long as 80.
25
line-length = 80
@@ -52,6 +57,9 @@ ignore = [
57
"B007",
58
# f-string format
59
"UP032",
60
+ # Python 3.8 typing
61
+ "UP006", "UP035",
62
+
63
]
64
65
[tool.mypy]
requirements-lint.txt
+1
-14
@@ -1,15 +1,2 @@
1
# formatting
2
-yapf==0.32.0
3
-toml==0.10.2
4
-tomli==2.0.2
5
-ruff==0.6.5
6
-codespell==2.3.0
7
-isort==5.13.2
8
-clang-format==18.1.5
9
-pymarkdownlnt==0.9.26
10
-
11
-# type checking
12
-mypy==1.11.1
13
-types-PyYAML
14
-types-requests
15
-types-setuptools
2
+pre-commit==4.0.1
tools/actionlint.sh
deleted
-13
@@ -1,13 +0,0 @@
1
-#!/bin/bash
2
-
3
-if command -v actionlint &> /dev/null; then
4
- actionlint "$@"
5
- exit 0
6
-elif [ -x ./actionlint ]; then
7
- ./actionlint "$@"
8
- exit 0
9
-fi
10
-
11
-# download a binary to the current directory - v1.7.3
12
-bash <(curl https://raw.githubusercontent.com/rhysd/actionlint/aa0a7be8e566b096e64a5df8ff290ec24fa58fbc/scripts/download-actionlint.bash)
13
-./actionlint "$@"
tools/doc-lint.sh
deleted
-3
@@ -1,3 +0,0 @@
1
-#!/bin/bash
2
-
3
-pymarkdownlnt scan docs -r