@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 3ea7b94523
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 02:00 UTC

Move linting to `pre-commit` (#11975)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>

Harry Mellor committed Jan 20, 2025 at 06:58 UTC 3ea7b94523f748faf464293ca5fdc4c94e3a3a89
26 files changed +725 -1287
.buildkite/nightly-benchmarks/scripts/nightly-annotate.sh
+1 -1
@@ -43,7 +43,7 @@ main() {
43
44
45
46 - # The figures should be genereated by a separate process outside the CI/CD pipeline
46 + # The figures should be generated by a separate process outside the CI/CD pipeline
47
48 # # generate figures
49 # python3 -m pip install tabulate pandas matplotlib
.github/workflows/actionlint.yml deleted
-40
@@ -1,40 +0,0 @@
1 -name: Lint GitHub Actions workflows
2 -on:
3 - push:
4 - branches:
5 - - "main"
6 - paths:
7 - - '.github/workflows/*.ya?ml'
8 - - '.github/workflows/actionlint.*'
9 - - '.github/workflows/matchers/actionlint.json'
10 - pull_request:
11 - branches:
12 - - "main"
13 - paths:
14 - - '.github/workflows/*.ya?ml'
15 - - '.github/workflows/actionlint.*'
16 - - '.github/workflows/matchers/actionlint.json'
17 -
18 -env:
19 - LC_ALL: en_US.UTF-8
20 -
21 -defaults:
22 - run:
23 - shell: bash
24 -
25 -permissions:
26 - contents: read
27 -
28 -jobs:
29 - actionlint:
30 - runs-on: ubuntu-latest
31 - steps:
32 - - name: "Checkout"
33 - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
34 - with:
35 - fetch-depth: 0
36 -
37 - - name: "Run actionlint"
38 - run: |
39 - echo "::add-matcher::.github/workflows/matchers/actionlint.json"
40 - tools/actionlint.sh -color
.github/workflows/clang-format.yml deleted
-53
@@ -1,53 +0,0 @@
1 -name: clang-format
2 -
3 -on:
4 - # Trigger the workflow on push or pull request,
5 - # but only for the main branch
6 - push:
7 - branches:
8 - - main
9 - paths:
10 - - '**/*.h'
11 - - '**/*.cpp'
12 - - '**/*.cu'
13 - - '**/*.cuh'
14 - - '.github/workflows/clang-format.yml'
15 - pull_request:
16 - branches:
17 - - main
18 - paths:
19 - - '**/*.h'
20 - - '**/*.cpp'
21 - - '**/*.cu'
22 - - '**/*.cuh'
23 - - '.github/workflows/clang-format.yml'
24 -
25 -jobs:
26 - clang-format:
27 - runs-on: ubuntu-latest
28 - strategy:
29 - matrix:
30 - python-version: ["3.11"]
31 - steps:
32 - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
33 - - name: Set up Python ${{ matrix.python-version }}
34 - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0
35 - with:
36 - python-version: ${{ matrix.python-version }}
37 - - name: Install dependencies
38 - run: |
39 - python -m pip install --upgrade pip
40 - pip install clang-format==18.1.5
41 - - name: Running clang-format
42 - run: |
43 - EXCLUDES=(
44 - 'csrc/moe/topk_softmax_kernels.cu'
45 - 'csrc/quantization/gguf/ggml-common.h'
46 - 'csrc/quantization/gguf/dequantize.cuh'
47 - 'csrc/quantization/gguf/vecdotq.cuh'
48 - 'csrc/quantization/gguf/mmq.cuh'
49 - 'csrc/quantization/gguf/mmvq.cuh'
50 - )
51 - find csrc/ \( -name '*.h' -o -name '*.cpp' -o -name '*.cu' -o -name '*.cuh' \) -print \
52 - | grep -vFf <(printf "%s\n" "${EXCLUDES[@]}") \
53 - | xargs clang-format --dry-run --Werror
.github/workflows/codespell.yml deleted
-45
@@ -1,45 +0,0 @@
1 -name: codespell
2 -
3 -on:
4 - # Trigger the workflow on push or pull request,
5 - # but only for the main branch
6 - push:
7 - branches:
8 - - main
9 - paths:
10 - - "**/*.py"
11 - - "**/*.md"
12 - - "**/*.rst"
13 - - pyproject.toml
14 - - requirements-lint.txt
15 - - .github/workflows/codespell.yml
16 - pull_request:
17 - branches:
18 - - main
19 - paths:
20 - - "**/*.py"
21 - - "**/*.md"
22 - - "**/*.rst"
23 - - pyproject.toml
24 - - requirements-lint.txt
25 - - .github/workflows/codespell.yml
26 -
27 -jobs:
28 - codespell:
29 - runs-on: ubuntu-latest
30 - strategy:
31 - matrix:
32 - python-version: ["3.12"]
33 - steps:
34 - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
35 - - name: Set up Python ${{ matrix.python-version }}
36 - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0
37 - with:
38 - python-version: ${{ matrix.python-version }}
39 - - name: Install dependencies
40 - run: |
41 - python -m pip install --upgrade pip
42 - pip install -r requirements-lint.txt
43 - - name: Spelling check with codespell
44 - run: |
45 - codespell --toml pyproject.toml
.github/workflows/doc-lint.yml deleted
-32
@@ -1,32 +0,0 @@
1 -name: Lint documentation
2 -
3 -on:
4 - push:
5 - branches:
6 - - main
7 - paths:
8 - - "docs/**"
9 - pull_request:
10 - branches:
11 - - main
12 - paths:
13 - - "docs/**"
14 -
15 -jobs:
16 - doc-lint:
17 - runs-on: ubuntu-latest
18 - strategy:
19 - matrix:
20 - python-version: ["3.12"]
21 - steps:
22 - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
23 - - name: Set up Python ${{ matrix.python-version }}
24 - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0
25 - with:
26 - python-version: ${{ matrix.python-version }}
27 - - name: Install dependencies
28 - run: |
29 - python -m pip install --upgrade pip
30 - pip install -r requirements-lint.txt
31 - - name: Linting docs
32 - run: tools/doc-lint.sh
.github/workflows/dummy.yml new
+20
@@ -0,0 +1,20 @@
1 +name: dummy-checks
2 +
3 +on:
4 + pull_request:
5 +
6 +jobs:
7 + mypy:
8 + runs-on: ubuntu-latest
9 + strategy:
10 + matrix:
11 + python-version: ["3.12"]
12 + steps:
13 + - run: echo "This is a dummy step that always passes"
14 + ruff:
15 + runs-on: ubuntu-latest
16 + strategy:
17 + matrix:
18 + python-version: ["3.12"]
19 + steps:
20 + - run: echo "This is a dummy step that always passes"
.github/workflows/matchers/ruff.json deleted
-17
@@ -1,17 +0,0 @@
1 -{
2 - "problemMatcher": [
3 - {
4 - "owner": "ruff",
5 - "pattern": [
6 - {
7 - "regexp": "^(.+?):(\\d+):(\\d+): (\\w+): (.+)$",
8 - "file": 1,
9 - "line": 2,
10 - "column": 3,
11 - "code": 4,
12 - "message": 5
13 - }
14 - ]
15 - }
16 - ]
17 - }
.github/workflows/mypy.yaml deleted
-51
@@ -1,51 +0,0 @@
1 -name: mypy
2 -
3 -on:
4 - # Trigger the workflow on push or pull request,
5 - # but only for the main branch
6 - push:
7 - branches:
8 - - main
9 - paths:
10 - - '**/*.py'
11 - - '.github/workflows/mypy.yaml'
12 - - 'tools/mypy.sh'
13 - - 'pyproject.toml'
14 - pull_request:
15 - branches:
16 - - main
17 - # This workflow is only relevant when one of the following files changes.
18 - # However, we have github configured to expect and require this workflow
19 - # to run and pass before github with auto-merge a pull request. Until github
20 - # allows more flexible auto-merge policy, we can just run this on every PR.
21 - # It doesn't take that long to run, anyway.
22 - #paths:
23 - # - '**/*.py'
24 - # - '.github/workflows/mypy.yaml'
25 - # - 'tools/mypy.sh'
26 - # - 'pyproject.toml'
27 -
28 -jobs:
29 - mypy:
30 - runs-on: ubuntu-latest
31 - strategy:
32 - matrix:
33 - python-version: ["3.9", "3.10", "3.11", "3.12"]
34 - steps:
35 - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
36 - - name: Set up Python ${{ matrix.python-version }}
37 - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0
38 - with:
39 - python-version: ${{ matrix.python-version }}
40 - - name: Install dependencies
41 - run: |
42 - python -m pip install --upgrade pip
43 - pip install mypy==1.11.1
44 - pip install types-setuptools
45 - pip install types-PyYAML
46 - pip install types-requests
47 - pip install types-setuptools
48 - - name: Mypy
49 - run: |
50 - echo "::add-matcher::.github/workflows/matchers/mypy.json"
51 - tools/mypy.sh 1 ${{ matrix.python-version }}
.github/workflows/png-lint.yml deleted
-37
@@ -1,37 +0,0 @@
1 -name: Lint PNG exports from excalidraw
2 -on:
3 - push:
4 - branches:
5 - - "main"
6 - paths:
7 - - '*.excalidraw.png'
8 - - '.github/workflows/png-lint.yml'
9 - pull_request:
10 - branches:
11 - - "main"
12 - paths:
13 - - '*.excalidraw.png'
14 - - '.github/workflows/png-lint.yml'
15 -
16 -env:
17 - LC_ALL: en_US.UTF-8
18 -
19 -defaults:
20 - run:
21 - shell: bash
22 -
23 -permissions:
24 - contents: read
25 -
26 -jobs:
27 - actionlint:
28 - runs-on: ubuntu-latest
29 - steps:
30 - - name: "Checkout"
31 - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
32 - with:
33 - fetch-depth: 0
34 -
35 - - name: "Run png-lint.sh to check excalidraw exported images"
36 - run: |
37 - tools/png-lint.sh
.github/workflows/pre-commit.yml new
+17
@@ -0,0 +1,17 @@
1 +name: pre-commit
2 +
3 +on:
4 + pull_request:
5 + push:
6 + branches: [main]
7 +
8 +jobs:
9 + pre-commit:
10 + runs-on: ubuntu-latest
11 + steps:
12 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
13 + - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0
14 + with:
15 + python-version: "3.12"
16 + - run: echo "::add-matcher::.github/workflows/matchers/actionlint.json"
17 + - uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd # v3.0.1
.github/workflows/ruff.yml deleted
-52
@@ -1,52 +0,0 @@
1 -name: ruff
2 -
3 -on:
4 - # Trigger the workflow on push or pull request,
5 - # but only for the main branch
6 - push:
7 - branches:
8 - - main
9 - paths:
10 - - "**/*.py"
11 - - pyproject.toml
12 - - requirements-lint.txt
13 - - .github/workflows/matchers/ruff.json
14 - - .github/workflows/ruff.yml
15 - pull_request:
16 - branches:
17 - - main
18 - # This workflow is only relevant when one of the following files changes.
19 - # However, we have github configured to expect and require this workflow
20 - # to run and pass before github with auto-merge a pull request. Until github
21 - # allows more flexible auto-merge policy, we can just run this on every PR.
22 - # It doesn't take that long to run, anyway.
23 - #paths:
24 - # - "**/*.py"
25 - # - pyproject.toml
26 - # - requirements-lint.txt
27 - # - .github/workflows/matchers/ruff.json
28 - # - .github/workflows/ruff.yml
29 -
30 -jobs:
31 - ruff:
32 - runs-on: ubuntu-latest
33 - strategy:
34 - matrix:
35 - python-version: ["3.12"]
36 - steps:
37 - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
38 - - name: Set up Python ${{ matrix.python-version }}
39 - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0
40 - with:
41 - python-version: ${{ matrix.python-version }}
42 - - name: Install dependencies
43 - run: |
44 - python -m pip install --upgrade pip
45 - pip install -r requirements-lint.txt
46 - - name: Analysing the code with ruff
47 - run: |
48 - echo "::add-matcher::.github/workflows/matchers/ruff.json"
49 - ruff check --output-format github .
50 - - name: Run isort
51 - run: |
52 - isort . --check-only
.github/workflows/shellcheck.yml deleted
-37
@@ -1,37 +0,0 @@
1 -name: Lint shell scripts
2 -on:
3 - push:
4 - branches:
5 - - "main"
6 - paths:
7 - - '**/*.sh'
8 - - '.github/workflows/shellcheck.yml'
9 - pull_request:
10 - branches:
11 - - "main"
12 - paths:
13 - - '**/*.sh'
14 - - '.github/workflows/shellcheck.yml'
15 -
16 -env:
17 - LC_ALL: en_US.UTF-8
18 -
19 -defaults:
20 - run:
21 - shell: bash
22 -
23 -permissions:
24 - contents: read
25 -
26 -jobs:
27 - shellcheck:
28 - runs-on: ubuntu-latest
29 - steps:
30 - - name: "Checkout"
31 - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
32 - with:
33 - fetch-depth: 0
34 -
35 - - name: "Check shell scripts"
36 - run: |
37 - tools/shellcheck.sh
.github/workflows/yapf.yml deleted
-38
@@ -1,38 +0,0 @@
1 -name: yapf
2 -
3 -on:
4 - # Trigger the workflow on push or pull request,
5 - # but only for the main branch
6 - push:
7 - branches:
8 - - main
9 - paths:
10 - - "**/*.py"
11 - - .github/workflows/yapf.yml
12 - pull_request:
13 - branches:
14 - - main
15 - paths:
16 - - "**/*.py"
17 - - .github/workflows/yapf.yml
18 -
19 -jobs:
20 - yapf:
21 - runs-on: ubuntu-latest
22 - strategy:
23 - matrix:
24 - python-version: ["3.12"]
25 - steps:
26 - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
27 - - name: Set up Python ${{ matrix.python-version }}
28 - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0
29 - with:
30 - python-version: ${{ matrix.python-version }}
31 - - name: Install dependencies
32 - run: |
33 - python -m pip install --upgrade pip
34 - pip install yapf==0.32.0
35 - pip install toml==0.10.2
36 - - name: Running yapf
37 - run: |
38 - yapf --diff --recursive .
.pre-commit-config.yaml new
+73
@@ -0,0 +1,73 @@
1 +repos:
2 +- repo: https://github.com/google/yapf
3 + rev: v0.32.0
4 + hooks:
5 + - id: yapf
6 + args: [--in-place, --verbose]
7 + additional_dependencies: [toml] # TODO: Remove when yapf is upgraded
8 +- repo: https://github.com/astral-sh/ruff-pre-commit
9 + rev: v0.6.5
10 + hooks:
11 + - id: ruff
12 + args: [--output-format, github]
13 +- repo: https://github.com/codespell-project/codespell
14 + rev: v2.3.0
15 + hooks:
16 + - id: codespell
17 + exclude: 'benchmarks/sonnet.txt|(build|tests/(lora/data|models/fixtures|prompts))/.*'
18 +- repo: https://github.com/PyCQA/isort
19 + rev: 5.13.2
20 + hooks:
21 + - id: isort
22 +- repo: https://github.com/pre-commit/mirrors-clang-format
23 + rev: v18.1.5
24 + hooks:
25 + - id: clang-format
26 + exclude: 'csrc/(moe/topk_softmax_kernels.cu|quantization/gguf/(ggml-common.h|dequantize.cuh|vecdotq.cuh|mmq.cuh|mmvq.cuh))'
27 + types_or: [c++, cuda]
28 + args: [--style=file, --verbose]
29 +- repo: https://github.com/jackdewinter/pymarkdown
30 + rev: v0.9.27
31 + hooks:
32 + - id: pymarkdown
33 + files: docs/.*
34 +- repo: local
35 + hooks:
36 + - id: mypy-3.9 # TODO: Use https://github.com/pre-commit/mirrors-mypy when mypy setup is less awkward
37 + name: Run mypy for Python 3.9
38 + entry: tools/mypy.sh 1 "3.9"
39 + language: python
40 + types: [python]
41 + additional_dependencies: &mypy_deps [mypy==1.11.1, types-setuptools, types-PyYAML, types-requests]
42 + - id: mypy-3.10 # TODO: Use https://github.com/pre-commit/mirrors-mypy when mypy setup is less awkward
43 + name: Run mypy for Python 3.10
44 + entry: tools/mypy.sh 1 "3.10"
45 + language: python
46 + types: [python]
47 + additional_dependencies: *mypy_deps
48 + - id: mypy-3.11 # TODO: Use https://github.com/pre-commit/mirrors-mypy when mypy setup is less awkward
49 + name: Run mypy for Python 3.11
50 + entry: tools/mypy.sh 1 "3.11"
51 + language: python
52 + types: [python]
53 + additional_dependencies: *mypy_deps
54 + - id: mypy-3.12 # TODO: Use https://github.com/pre-commit/mirrors-mypy when mypy setup is less awkward
55 + name: Run mypy for Python 3.12
56 + entry: tools/mypy.sh 1 "3.12"
57 + language: python
58 + types: [python]
59 + additional_dependencies: *mypy_deps
60 + - id: shellcheck
61 + name: Lint shell scripts
62 + entry: tools/shellcheck.sh
63 + language: script
64 + types: [shell]
65 + - id: png-lint
66 + name: Lint PNG exports from excalidraw
67 + entry: tools/png-lint.sh
68 + language: script
69 + types: [png]
70 +- repo: https://github.com/rhysd/actionlint
71 + rev: v1.7.6
72 + hooks:
73 + - id: actionlint
csrc/core/scalar_type.hpp
+1 -1
@@ -32,7 +32,7 @@ class ScalarType {
32 signed_(signed_),
33 bias(bias),
34 finite_values_only(finite_values_only),
35 - nan_repr(nan_repr){};
35 + nan_repr(nan_repr) {};
36
37 static constexpr ScalarType int_(uint8_t size_bits, int32_t bias = 0) {
38 return ScalarType(0, size_bits - 1, true, bias);
csrc/cpu/cpu_types.hpp
+3 -3
@@ -2,13 +2,13 @@
2 #define CPU_TYPES_HPP
3
4 #if defined(__x86_64__)
5 - //x86 implementation
5 + // x86 implementation
6 #include "cpu_types_x86.hpp"
7 #elif defined(__POWER9_VECTOR__)
8 - //ppc implementation
8 + // ppc implementation
9 #include "cpu_types_vsx.hpp"
10 #elif defined(__aarch64__)
11 - //arm implementation
11 + // arm implementation
12 #include "cpu_types_arm.hpp"
13 #else
14 #warning "unsupported vLLM cpu implementation"
csrc/cpu/cpu_types_arm.hpp
+284 -265
@@ -1,48 +1,50 @@
1 #include <arm_neon.h>
2 -#include <torch/all.h>
2 +#include <torch/all.h>
3 #include <cmath>
4
5 namespace vec_op {
6
7 #ifdef ARM_BF16_SUPPORT
8 - #define VLLM_DISPATCH_CASE_FLOATING_TYPES(...) \
9 - AT_DISPATCH_CASE(at::ScalarType::Float, __VA_ARGS__) \
10 - AT_DISPATCH_CASE(at::ScalarType::Half, __VA_ARGS__) \
11 - AT_DISPATCH_CASE(at::ScalarType::BFloat16, __VA_ARGS__)
8 + #define VLLM_DISPATCH_CASE_FLOATING_TYPES(...) \
9 + AT_DISPATCH_CASE(at::ScalarType::Float, __VA_ARGS__) \
10 + AT_DISPATCH_CASE(at::ScalarType::Half, __VA_ARGS__) \
11 + AT_DISPATCH_CASE(at::ScalarType::BFloat16, __VA_ARGS__)
12 #else
13 - #define VLLM_DISPATCH_CASE_FLOATING_TYPES(...) \
14 - AT_DISPATCH_CASE(at::ScalarType::Float, __VA_ARGS__) \
13 + #define VLLM_DISPATCH_CASE_FLOATING_TYPES(...) \
14 + AT_DISPATCH_CASE(at::ScalarType::Float, __VA_ARGS__) \
15 AT_DISPATCH_CASE(at::ScalarType::Half, __VA_ARGS__)
16 #endif
17
18 -#define VLLM_DISPATCH_FLOATING_TYPES(TYPE, NAME, ...) \
18 +#define VLLM_DISPATCH_FLOATING_TYPES(TYPE, NAME, ...) \
19 AT_DISPATCH_SWITCH(TYPE, NAME, VLLM_DISPATCH_CASE_FLOATING_TYPES(__VA_ARGS__))
20
21 #ifndef CPU_OP_GUARD
22 -#define CPU_KERNEL_GUARD_IN(NAME)
23 -#define CPU_KERNEL_GUARD_OUT(NAME)
22 + #define CPU_KERNEL_GUARD_IN(NAME)
23 + #define CPU_KERNEL_GUARD_OUT(NAME)
24 #else
25 -#define CPU_KERNEL_GUARD_IN(NAME) \
26 - std::cout << #NAME << " invoked." << std::endl;
27 -#define CPU_KERNEL_GUARD_OUT(NAME) std::cout << #NAME << " exit." << std::endl;
25 + #define CPU_KERNEL_GUARD_IN(NAME) \
26 + std::cout << #NAME << " invoked." << std::endl;
27 + #define CPU_KERNEL_GUARD_OUT(NAME) \
28 + std::cout << #NAME << " exit." << std::endl;
29 #endif
30
31 #define FORCE_INLINE __attribute__((always_inline)) inline
32
33 namespace {
33 - template <typename T, T... indexes, typename F>
34 - constexpr void unroll_loop_item(std::integer_sequence<T, indexes...>, F &&f) {
35 - (f(std::integral_constant<T, indexes>{}), ...);
36 - };
37 -};
34 +template <typename T, T... indexes, typename F>
35 +constexpr void unroll_loop_item(std::integer_sequence<T, indexes...>, F&& f) {
36 + (f(std::integral_constant<T, indexes>{}), ...);
37 +};
38 +}; // namespace
39
40 template <typename T, T count, typename F,
41 typename = std::enable_if_t<std::is_invocable_v<F, T>>>
41 -constexpr void unroll_loop(F &&f) {
42 +constexpr void unroll_loop(F&& f) {
43 unroll_loop_item(std::make_integer_sequence<T, count>{}, std::forward<F>(f));
44 }
45
45 -template <typename T> struct Vec {
46 +template <typename T>
47 +struct Vec {
48 constexpr static int get_elem_num() { return T::VEC_ELEM_NUM; };
49 };
50
@@ -54,127 +56,124 @@ struct FP16Vec8 : public Vec<FP16Vec8> {
56
57 float16x8_t reg;
58
57 - explicit FP16Vec8(const void *ptr)
58 - : reg(vld1q_f16(static_cast<const __fp16 *>(ptr))) {};
59 + explicit FP16Vec8(const void* ptr)
60 + : reg(vld1q_f16(static_cast<const __fp16*>(ptr))) {};
61
60 - explicit FP16Vec8(const FP32Vec8 &);
62 + explicit FP16Vec8(const FP32Vec8&);
63
62 - void save(void *ptr) const {
63 - vst1q_f16(static_cast<__fp16 *>(ptr), reg);
64 - }
64 + void save(void* ptr) const { vst1q_f16(static_cast<__fp16*>(ptr), reg); }
65 };
66
67 struct FP16Vec16 : public Vec<FP16Vec16> {
68 - constexpr static int VEC_ELEM_NUM = 16;
69 -
70 - float16x8x2_t reg;
71 -
72 - explicit FP16Vec16(const void *ptr) {
73 - reg.val[0] = vld1q_f16(reinterpret_cast<const __fp16*>(ptr));
74 - reg.val[1] = vld1q_f16(reinterpret_cast<const __fp16*>(ptr) + 8);
75 - }
76 -
77 - explicit FP16Vec16(const FP32Vec16& vec);
78 -
79 - void save(void *ptr) const {
80 - vst1q_f16(reinterpret_cast<__fp16*>(ptr), reg.val[0]);
81 - vst1q_f16(reinterpret_cast<__fp16*>(ptr) + 8, reg.val[1]);
68 + constexpr static int VEC_ELEM_NUM = 16;
69 +
70 + float16x8x2_t reg;
71 +
72 + explicit FP16Vec16(const void* ptr) {
73 + reg.val[0] = vld1q_f16(reinterpret_cast<const __fp16*>(ptr));
74 + reg.val[1] = vld1q_f16(reinterpret_cast<const __fp16*>(ptr) + 8);
75 + }
76 +
77 + explicit FP16Vec16(const FP32Vec16& vec);
78 +
79 + void save(void* ptr) const {
80 + vst1q_f16(reinterpret_cast<__fp16*>(ptr), reg.val[0]);
81 + vst1q_f16(reinterpret_cast<__fp16*>(ptr) + 8, reg.val[1]);
82 + }
83 +
84 + void save(void* ptr, const int elem_num) const {
85 + int full_blocks = elem_num / 8;
86 + int remainder = elem_num % 8;
87 +
88 + if (full_blocks > 0) {
89 + vst1q_f16(reinterpret_cast<__fp16*>(ptr), reg.val[0]);
90 + if (full_blocks > 1) {
91 + vst1q_f16(reinterpret_cast<__fp16*>(ptr) + 8, reg.val[1]);
92 + }
93 }
83 -
84 - void save(void *ptr, const int elem_num) const {
85 - int full_blocks = elem_num / 8;
86 - int remainder = elem_num % 8;
87 -
88 - if (full_blocks > 0) {
89 - vst1q_f16(reinterpret_cast<__fp16*>(ptr), reg.val[0]);
90 - if (full_blocks > 1) {
91 - vst1q_f16(reinterpret_cast<__fp16*>(ptr) + 8, reg.val[1]);
92 - }
93 - }
94 -
95 - // Note: below is the unrolled version of the following code:
96 - //
97 - // for (int i = 0; i < remainder; ++i) {
98 - // reinterpret_cast<__fp16*>(ptr)[full_blocks * 8 + i] =
99 - // vgetq_lane_f16(temp, i);
100 - // }
101 - //
102 - // For macOS build (Clang), the arm/neon intrinsics function
103 - // `vgetq_lane_f16` needs the parameter `i` to be constant at compile
104 - // time.
105 -
106 - if (remainder > 0) {
107 - float16x8_t temp = reg.val[full_blocks];
108 - __fp16* fp16_ptr = reinterpret_cast<__fp16*>(ptr);
109 - switch (remainder)
110 - {
111 - case 1:
112 - fp16_ptr[full_blocks * 8 + 0] = vgetq_lane_f16(temp, 0);
113 - break;
114 - case 2:
115 - fp16_ptr[full_blocks * 8 + 0] = vgetq_lane_f16(temp, 0);
116 - fp16_ptr[full_blocks * 8 + 1] = vgetq_lane_f16(temp, 1);
117 - break;
118 - case 3:
119 - fp16_ptr[full_blocks * 8 + 0] = vgetq_lane_f16(temp, 0);
120 - fp16_ptr[full_blocks * 8 + 1] = vgetq_lane_f16(temp, 1);
121 - fp16_ptr[full_blocks * 8 + 2] = vgetq_lane_f16(temp, 2);
122 - break;
123 - case 4:
124 - fp16_ptr[full_blocks * 8 + 0] = vgetq_lane_f16(temp, 0);
125 - fp16_ptr[full_blocks * 8 + 1] = vgetq_lane_f16(temp, 1);
126 - fp16_ptr[full_blocks * 8 + 2] = vgetq_lane_f16(temp, 2);
127 - fp16_ptr[full_blocks * 8 + 3] = vgetq_lane_f16(temp, 3);
128 - break;
129 - case 5:
130 - fp16_ptr[full_blocks * 8 + 0] = vgetq_lane_f16(temp, 0);
131 - fp16_ptr[full_blocks * 8 + 1] = vgetq_lane_f16(temp, 1);
132 - fp16_ptr[full_blocks * 8 + 2] = vgetq_lane_f16(temp, 2);
133 - fp16_ptr[full_blocks * 8 + 3] = vgetq_lane_f16(temp, 3);
134 - fp16_ptr[full_blocks * 8 + 4] = vgetq_lane_f16(temp, 4);
135 - break;
136 - case 6:
137 - fp16_ptr[full_blocks * 8 + 0] = vgetq_lane_f16(temp, 0);
138 - fp16_ptr[full_blocks * 8 + 1] = vgetq_lane_f16(temp, 1);
139 - fp16_ptr[full_blocks * 8 + 2] = vgetq_lane_f16(temp, 2);
140 - fp16_ptr[full_blocks * 8 + 3] = vgetq_lane_f16(temp, 3);
141 - fp16_ptr[full_blocks * 8 + 4] = vgetq_lane_f16(temp, 4);
142 - fp16_ptr[full_blocks * 8 + 5] = vgetq_lane_f16(temp, 5);
143 - break;
144 - case 7:
145 - fp16_ptr[full_blocks * 8 + 0] = vgetq_lane_f16(temp, 0);
146 - fp16_ptr[full_blocks * 8 + 1] = vgetq_lane_f16(temp, 1);
147 - fp16_ptr[full_blocks * 8 + 2] = vgetq_lane_f16(temp, 2);
148 - fp16_ptr[full_blocks * 8 + 3] = vgetq_lane_f16(temp, 3);
149 - fp16_ptr[full_blocks * 8 + 4] = vgetq_lane_f16(temp, 4);
150 - fp16_ptr[full_blocks * 8 + 5] = vgetq_lane_f16(temp, 5);
151 - fp16_ptr[full_blocks * 8 + 6] = vgetq_lane_f16(temp, 6);
152 - break;
153 -
154 - default:
155 - break;
156 - }
157 - }
94 +
95 + // Note: below is the unrolled version of the following code:
96 + //
97 + // for (int i = 0; i < remainder; ++i) {
98 + // reinterpret_cast<__fp16*>(ptr)[full_blocks * 8 + i] =
99 + // vgetq_lane_f16(temp, i);
100 + // }
101 + //
102 + // For macOS build (Clang), the arm/neon intrinsics function
103 + // `vgetq_lane_f16` needs the parameter `i` to be constant at compile
104 + // time.
105 +
106 + if (remainder > 0) {
107 + float16x8_t temp = reg.val[full_blocks];
108 + __fp16* fp16_ptr = reinterpret_cast<__fp16*>(ptr);
109 + switch (remainder) {
110 + case 1:
111 + fp16_ptr[full_blocks * 8 + 0] = vgetq_lane_f16(temp, 0);
112 + break;
113 + case 2:
114 + fp16_ptr[full_blocks * 8 + 0] = vgetq_lane_f16(temp, 0);
115 + fp16_ptr[full_blocks * 8 + 1] = vgetq_lane_f16(temp, 1);
116 + break;
117 + case 3:
118 + fp16_ptr[full_blocks * 8 + 0] = vgetq_lane_f16(temp, 0);
119 + fp16_ptr[full_blocks * 8 + 1] = vgetq_lane_f16(temp, 1);
120 + fp16_ptr[full_blocks * 8 + 2] = vgetq_lane_f16(temp, 2);
121 + break;
122 + case 4:
123 + fp16_ptr[full_blocks * 8 + 0] = vgetq_lane_f16(temp, 0);
124 + fp16_ptr[full_blocks * 8 + 1] = vgetq_lane_f16(temp, 1);
125 + fp16_ptr[full_blocks * 8 + 2] = vgetq_lane_f16(temp, 2);
126 + fp16_ptr[full_blocks * 8 + 3] = vgetq_lane_f16(temp, 3);
127 + break;
128 + case 5:
129 + fp16_ptr[full_blocks * 8 + 0] = vgetq_lane_f16(temp, 0);
130 + fp16_ptr[full_blocks * 8 + 1] = vgetq_lane_f16(temp, 1);
131 + fp16_ptr[full_blocks * 8 + 2] = vgetq_lane_f16(temp, 2);
132 + fp16_ptr[full_blocks * 8 + 3] = vgetq_lane_f16(temp, 3);
133 + fp16_ptr[full_blocks * 8 + 4] = vgetq_lane_f16(temp, 4);
134 + break;
135 + case 6:
136 + fp16_ptr[full_blocks * 8 + 0] = vgetq_lane_f16(temp, 0);
137 + fp16_ptr[full_blocks * 8 + 1] = vgetq_lane_f16(temp, 1);
138 + fp16_ptr[full_blocks * 8 + 2] = vgetq_lane_f16(temp, 2);
139 + fp16_ptr[full_blocks * 8 + 3] = vgetq_lane_f16(temp, 3);
140 + fp16_ptr[full_blocks * 8 + 4] = vgetq_lane_f16(temp, 4);
141 + fp16_ptr[full_blocks * 8 + 5] = vgetq_lane_f16(temp, 5);
142 + break;
143 + case 7:
144 + fp16_ptr[full_blocks * 8 + 0] = vgetq_lane_f16(temp, 0);
145 + fp16_ptr[full_blocks * 8 + 1] = vgetq_lane_f16(temp, 1);
146 + fp16_ptr[full_blocks * 8 + 2] = vgetq_lane_f16(temp, 2);
147 + fp16_ptr[full_blocks * 8 + 3] = vgetq_lane_f16(temp, 3);
148 + fp16_ptr[full_blocks * 8 + 4] = vgetq_lane_f16(temp, 4);
149 + fp16_ptr[full_blocks * 8 + 5] = vgetq_lane_f16(temp, 5);
150 + fp16_ptr[full_blocks * 8 + 6] = vgetq_lane_f16(temp, 6);
151 + break;
152 +
153 + default:
154 + break;
155 + }
156 }
157 + }
158 };
159
161 -
160 #ifdef ARM_BF16_SUPPORT
161 struct BF16Vec8 : public Vec<BF16Vec8> {
162 constexpr static int VEC_ELEM_NUM = 8;
163
164 bfloat16x8_t reg;
165
168 - explicit BF16Vec8(const void *ptr)
169 - : reg(*reinterpret_cast<const bfloat16x8_t *>(ptr)) {};
166 + explicit BF16Vec8(const void* ptr)
167 + : reg(*reinterpret_cast<const bfloat16x8_t*>(ptr)) {};
168
169 explicit BF16Vec8(bfloat16x8_t data) : reg(data) {};
170
173 - explicit BF16Vec8(const FP32Vec8 &);
171 + explicit BF16Vec8(const FP32Vec8&);
172
175 - explicit BF16Vec8(float32x4x2_t v) : reg(vcvtq_high_bf16_f32(vcvtq_low_bf16_f32(v.val[0]), v.val[1])) {};
173 + explicit BF16Vec8(float32x4x2_t v)
174 + : reg(vcvtq_high_bf16_f32(vcvtq_low_bf16_f32(v.val[0]), v.val[1])) {};
175
177 - void save(void *ptr) const { *reinterpret_cast<bfloat16x8_t *>(ptr) = reg; }
176 + void save(void* ptr) const { *reinterpret_cast<bfloat16x8_t*>(ptr) = reg; }
177 };
178
179 struct BF16Vec16 : public Vec<BF16Vec16> {
@@ -182,19 +181,18 @@ struct BF16Vec16 : public Vec<BF16Vec16> {
181
182 bfloat16x8x2_t reg;
183
185 - explicit BF16Vec16(const void *ptr)
186 - : reg(*reinterpret_cast<const bfloat16x8x2_t *>(ptr)) {};
184 + explicit BF16Vec16(const void* ptr)
185 + : reg(*reinterpret_cast<const bfloat16x8x2_t*>(ptr)) {};
186
187 explicit BF16Vec16(bfloat16x8x2_t data) : reg(data) {};
188
190 - explicit BF16Vec16(const FP32Vec16 &);
189 + explicit BF16Vec16(const FP32Vec16&);
190
192 - explicit BF16Vec16(float32x4x4_t v) : reg({
193 - vcvtq_high_bf16_f32(vcvtq_low_bf16_f32(v.val[0]), v.val[1]),
194 - vcvtq_high_bf16_f32(vcvtq_low_bf16_f32(v.val[2]), v.val[3])
195 - }){};
191 + explicit BF16Vec16(float32x4x4_t v)
192 + : reg({vcvtq_high_bf16_f32(vcvtq_low_bf16_f32(v.val[0]), v.val[1]),
193 + vcvtq_high_bf16_f32(vcvtq_low_bf16_f32(v.val[2]), v.val[3])}) {};
194
197 - void save(void *ptr) const { *reinterpret_cast<bfloat16x8x2_t *>(ptr) = reg; };
195 + void save(void* ptr) const { *reinterpret_cast<bfloat16x8x2_t*>(ptr) = reg; };
196 };
197
198 struct BF16Vec32 : public Vec<BF16Vec32> {
@@ -202,19 +200,15 @@ struct BF16Vec32 : public Vec<BF16Vec32> {
200
201 bfloat16x8x4_t reg;
202
205 - explicit BF16Vec32(const void *ptr)
206 - : reg(*reinterpret_cast<const bfloat16x8x4_t *>(ptr)) {};
203 + explicit BF16Vec32(const void* ptr)
204 + : reg(*reinterpret_cast<const bfloat16x8x4_t*>(ptr)) {};
205
206 explicit BF16Vec32(bfloat16x8x4_t data) : reg(data) {};
207
210 - explicit BF16Vec32(const BF16Vec8 &vec8_data) : reg({
211 - vec8_data.reg,
212 - vec8_data.reg,
213 - vec8_data.reg,
214 - vec8_data.reg
215 - }) {};
208 + explicit BF16Vec32(const BF16Vec8& vec8_data)
209 + : reg({vec8_data.reg, vec8_data.reg, vec8_data.reg, vec8_data.reg}) {};
210
217 - void save(void *ptr) const { *reinterpret_cast<bfloat16x8x4_t *>(ptr) = reg; };
211 + void save(void* ptr) const { *reinterpret_cast<bfloat16x8x4_t*>(ptr) = reg; };
212 };
213 #endif
214
@@ -232,11 +226,11 @@ struct FP32Vec4 : public Vec<FP32Vec4> {
226
227 explicit FP32Vec4() : reg(vdupq_n_f32(0.0f)) {};
228
235 - explicit FP32Vec4(const float *ptr) : reg(vld1q_f32(ptr)) {};
229 + explicit FP32Vec4(const float* ptr) : reg(vld1q_f32(ptr)) {};
230
231 explicit FP32Vec4(float32x4_t data) : reg(data) {};
232
239 - explicit FP32Vec4(const FP32Vec4 &data) : reg(data.reg) {};
233 + explicit FP32Vec4(const FP32Vec4& data) : reg(data.reg) {};
234 };
235
236 struct FP32Vec8 : public Vec<FP32Vec8> {
@@ -252,32 +246,37 @@ struct FP32Vec8 : public Vec<FP32Vec8> {
246
247 explicit FP32Vec8() : reg({vmovq_n_f32(0.0), vmovq_n_f32(0.0)}) {};
248
255 - explicit FP32Vec8(const float *ptr) : reg({vld1q_f32(ptr), vld1q_f32(ptr + 4)}) {};
249 + explicit FP32Vec8(const float* ptr)
250 + : reg({vld1q_f32(ptr), vld1q_f32(ptr + 4)}) {};
251
252 explicit FP32Vec8(float32x4x2_t data) : reg(data) {};
253
259 - explicit FP32Vec8(const FP32Vec8 &data) : reg(data.reg) {};
254 + explicit FP32Vec8(const FP32Vec8& data) : reg(data.reg) {};
255
261 - explicit FP32Vec8(const FP16Vec8 &v) {
262 - reg.val[0] = vcvt_f32_f16(vget_low_f16(v.reg));
263 - reg.val[1] = vcvt_f32_f16(vget_high_f16(v.reg));
264 - };
256 + explicit FP32Vec8(const FP16Vec8& v) {
257 + reg.val[0] = vcvt_f32_f16(vget_low_f16(v.reg));
258 + reg.val[1] = vcvt_f32_f16(vget_high_f16(v.reg));
259 + };
260
266 - explicit FP32Vec8(float16x8_t v) : reg({vcvt_f32_f16(vget_low_f16(v)), vcvt_f32_f16(vget_high_f16(v))}) {};
261 + explicit FP32Vec8(float16x8_t v)
262 + : reg({vcvt_f32_f16(vget_low_f16(v)), vcvt_f32_f16(vget_high_f16(v))}) {};
263
268 - #ifdef ARM_BF16_SUPPORT
264 +#ifdef ARM_BF16_SUPPORT
265
270 - explicit FP32Vec8(bfloat16x8_t v) : reg({vcvtq_low_f32_bf16(v), vcvtq_high_f32_bf16(v)}) {};
266 + explicit FP32Vec8(bfloat16x8_t v)
267 + : reg({vcvtq_low_f32_bf16(v), vcvtq_high_f32_bf16(v)}) {};
268
272 - explicit FP32Vec8(const BF16Vec8 &v) : reg({vcvtq_low_f32_bf16(v.reg), vcvtq_high_f32_bf16(v.reg)}) {};
269 + explicit FP32Vec8(const BF16Vec8& v)
270 + : reg({vcvtq_low_f32_bf16(v.reg), vcvtq_high_f32_bf16(v.reg)}) {};
271
274 - #endif
272 +#endif
273
274 float reduce_sum() const {
275 AliasReg ar;
276 ar.reg = reg;
277 float answer = 0;
280 - unroll_loop<int, VEC_ELEM_NUM>([&answer, &ar](int i) { answer += ar.values[i]; });
278 + unroll_loop<int, VEC_ELEM_NUM>(
279 + [&answer, &ar](int i) { answer += ar.values[i]; });
280
281 return answer;
282 }
@@ -324,10 +323,14 @@ struct FP32Vec8 : public Vec<FP32Vec8> {
323 AliasReg ar;
324 ar.reg = reg;
325
327 - float32x2_t er_vec0 = {static_cast<float32_t>(erf(ar.values[0])), static_cast<float32_t>(erf(ar.values[1]))};
328 - float32x2_t er_vec1 = {static_cast<float32_t>(erf(ar.values[2])), static_cast<float32_t>(erf(ar.values[3]))};
329 - float32x2_t er_vec2 = {static_cast<float32_t>(erf(ar.values[4])), static_cast<float32_t>(erf(ar.values[5]))};
330 - float32x2_t er_vec3 = {static_cast<float32_t>(erf(ar.values[6])), static_cast<float32_t>(erf(ar.values[7]))};
326 + float32x2_t er_vec0 = {static_cast<float32_t>(erf(ar.values[0])),
327 + static_cast<float32_t>(erf(ar.values[1]))};
328 + float32x2_t er_vec1 = {static_cast<float32_t>(erf(ar.values[2])),
329 + static_cast<float32_t>(erf(ar.values[3]))};
330 + float32x2_t er_vec2 = {static_cast<float32_t>(erf(ar.values[4])),
331 + static_cast<float32_t>(erf(ar.values[5]))};
332 + float32x2_t er_vec3 = {static_cast<float32_t>(erf(ar.values[6])),
333 + static_cast<float32_t>(erf(ar.values[7]))};
334
335 float32x4_t result0 = vcombine_f32(er_vec0, er_vec1);
336 float32x4_t result1 = vcombine_f32(er_vec2, er_vec3);
@@ -337,25 +340,29 @@ struct FP32Vec8 : public Vec<FP32Vec8> {
340 result.val[1] = result1;
341
342 return FP32Vec8(result);
340 - }
343 + }
344
342 - FP32Vec8 operator*(const FP32Vec8 &b) const {
343 - return FP32Vec8(float32x4x2_t({vmulq_f32(reg.val[0], b.reg.val[0]), vmulq_f32(reg.val[1], b.reg.val[1])}));
345 + FP32Vec8 operator*(const FP32Vec8& b) const {
346 + return FP32Vec8(float32x4x2_t({vmulq_f32(reg.val[0], b.reg.val[0]),
347 + vmulq_f32(reg.val[1], b.reg.val[1])}));
348 }
349
346 - FP32Vec8 operator+(const FP32Vec8 &b) const {
347 - return FP32Vec8(float32x4x2_t({vaddq_f32(reg.val[0], b.reg.val[0]), vaddq_f32(reg.val[1], b.reg.val[1])}));
350 + FP32Vec8 operator+(const FP32Vec8& b) const {
351 + return FP32Vec8(float32x4x2_t({vaddq_f32(reg.val[0], b.reg.val[0]),
352 + vaddq_f32(reg.val[1], b.reg.val[1])}));
353 }
354
350 - FP32Vec8 operator-(const FP32Vec8 &b) const {
351 - return FP32Vec8(float32x4x2_t({vsubq_f32(reg.val[0], b.reg.val[0]), vsubq_f32(reg.val[1], b.reg.val[1])}));
355 + FP32Vec8 operator-(const FP32Vec8& b) const {
356 + return FP32Vec8(float32x4x2_t({vsubq_f32(reg.val[0], b.reg.val[0]),
357 + vsubq_f32(reg.val[1], b.reg.val[1])}));
358 }
359
354 - FP32Vec8 operator/(const FP32Vec8 &b) const {
355 - return FP32Vec8(float32x4x2_t({vdivq_f32(reg.val[0], b.reg.val[0]), vdivq_f32(reg.val[1], b.reg.val[1])}));
360 + FP32Vec8 operator/(const FP32Vec8& b) const {
361 + return FP32Vec8(float32x4x2_t({vdivq_f32(reg.val[0], b.reg.val[0]),
362 + vdivq_f32(reg.val[1], b.reg.val[1])}));
363 }
364
358 - void save(float *ptr) const {
365 + void save(float* ptr) const {
366 vst1q_f32(ptr, reg.val[0]);
367 vst1q_f32(ptr + 4, reg.val[1]);
368 }
@@ -370,103 +377,100 @@ struct FP32Vec16 : public Vec<FP32Vec16> {
377
378 float32x4x4_t reg;
379
373 - explicit FP32Vec16(float v) : reg({vmovq_n_f32(v), vmovq_n_f32(v), vmovq_n_f32(v), vmovq_n_f32(v)}) {}
380 + explicit FP32Vec16(float v)
381 + : reg({vmovq_n_f32(v), vmovq_n_f32(v), vmovq_n_f32(v), vmovq_n_f32(v)}) {}
382
375 - explicit FP32Vec16() : reg({vmovq_n_f32(0.0), vmovq_n_f32(0.0), vmovq_n_f32(0.0), vmovq_n_f32(0.0)}) {}
383 + explicit FP32Vec16()
384 + : reg({vmovq_n_f32(0.0), vmovq_n_f32(0.0), vmovq_n_f32(0.0),
385 + vmovq_n_f32(0.0)}) {}
386
377 - explicit FP32Vec16(const float *ptr) : reg({vld1q_f32(ptr), vld1q_f32(ptr + 4), vld1q_f32(ptr + 8), vld1q_f32(ptr + 12)}) {}
387 + explicit FP32Vec16(const float* ptr)
388 + : reg({vld1q_f32(ptr), vld1q_f32(ptr + 4), vld1q_f32(ptr + 8),
389 + vld1q_f32(ptr + 12)}) {}
390
391 explicit FP32Vec16(float32x4x4_t data) : reg(data) {}
392
381 - explicit FP32Vec16(const FP32Vec8 &data) {
382 - reg.val[0] = data.reg.val[0];
383 - reg.val[1] = data.reg.val[1];
384 - reg.val[2] = data.reg.val[0];
385 - reg.val[3] = data.reg.val[1];
393 + explicit FP32Vec16(const FP32Vec8& data) {
394 + reg.val[0] = data.reg.val[0];
395 + reg.val[1] = data.reg.val[1];
396 + reg.val[2] = data.reg.val[0];
397 + reg.val[3] = data.reg.val[1];
398 }
399
388 - explicit FP32Vec16(const FP32Vec16 &data) : reg(data.reg) {}
400 + explicit FP32Vec16(const FP32Vec16& data) : reg(data.reg) {}
401
390 - explicit FP32Vec16(const FP16Vec8 &v) : FP32Vec16(FP32Vec8(v.reg)) {}
402 + explicit FP32Vec16(const FP16Vec8& v) : FP32Vec16(FP32Vec8(v.reg)) {}
403
392 - #ifdef ARM_BF16_SUPPORT
393 - explicit FP32Vec16(bfloat16x8x2_t v) : reg({
394 - vcvtq_low_f32_bf16(v.val[0]),
395 - vcvtq_high_f32_bf16(v.val[0]),
396 - vcvtq_low_f32_bf16(v.val[1]),
397 - vcvtq_high_f32_bf16(v.val[1])
398 - }) {};
399 - #endif
404 +#ifdef ARM_BF16_SUPPORT
405 + explicit FP32Vec16(bfloat16x8x2_t v)
406 + : reg({vcvtq_low_f32_bf16(v.val[0]), vcvtq_high_f32_bf16(v.val[0]),
407 + vcvtq_low_f32_bf16(v.val[1]), vcvtq_high_f32_bf16(v.val[1])}) {};
408 +#endif
409
401 - explicit FP32Vec16(const FP32Vec4 &data) {
410 + explicit FP32Vec16(const FP32Vec4& data) {
411 reg.val[0] = data.reg;
412 reg.val[1] = data.reg;
413 reg.val[2] = data.reg;
414 reg.val[3] = data.reg;
415 };
416
408 - #ifdef ARM_BF16_SUPPORT
409 - explicit FP32Vec16(const BF16Vec16 &v) : reg({
410 - vcvtq_low_f32_bf16(v.reg.val[0]),
411 - vcvtq_high_f32_bf16(v.reg.val[0]),
412 - vcvtq_low_f32_bf16(v.reg.val[1]),
413 - vcvtq_high_f32_bf16(v.reg.val[1])
414 - }) {};
415 -
416 - explicit FP32Vec16(const BF16Vec8 &v) : FP32Vec16(FP32Vec8(v)) {};
417 - #endif
418 -
419 - explicit FP32Vec16(const FP16Vec16 &v) {
420 - reg.val[0] = vcvt_f32_f16(vget_low_f16(v.reg.val[0]));
421 - reg.val[1] = vcvt_f32_f16(vget_high_f16(v.reg.val[0]));
422 - reg.val[2] = vcvt_f32_f16(vget_low_f16(v.reg.val[1]));
423 - reg.val[3] = vcvt_f32_f16(vget_high_f16(v.reg.val[1]));
417 +#ifdef ARM_BF16_SUPPORT
418 + explicit FP32Vec16(const BF16Vec16& v)
419 + : reg({vcvtq_low_f32_bf16(v.reg.val[0]),
420 + vcvtq_high_f32_bf16(v.reg.val[0]),
421 + vcvtq_low_f32_bf16(v.reg.val[1]),
422 + vcvtq_high_f32_bf16(v.reg.val[1])}) {};
423 +
424 + explicit FP32Vec16(const BF16Vec8& v) : FP32Vec16(FP32Vec8(v)) {};
425 +#endif
426 +
427 + explicit FP32Vec16(const FP16Vec16& v) {
428 + reg.val[0] = vcvt_f32_f16(vget_low_f16(v.reg.val[0]));
429 + reg.val[1] = vcvt_f32_f16(vget_high_f16(v.reg.val[0]));
430 + reg.val[2] = vcvt_f32_f16(vget_low_f16(v.reg.val[1]));
431 + reg.val[3] = vcvt_f32_f16(vget_high_f16(v.reg.val[1]));
432 };
433
426 - FP32Vec16 operator+(const FP32Vec16 &b) const {
427 - return FP32Vec16(float32x4x4_t({
428 - vaddq_f32(reg.val[0], b.reg.val[0]),
429 - vaddq_f32(reg.val[1], b.reg.val[1]),
430 - vaddq_f32(reg.val[2], b.reg.val[2]),
431 - vaddq_f32(reg.val[3], b.reg.val[3])}));
434 + FP32Vec16 operator+(const FP32Vec16& b) const {
435 + return FP32Vec16(float32x4x4_t({vaddq_f32(reg.val[0], b.reg.val[0]),
436 + vaddq_f32(reg.val[1], b.reg.val[1]),
437 + vaddq_f32(reg.val[2], b.reg.val[2]),
438 + vaddq_f32(reg.val[3], b.reg.val[3])}));
439 };
440
434 - FP32Vec16 operator*(const FP32Vec16 &b) const {
435 - return FP32Vec16(float32x4x4_t({
436 - vmulq_f32(reg.val[0], b.reg.val[0]),
437 - vmulq_f32(reg.val[1], b.reg.val[1]),
438 - vmulq_f32(reg.val[2], b.reg.val[2]),
439 - vmulq_f32(reg.val[3], b.reg.val[3])}));
441 + FP32Vec16 operator*(const FP32Vec16& b) const {
442 + return FP32Vec16(float32x4x4_t({vmulq_f32(reg.val[0], b.reg.val[0]),
443 + vmulq_f32(reg.val[1], b.reg.val[1]),
444 + vmulq_f32(reg.val[2], b.reg.val[2]),
445 + vmulq_f32(reg.val[3], b.reg.val[3])}));
446 };
447
442 - FP32Vec16 operator-(const FP32Vec16 &b) const {
443 - return FP32Vec16(float32x4x4_t({
444 - vsubq_f32(reg.val[0], b.reg.val[0]),
445 - vsubq_f32(reg.val[1], b.reg.val[1]),
446 - vsubq_f32(reg.val[2], b.reg.val[2]),
447 - vsubq_f32(reg.val[3], b.reg.val[3])
448 - }));
448 + FP32Vec16 operator-(const FP32Vec16& b) const {
449 + return FP32Vec16(float32x4x4_t({vsubq_f32(reg.val[0], b.reg.val[0]),
450 + vsubq_f32(reg.val[1], b.reg.val[1]),
451 + vsubq_f32(reg.val[2], b.reg.val[2]),
452 + vsubq_f32(reg.val[3], b.reg.val[3])}));
453 };
454
451 - FP32Vec16 operator/(const FP32Vec16 &b) const {
452 - return FP32Vec16(float32x4x4_t({
453 - vdivq_f32(reg.val[0], b.reg.val[0]),
454 - vdivq_f32(reg.val[1], b.reg.val[1]),
455 - vdivq_f32(reg.val[2], b.reg.val[2]),
456 - vdivq_f32(reg.val[3], b.reg.val[3])
457 - }));
455 + FP32Vec16 operator/(const FP32Vec16& b) const {
456 + return FP32Vec16(float32x4x4_t({vdivq_f32(reg.val[0], b.reg.val[0]),
457 + vdivq_f32(reg.val[1], b.reg.val[1]),
458 + vdivq_f32(reg.val[2], b.reg.val[2]),
459 + vdivq_f32(reg.val[3], b.reg.val[3])}));
460 };
461
462 float reduce_sum() const {
463 AliasReg ar;
464 ar.reg = reg;
465 float answer = 0;
464 - unroll_loop<int, VEC_ELEM_NUM>([&answer, &ar](int i) { answer += ar.values[i]; });
466 + unroll_loop<int, VEC_ELEM_NUM>(
467 + [&answer, &ar](int i) { answer += ar.values[i]; });
468
469 return answer;
470 };
471
469 - template <int group_size> float reduce_sub_sum(int idx) {
472 + template <int group_size>
473 + float reduce_sub_sum(int idx) {
474 static_assert(VEC_ELEM_NUM % group_size == 0);
475
476 AliasReg ar;
@@ -479,7 +483,7 @@ struct FP32Vec16 : public Vec<FP32Vec16> {
483 return answer;
484 };
485
482 - void save(float *ptr) const {
486 + void save(float* ptr) const {
487 vst1q_f32(ptr, reg.val[0]);
488 vst1q_f32(ptr + 4, reg.val[1]);
489 vst1q_f32(ptr + 8, reg.val[2]);
@@ -487,43 +491,59 @@ struct FP32Vec16 : public Vec<FP32Vec16> {
491 };
492 };
493
490 -template <typename T> struct VecType { using vec_type = void; };
494 +template <typename T>
495 +struct VecType {
496 + using vec_type = void;
497 +};
498
492 -template <typename T> using vec_t = typename VecType<T>::vec_type;
499 +template <typename T>
500 +using vec_t = typename VecType<T>::vec_type;
501
494 -template <> struct VecType<float> { using vec_type = FP32Vec8; };
502 +template <>
503 +struct VecType<float> {
504 + using vec_type = FP32Vec8;
505 +};
506
496 -template <> struct VecType<c10::Half> { using vec_type = FP16Vec8; };
507 +template <>
508 +struct VecType<c10::Half> {
509 + using vec_type = FP16Vec8;
510 +};
511
512 #ifdef ARM_BF16_SUPPORT
499 -template <> struct VecType<c10::BFloat16> { using vec_type = BF16Vec8; };
513 +template <>
514 +struct VecType<c10::BFloat16> {
515 + using vec_type = BF16Vec8;
516 +};
517 #endif
518
502 -template <typename T> void storeFP32(float v, T *ptr) { *ptr = v; }
519 +template <typename T>
520 +void storeFP32(float v, T* ptr) {
521 + *ptr = v;
522 +}
523
504 -template <> inline void storeFP32<c10::Half>(float v, c10::Half *ptr) {
505 - *reinterpret_cast<__fp16 *>(ptr) = v;
524 +template <>
525 +inline void storeFP32<c10::Half>(float v, c10::Half* ptr) {
526 + *reinterpret_cast<__fp16*>(ptr) = v;
527 }
528
508 -inline FP16Vec16::FP16Vec16(const FP32Vec16 &v) {
509 - float16x4_t low_0 = vcvt_f16_f32(v.reg.val[0]);
510 - float16x4_t high_0 = vcvt_f16_f32(v.reg.val[1]);
511 - float16x4_t low_1 = vcvt_f16_f32(v.reg.val[2]);
512 - float16x4_t high_1 = vcvt_f16_f32(v.reg.val[3]);
529 +inline FP16Vec16::FP16Vec16(const FP32Vec16& v) {
530 + float16x4_t low_0 = vcvt_f16_f32(v.reg.val[0]);
531 + float16x4_t high_0 = vcvt_f16_f32(v.reg.val[1]);
532 + float16x4_t low_1 = vcvt_f16_f32(v.reg.val[2]);
533 + float16x4_t high_1 = vcvt_f16_f32(v.reg.val[3]);
534
514 - reg.val[0] = vcombine_f16(low_0, high_0);
515 - reg.val[1] = vcombine_f16(low_1, high_1);
535 + reg.val[0] = vcombine_f16(low_0, high_0);
536 + reg.val[1] = vcombine_f16(low_1, high_1);
537 };
538
518 -inline FP16Vec8 :: FP16Vec8(const FP32Vec8 &v) {
519 - float16x4_t lower_half = vcvt_f16_f32(v.reg.val[0]);
520 - float16x4_t upper_half = vcvt_f16_f32(v.reg.val[1]);
539 +inline FP16Vec8 ::FP16Vec8(const FP32Vec8& v) {
540 + float16x4_t lower_half = vcvt_f16_f32(v.reg.val[0]);
541 + float16x4_t upper_half = vcvt_f16_f32(v.reg.val[1]);
542
522 - reg = vcombine_f16(lower_half, upper_half);
543 + reg = vcombine_f16(lower_half, upper_half);
544 };
545
525 -inline void fma(FP32Vec16 &acc, FP32Vec16 &a, FP32Vec16 &b) {
526 -
546 +inline void fma(FP32Vec16& acc, FP32Vec16& a, FP32Vec16& b) {
547 acc.reg.val[0] = vfmaq_f32(acc.reg.val[0], a.reg.val[0], b.reg.val[0]);
548 acc.reg.val[1] = vfmaq_f32(acc.reg.val[1], a.reg.val[1], b.reg.val[1]);
549 acc.reg.val[2] = vfmaq_f32(acc.reg.val[2], a.reg.val[2], b.reg.val[2]);
@@ -531,8 +551,7 @@ inline void fma(FP32Vec16 &acc, FP32Vec16 &a, FP32Vec16 &b) {
551 };
552
553 #ifdef ARM_BF16_SUPPORT
534 -inline void fma(FP32Vec16 &acc, BF16Vec32 &a, BF16Vec32 &b) {
535 -
554 +inline void fma(FP32Vec16& acc, BF16Vec32& a, BF16Vec32& b) {
555 float32x4_t a0_low = vcvt_f32_bf16(vget_low_bf16(a.reg.val[0]));
556 float32x4_t a0_high = vcvt_f32_bf16(vget_high_bf16(a.reg.val[0]));
557 float32x4_t a1_low = vcvt_f32_bf16(vget_low_bf16(a.reg.val[1]));
@@ -551,22 +570,22 @@ inline void fma(FP32Vec16 &acc, BF16Vec32 &a, BF16Vec32 &b) {
570 #endif
571
572 #ifdef ARM_BF16_SUPPORT
554 -inline BF16Vec8::BF16Vec8(const FP32Vec8 &v) : reg(vcvtq_high_bf16_f32(vcvtq_low_bf16_f32(v.reg.val[0]), v.reg.val[1])) {};
555 -
556 -inline BF16Vec16::BF16Vec16(const FP32Vec16 &v) : reg({
557 - vcvtq_high_bf16_f32(vcvtq_low_bf16_f32(v.reg.val[0]), v.reg.val[1]),
558 - vcvtq_high_bf16_f32(vcvtq_low_bf16_f32(v.reg.val[2]), v.reg.val[3])
559 - }){};
573 +inline BF16Vec8::BF16Vec8(const FP32Vec8& v)
574 + : reg(vcvtq_high_bf16_f32(vcvtq_low_bf16_f32(v.reg.val[0]), v.reg.val[1])) {
575 + };
576 +
577 +inline BF16Vec16::BF16Vec16(const FP32Vec16& v)
578 + : reg({vcvtq_high_bf16_f32(vcvtq_low_bf16_f32(v.reg.val[0]), v.reg.val[1]),
579 + vcvtq_high_bf16_f32(vcvtq_low_bf16_f32(v.reg.val[2]),
580 + v.reg.val[3])}) {};
581 #endif
582
562 -inline void prefetch(const void *addr) {
563 - __builtin_prefetch(addr, 0, 1);
564 -};
583 +inline void prefetch(const void* addr) { __builtin_prefetch(addr, 0, 1); };
584
585 #ifdef ARM_BF16_SUPPORT
586 template <>
568 -inline void storeFP32<c10::BFloat16>(float v, c10::BFloat16 *ptr) {
569 - *reinterpret_cast<__bf16 *>(ptr) = vcvth_bf16_f32(v);
587 +inline void storeFP32<c10::BFloat16>(float v, c10::BFloat16* ptr) {
588 + *reinterpret_cast<__bf16*>(ptr) = vcvth_bf16_f32(v);
589 };
590 #endif
572 -};
\ No newline at end of file
591 +}; // namespace vec_op
\ No newline at end of file
csrc/cpu/cpu_types_vsx.hpp
+143 -111
@@ -9,38 +9,40 @@
9 namespace vec_op {
10
11 // FIXME: FP16 is not fully supported in Torch-CPU
12 -#define VLLM_DISPATCH_CASE_FLOATING_TYPES(...) \
13 - AT_DISPATCH_CASE(at::ScalarType::Float, __VA_ARGS__) \
12 +#define VLLM_DISPATCH_CASE_FLOATING_TYPES(...) \
13 + AT_DISPATCH_CASE(at::ScalarType::Float, __VA_ARGS__) \
14 AT_DISPATCH_CASE(at::ScalarType::BFloat16, __VA_ARGS__)
15
16 -#define VLLM_DISPATCH_FLOATING_TYPES(TYPE, NAME, ...) \
16 +#define VLLM_DISPATCH_FLOATING_TYPES(TYPE, NAME, ...) \
17 AT_DISPATCH_SWITCH(TYPE, NAME, VLLM_DISPATCH_CASE_FLOATING_TYPES(__VA_ARGS__))
18
19 #ifndef CPU_OP_GUARD
20 -#define CPU_KERNEL_GUARD_IN(NAME)
21 -#define CPU_KERNEL_GUARD_OUT(NAME)
20 + #define CPU_KERNEL_GUARD_IN(NAME)
21 + #define CPU_KERNEL_GUARD_OUT(NAME)
22 #else
23 -#define CPU_KERNEL_GUARD_IN(NAME) \
24 - std::cout << #NAME << " invoked." << std::endl;
25 -#define CPU_KERNEL_GUARD_OUT(NAME) std::cout << #NAME << " exit." << std::endl;
23 + #define CPU_KERNEL_GUARD_IN(NAME) \
24 + std::cout << #NAME << " invoked." << std::endl;
25 + #define CPU_KERNEL_GUARD_OUT(NAME) \
26 + std::cout << #NAME << " exit." << std::endl;
27 #endif
28
29 #define FORCE_INLINE __attribute__((always_inline)) inline
30
31 namespace {
32 template <typename T, T... indexes, typename F>
32 -constexpr void unroll_loop_item(std::integer_sequence<T, indexes...>, F &&f) {
33 +constexpr void unroll_loop_item(std::integer_sequence<T, indexes...>, F&& f) {
34 (f(std::integral_constant<T, indexes>{}), ...);
35 }
35 -}; // namespace
36 +}; // namespace
37
38 template <typename T, T count, typename F,
39 typename = std::enable_if_t<std::is_invocable_v<F, T>>>
39 -constexpr void unroll_loop(F &&f) {
40 +constexpr void unroll_loop(F&& f) {
41 unroll_loop_item(std::make_integer_sequence<T, count>{}, std::forward<F>(f));
42 }
43
43 -template <typename T> struct Vec {
44 +template <typename T>
45 +struct Vec {
46 constexpr static int get_elem_num() { return T::VEC_ELEM_NUM; }
47 };
48
@@ -68,12 +70,14 @@ struct BF16Vec8 : public Vec<BF16Vec8> {
70
71 __vector signed short reg;
72
71 - explicit BF16Vec8(const void *ptr)
72 - : reg((__vector signed short)vec_xl(0, (__vector signed short *)ptr)) {}
73 + explicit BF16Vec8(const void* ptr)
74 + : reg((__vector signed short)vec_xl(0, (__vector signed short*)ptr)) {}
75
74 - explicit BF16Vec8(const FP32Vec8 &);
76 + explicit BF16Vec8(const FP32Vec8&);
77
76 - void save(void *ptr) const { *reinterpret_cast<__vector signed short *>(ptr) = reg; }
78 + void save(void* ptr) const {
79 + *reinterpret_cast<__vector signed short*>(ptr) = reg;
80 + }
81 };
82
83 struct BF16Vec16 : public Vec<BF16Vec16> {
@@ -81,18 +85,18 @@ struct BF16Vec16 : public Vec<BF16Vec16> {
85
86 ss16x8x2_t reg;
87
84 - explicit BF16Vec16(const void *ptr) {
88 + explicit BF16Vec16(const void* ptr) {
89 // Load 256 bits in two parts
86 - reg.val[0] = (__vector signed short)vec_xl(0, (signed short *)ptr);
87 - reg.val[1] = (__vector signed short)vec_xl(16, (signed short *)ptr);
90 + reg.val[0] = (__vector signed short)vec_xl(0, (signed short*)ptr);
91 + reg.val[1] = (__vector signed short)vec_xl(16, (signed short*)ptr);
92 }
93
90 - explicit BF16Vec16(const FP32Vec16 &);
94 + explicit BF16Vec16(const FP32Vec16&);
95
92 - void save(void *ptr) const {
96 + void save(void* ptr) const {
97 // Save 256 bits in two parts
94 - vec_xst(reg.val[0], 0, (signed short *)ptr);
95 - vec_xst(reg.val[1], 16, (signed short *)ptr);
98 + vec_xst(reg.val[0], 0, (signed short*)ptr);
99 + vec_xst(reg.val[1], 16, (signed short*)ptr);
100 }
101 };
102
@@ -102,19 +106,15 @@ struct BF16Vec32 : public Vec<BF16Vec32> {
106 constexpr static int VEC_ELEM_NUM = 32;
107
108 ss16x8x4_t reg;
105 - explicit BF16Vec32(const void *ptr)
106 - : reg(*reinterpret_cast<const ss16x8x4_t *>(ptr)) {}
109 + explicit BF16Vec32(const void* ptr)
110 + : reg(*reinterpret_cast<const ss16x8x4_t*>(ptr)) {}
111
112 explicit BF16Vec32(ss16x8x4_t data) : reg(data) {}
113
110 - explicit BF16Vec32(const BF16Vec8 &vec8_data) : reg({
111 - vec8_data.reg,
112 - vec8_data.reg,
113 - vec8_data.reg,
114 - vec8_data.reg
115 - }) {}
114 + explicit BF16Vec32(const BF16Vec8& vec8_data)
115 + : reg({vec8_data.reg, vec8_data.reg, vec8_data.reg, vec8_data.reg}) {}
116
117 - void save(void *ptr) const { *reinterpret_cast<ss16x8x4_t *>(ptr) = reg; }
117 + void save(void* ptr) const { *reinterpret_cast<ss16x8x4_t*>(ptr) = reg; }
118 };
119
120 struct FP32Vec4 : public Vec<FP32Vec4> {
@@ -130,11 +130,11 @@ struct FP32Vec4 : public Vec<FP32Vec4> {
130
131 explicit FP32Vec4() : reg(vec_splats(0.0f)) {}
132
133 - explicit FP32Vec4(const float *ptr) : reg(vec_xl(0, ptr)) {}
133 + explicit FP32Vec4(const float* ptr) : reg(vec_xl(0, ptr)) {}
134
135 explicit FP32Vec4(__vector float data) : reg(data) {}
136
137 - explicit FP32Vec4(const FP32Vec4 &data) : reg(data.reg) {}
137 + explicit FP32Vec4(const FP32Vec4& data) : reg(data.reg) {}
138 };
139
140 struct FP32Vec8 : public Vec<FP32Vec8> {
@@ -156,19 +156,19 @@ struct FP32Vec8 : public Vec<FP32Vec8> {
156 reg.val[1] = vec_splats(0.0f);
157 }
158
159 - explicit FP32Vec8(const float *ptr) {
159 + explicit FP32Vec8(const float* ptr) {
160 reg.val[0] = vec_xl(0, ptr);
161 reg.val[1] = vec_xl(16, ptr);
162 }
163
164 explicit FP32Vec8(f32x4x2_t data) : reg(data) {}
165
166 - explicit FP32Vec8(const FP32Vec8 &data) {
166 + explicit FP32Vec8(const FP32Vec8& data) {
167 reg.val[0] = data.reg.val[0];
168 reg.val[1] = data.reg.val[1];
169 }
170
171 - explicit FP32Vec8(const BF16Vec8 &v) {
171 + explicit FP32Vec8(const BF16Vec8& v) {
172 reg.val[0] = (__vector float)vec_mergeh(zero, v.reg);
173 reg.val[1] = (__vector float)vec_mergel(zero, v.reg);
174 }
@@ -177,7 +177,8 @@ struct FP32Vec8 : public Vec<FP32Vec8> {
177 AliasReg ar;
178 ar.reg = reg;
179 float result = 0;
180 - unroll_loop<int, VEC_ELEM_NUM>([&result, &ar](int i) { result += ar.values[i]; });
180 + unroll_loop<int, VEC_ELEM_NUM>(
181 + [&result, &ar](int i) { result += ar.values[i]; });
182
183 return result;
184 }
@@ -230,23 +231,27 @@ struct FP32Vec8 : public Vec<FP32Vec8> {
231 return FP32Vec8(f32x4x2_t({ret.val[0], ret.val[1]}));
232 }
233
233 - FP32Vec8 operator*(const FP32Vec8 &b) const {
234 - return FP32Vec8({vec_mul(reg.val[0], b.reg.val[0]), vec_mul(reg.val[1], b.reg.val[1])});
234 + FP32Vec8 operator*(const FP32Vec8& b) const {
235 + return FP32Vec8(
236 + {vec_mul(reg.val[0], b.reg.val[0]), vec_mul(reg.val[1], b.reg.val[1])});
237 }
238
237 - FP32Vec8 operator+(const FP32Vec8 &b) const {
238 - return FP32Vec8({vec_add(reg.val[0], b.reg.val[0]), vec_add(reg.val[1], b.reg.val[1])});
239 + FP32Vec8 operator+(const FP32Vec8& b) const {
240 + return FP32Vec8(
241 + {vec_add(reg.val[0], b.reg.val[0]), vec_add(reg.val[1], b.reg.val[1])});
242 }
243
241 - FP32Vec8 operator-(const FP32Vec8 &b) const {
242 - return FP32Vec8({vec_sub(reg.val[0], b.reg.val[0]), vec_sub(reg.val[1], b.reg.val[1])});
244 + FP32Vec8 operator-(const FP32Vec8& b) const {
245 + return FP32Vec8(
246 + {vec_sub(reg.val[0], b.reg.val[0]), vec_sub(reg.val[1], b.reg.val[1])});
247 }
248
245 - FP32Vec8 operator/(const FP32Vec8 &b) const {
246 - return FP32Vec8({vec_div(reg.val[0], b.reg.val[0]), vec_div(reg.val[1], b.reg.val[1])});
249 + FP32Vec8 operator/(const FP32Vec8& b) const {
250 + return FP32Vec8(
251 + {vec_div(reg.val[0], b.reg.val[0]), vec_div(reg.val[1], b.reg.val[1])});
252 }
253
249 - void save(float *ptr) const {
254 + void save(float* ptr) const {
255 vec_xst(reg.val[0], 0, ptr);
256 vec_xst(reg.val[1], 16, ptr);
257 }
@@ -275,7 +280,7 @@ struct FP32Vec16 : public Vec<FP32Vec16> {
280 reg.val[3] = vec_splats(0.0f);
281 }
282
278 - explicit FP32Vec16(const float *ptr) {
283 + explicit FP32Vec16(const float* ptr) {
284 reg.val[0] = vec_xl(0, ptr);
285 reg.val[1] = vec_xl(16, ptr);
286 reg.val[2] = vec_xl(32, ptr);
@@ -284,78 +289,76 @@ struct FP32Vec16 : public Vec<FP32Vec16> {
289
290 explicit FP32Vec16(f32x4x4_t data) : reg(data) {}
291
287 - explicit FP32Vec16(const FP32Vec16 &data) {
292 + explicit FP32Vec16(const FP32Vec16& data) {
293 reg.val[0] = data.reg.val[0];
294 reg.val[1] = data.reg.val[1];
295 reg.val[2] = data.reg.val[2];
296 reg.val[3] = data.reg.val[3];
297 }
298
294 - explicit FP32Vec16(const FP32Vec4 &data) {
299 + explicit FP32Vec16(const FP32Vec4& data) {
300 reg.val[0] = data.reg;
301 reg.val[1] = data.reg;
302 reg.val[2] = data.reg;
303 reg.val[3] = data.reg;
304 }
305
301 - explicit FP32Vec16(const FP32Vec8 &data) {
306 + explicit FP32Vec16(const FP32Vec8& data) {
307 reg.val[0] = data.reg.val[0];
308 reg.val[1] = data.reg.val[1];
309 reg.val[2] = data.reg.val[0];
310 reg.val[3] = data.reg.val[1];
311 }
312
308 - explicit FP32Vec16(const BF16Vec16 &v) {
313 + explicit FP32Vec16(const BF16Vec16& v) {
314 reg.val[0] = (__vector float)vec_mergeh(zero, v.reg.val[0]);
315 reg.val[1] = (__vector float)vec_mergel(zero, v.reg.val[0]);
316 reg.val[2] = (__vector float)vec_mergeh(zero, v.reg.val[1]);
317 reg.val[3] = (__vector float)vec_mergel(zero, v.reg.val[1]);
318 }
319
315 - explicit FP32Vec16(const BF16Vec8 &v) : FP32Vec16(FP32Vec8(v)) {}
320 + explicit FP32Vec16(const BF16Vec8& v) : FP32Vec16(FP32Vec8(v)) {}
321
317 - FP32Vec16 operator*(const FP32Vec16 &b) const {
318 - return FP32Vec16(f32x4x4_t({
319 - vec_mul(reg.val[0], b.reg.val[0]),
320 - vec_mul(reg.val[1], b.reg.val[1]),
321 - vec_mul(reg.val[2], b.reg.val[2]),
322 - vec_mul(reg.val[3], b.reg.val[3])}));
322 + FP32Vec16 operator*(const FP32Vec16& b) const {
323 + return FP32Vec16(f32x4x4_t({vec_mul(reg.val[0], b.reg.val[0]),
324 + vec_mul(reg.val[1], b.reg.val[1]),
325 + vec_mul(reg.val[2], b.reg.val[2]),
326 + vec_mul(reg.val[3], b.reg.val[3])}));
327 }
328
325 - FP32Vec16 operator+(const FP32Vec16 &b) const {
326 - return FP32Vec16(f32x4x4_t({
327 - vec_add(reg.val[0], b.reg.val[0]),
328 - vec_add(reg.val[1], b.reg.val[1]),
329 - vec_add(reg.val[2], b.reg.val[2]),
330 - vec_add(reg.val[3], b.reg.val[3])}));
329 + FP32Vec16 operator+(const FP32Vec16& b) const {
330 + return FP32Vec16(f32x4x4_t({vec_add(reg.val[0], b.reg.val[0]),
331 + vec_add(reg.val[1], b.reg.val[1]),
332 + vec_add(reg.val[2], b.reg.val[2]),
333 + vec_add(reg.val[3], b.reg.val[3])}));
334 }
335
333 - FP32Vec16 operator-(const FP32Vec16 &b) const {
334 - return FP32Vec16(f32x4x4_t({
335 - vec_sub(reg.val[0], b.reg.val[0]),
336 - vec_sub(reg.val[1], b.reg.val[1]),
337 - vec_sub(reg.val[2], b.reg.val[2]),
338 - vec_sub(reg.val[3], b.reg.val[3])}));
336 + FP32Vec16 operator-(const FP32Vec16& b) const {
337 + return FP32Vec16(f32x4x4_t({vec_sub(reg.val[0], b.reg.val[0]),
338 + vec_sub(reg.val[1], b.reg.val[1]),
339 + vec_sub(reg.val[2], b.reg.val[2]),
340 + vec_sub(reg.val[3], b.reg.val[3])}));
341 }
342
341 - FP32Vec16 operator/(const FP32Vec16 &b) const {
342 - return FP32Vec16(f32x4x4_t({
343 - vec_div(reg.val[0], b.reg.val[0]),
344 - vec_div(reg.val[1], b.reg.val[1]),
345 - vec_div(reg.val[2], b.reg.val[2]),
346 - vec_div(reg.val[3], b.reg.val[3])}));
343 + FP32Vec16 operator/(const FP32Vec16& b) const {
344 + return FP32Vec16(f32x4x4_t({vec_div(reg.val[0], b.reg.val[0]),
345 + vec_div(reg.val[1], b.reg.val[1]),
346 + vec_div(reg.val[2], b.reg.val[2]),
347 + vec_div(reg.val[3], b.reg.val[3])}));
348 }
349
350 float reduce_sum() const {
351 AliasReg ar;
352 ar.reg = reg;
353 float result = 0;
353 - unroll_loop<int, VEC_ELEM_NUM>([&result, &ar](int i) { result += ar.values[i]; });
354 + unroll_loop<int, VEC_ELEM_NUM>(
355 + [&result, &ar](int i) { result += ar.values[i]; });
356
357 return result;
358 }
359
358 - template <int group_size> float reduce_sub_sum(int idx) {
360 + template <int group_size>
361 + float reduce_sub_sum(int idx) {
362 static_assert(VEC_ELEM_NUM % group_size == 0);
363
364 AliasReg ar;
@@ -368,7 +371,7 @@ struct FP32Vec16 : public Vec<FP32Vec16> {
371 return result;
372 }
373
371 - void save(float *ptr) const {
374 + void save(float* ptr) const {
375 vec_xst(reg.val[0], 0, ptr);
376 vec_xst(reg.val[1], 16, ptr);
377 vec_xst(reg.val[2], 32, ptr);
@@ -376,43 +379,62 @@ struct FP32Vec16 : public Vec<FP32Vec16> {
379 }
380 };
381
379 -template <typename T> struct VecType { using vec_type = void; };
382 +template <typename T>
383 +struct VecType {
384 + using vec_type = void;
385 +};
386
381 -template <typename T> using vec_t = typename VecType<T>::vec_type;
387 +template <typename T>
388 +using vec_t = typename VecType<T>::vec_type;
389
383 -template <> struct VecType<float> { using vec_type = FP32Vec8; };
390 +template <>
391 +struct VecType<float> {
392 + using vec_type = FP32Vec8;
393 +};
394
385 -template <> struct VecType<c10::BFloat16> { using vec_type = BF16Vec8; };
395 +template <>
396 +struct VecType<c10::BFloat16> {
397 + using vec_type = BF16Vec8;
398 +};
399
387 -template <typename T> void storeFP32(float v, T *ptr) { *ptr = v; }
400 +template <typename T>
401 +void storeFP32(float v, T* ptr) {
402 + *ptr = v;
403 +}
404
389 -inline void fma(FP32Vec16 &acc, FP32Vec16 &a, FP32Vec16 &b) {
405 +inline void fma(FP32Vec16& acc, FP32Vec16& a, FP32Vec16& b) {
406 acc = acc + a * b;
407 }
408
393 -template <> inline void storeFP32<c10::BFloat16>(float v, c10::BFloat16 *ptr) {
394 - c10::BFloat16 __attribute__((__may_alias__)) *v_ptr =
395 - reinterpret_cast<c10::BFloat16 *>(&v);
409 +template <>
410 +inline void storeFP32<c10::BFloat16>(float v, c10::BFloat16* ptr) {
411 + c10::BFloat16 __attribute__((__may_alias__))* v_ptr =
412 + reinterpret_cast<c10::BFloat16*>(&v);
413 *ptr = *(v_ptr + 1);
414 }
415
416 #ifndef __VEC_CLASS_FP_NAN
400 -#define __VEC_CLASS_FP_NAN (1 << 6)
417 + #define __VEC_CLASS_FP_NAN (1 << 6)
418 #endif
419
403 -const static __vector unsigned char omask = { 0, 1, 4, 5, 8, 9, 12, 13, 16, 17, 20, 21, 24, 25, 28, 29 };
420 +const static __vector unsigned char omask = {0, 1, 4, 5, 8, 9, 12, 13,
421 + 16, 17, 20, 21, 24, 25, 28, 29};
422 #ifndef _ARCH_PWR10
405 -const static __vector unsigned int bias = { 0x00007fff, 0x00007fff, 0x00007fff, 0x00007fff };
406 -const static __vector unsigned int nan = { 0x7fc00000, 0x7fc00000, 0x7fc00000, 0x7fc00000 };
407 -const static __vector unsigned int sh16 = { 16, 16, 16, 16 };
408 -const static __vector unsigned int one = { 1, 1, 1, 1 };
423 +const static __vector unsigned int bias = {0x00007fff, 0x00007fff, 0x00007fff,
424 + 0x00007fff};
425 +const static __vector unsigned int nan = {0x7fc00000, 0x7fc00000, 0x7fc00000,
426 + 0x7fc00000};
427 +const static __vector unsigned int sh16 = {16, 16, 16, 16};
428 +const static __vector unsigned int one = {1, 1, 1, 1};
429 #endif
430
411 -inline BF16Vec8::BF16Vec8(const FP32Vec8 &v) {
431 +inline BF16Vec8::BF16Vec8(const FP32Vec8& v) {
432 #ifdef _ARCH_PWR10
433 __vector signed short ret[2];
414 - ret[0] = (__vector signed short)__builtin_vsx_xvcvspbf16((__vector unsigned char)v.reg.val[0]);
415 - ret[1] = (__vector signed short)__builtin_vsx_xvcvspbf16((__vector unsigned char)v.reg.val[1]);
434 + ret[0] = (__vector signed short)__builtin_vsx_xvcvspbf16(
435 + (__vector unsigned char)v.reg.val[0]);
436 + ret[1] = (__vector signed short)__builtin_vsx_xvcvspbf16(
437 + (__vector unsigned char)v.reg.val[1]);
438 reg = vec_perm(ret[0], ret[1], omask);
439 #elif defined(_ARCH_PWR9)
440 __vector unsigned int inp0 = (__vector unsigned int)(v.reg.val[0]);
@@ -425,8 +447,10 @@ inline BF16Vec8::BF16Vec8(const FP32Vec8 &v) {
447 __vector unsigned int rnd1 = vec_add(lsb1, bias);
448 inp0 = vec_add(inp0, rnd0);
449 inp1 = vec_add(inp1, rnd1);
428 - __vector __bool int sel0 = vec_test_data_class(v.reg.val[0], __VEC_CLASS_FP_NAN);
429 - __vector __bool int sel1 = vec_test_data_class(v.reg.val[1], __VEC_CLASS_FP_NAN);
450 + __vector __bool int sel0 =
451 + vec_test_data_class(v.reg.val[0], __VEC_CLASS_FP_NAN);
452 + __vector __bool int sel1 =
453 + vec_test_data_class(v.reg.val[1], __VEC_CLASS_FP_NAN);
454 inp0 = vec_sel(inp0, nan, sel0);
455 inp1 = vec_sel(inp1, nan, sel1);
456 inp0 = vec_sr(inp0, sh16);
@@ -435,13 +459,17 @@ inline BF16Vec8::BF16Vec8(const FP32Vec8 &v) {
459 #endif
460 }
461
438 -inline BF16Vec16::BF16Vec16(const FP32Vec16 &v) {
462 +inline BF16Vec16::BF16Vec16(const FP32Vec16& v) {
463 #ifdef _ARCH_PWR10
464 __vector signed short ret[4];
441 - ret[0] = (__vector signed short)__builtin_vsx_xvcvspbf16((__vector unsigned char)v.reg.val[0]);
442 - ret[1] = (__vector signed short)__builtin_vsx_xvcvspbf16((__vector unsigned char)v.reg.val[1]);
443 - ret[2] = (__vector signed short)__builtin_vsx_xvcvspbf16((__vector unsigned char)v.reg.val[2]);
444 - ret[3] = (__vector signed short)__builtin_vsx_xvcvspbf16((__vector unsigned char)v.reg.val[3]);
465 + ret[0] = (__vector signed short)__builtin_vsx_xvcvspbf16(
466 + (__vector unsigned char)v.reg.val[0]);
467 + ret[1] = (__vector signed short)__builtin_vsx_xvcvspbf16(
468 + (__vector unsigned char)v.reg.val[1]);
469 + ret[2] = (__vector signed short)__builtin_vsx_xvcvspbf16(
470 + (__vector unsigned char)v.reg.val[2]);
471 + ret[3] = (__vector signed short)__builtin_vsx_xvcvspbf16(
472 + (__vector unsigned char)v.reg.val[3]);
473 reg.val[0] = vec_perm(ret[0], ret[1], omask);
474 reg.val[1] = vec_perm(ret[2], ret[3], omask);
475 #elif defined(_ARCH_PWR9)
@@ -465,10 +493,14 @@ inline BF16Vec16::BF16Vec16(const FP32Vec16 &v) {
493 inp1 = vec_add(inp1, rnd1);
494 inp2 = vec_add(inp2, rnd2);
495 inp3 = vec_add(inp3, rnd3);
468 - __vector __bool int sel0 = vec_test_data_class(v.reg.val[0], __VEC_CLASS_FP_NAN);
469 - __vector __bool int sel1 = vec_test_data_class(v.reg.val[1], __VEC_CLASS_FP_NAN);
470 - __vector __bool int sel2 = vec_test_data_class(v.reg.val[2], __VEC_CLASS_FP_NAN);
471 - __vector __bool int sel3 = vec_test_data_class(v.reg.val[3], __VEC_CLASS_FP_NAN);
496 + __vector __bool int sel0 =
497 + vec_test_data_class(v.reg.val[0], __VEC_CLASS_FP_NAN);
498 + __vector __bool int sel1 =
499 + vec_test_data_class(v.reg.val[1], __VEC_CLASS_FP_NAN);
500 + __vector __bool int sel2 =
501 + vec_test_data_class(v.reg.val[2], __VEC_CLASS_FP_NAN);
502 + __vector __bool int sel3 =
503 + vec_test_data_class(v.reg.val[3], __VEC_CLASS_FP_NAN);
504 inp0 = vec_sel(inp0, nan, sel0);
505 inp1 = vec_sel(inp1, nan, sel1);
506 inp2 = vec_sel(inp2, nan, sel2);
@@ -482,10 +514,10 @@ inline BF16Vec16::BF16Vec16(const FP32Vec16 &v) {
514 #endif
515 }
516
485 -inline void prefetch(const void *addr) {
517 +inline void prefetch(const void* addr) {
518 __asm__ __volatile__("dcbt 0, %0" : : "r"(addr) : "memory");
519 }
520
489 -}; // namespace vec_op
521 +}; // namespace vec_op
522
523 #endif
csrc/cpu/cpu_types_x86.hpp
+165 -146
@@ -11,39 +11,40 @@ static_assert(false, "AVX2 must be supported for the current implementation.");
11
12 namespace vec_op {
13
14 -#define VLLM_DISPATCH_CASE_FLOATING_TYPES(...) \
15 - AT_DISPATCH_CASE(at::ScalarType::Float, __VA_ARGS__) \
16 - AT_DISPATCH_CASE(at::ScalarType::BFloat16, __VA_ARGS__) \
14 +#define VLLM_DISPATCH_CASE_FLOATING_TYPES(...) \
15 + AT_DISPATCH_CASE(at::ScalarType::Float, __VA_ARGS__) \
16 + AT_DISPATCH_CASE(at::ScalarType::BFloat16, __VA_ARGS__) \
17 AT_DISPATCH_CASE(at::ScalarType::Half, __VA_ARGS__)
18
19 -#define VLLM_DISPATCH_FLOATING_TYPES(TYPE, NAME, ...) \
19 +#define VLLM_DISPATCH_FLOATING_TYPES(TYPE, NAME, ...) \
20 AT_DISPATCH_SWITCH(TYPE, NAME, VLLM_DISPATCH_CASE_FLOATING_TYPES(__VA_ARGS__))
21
22 #ifndef CPU_OP_GUARD
23 -#define CPU_KERNEL_GUARD_IN(NAME)
24 -#define CPU_KERNEL_GUARD_OUT(NAME)
23 + #define CPU_KERNEL_GUARD_IN(NAME)
24 + #define CPU_KERNEL_GUARD_OUT(NAME)
25 #else
26 -#define CPU_KERNEL_GUARD_IN(NAME) \
27 - RECORD_FUNCTION(#NAME, c10::ArrayRef<c10::IValue>({}));
28 -#define CPU_KERNEL_GUARD_OUT(NAME)
26 + #define CPU_KERNEL_GUARD_IN(NAME) \
27 + RECORD_FUNCTION(#NAME, c10::ArrayRef<c10::IValue>({}));
28 + #define CPU_KERNEL_GUARD_OUT(NAME)
29 #endif
30
31 #define FORCE_INLINE __attribute__((always_inline)) inline
32
33 namespace {
34 template <typename T, T... indexes, typename F>
35 -constexpr void unroll_loop_item(std::integer_sequence<T, indexes...>, F &&f) {
35 +constexpr void unroll_loop_item(std::integer_sequence<T, indexes...>, F&& f) {
36 (f(std::integral_constant<T, indexes>{}), ...);
37 }
38 -}; // namespace
38 +}; // namespace
39
40 template <typename T, T count, typename F,
41 typename = std::enable_if_t<std::is_invocable_v<F, T>>>
42 -constexpr void unroll_loop(F &&f) {
42 +constexpr void unroll_loop(F&& f) {
43 unroll_loop_item(std::make_integer_sequence<T, count>{}, std::forward<F>(f));
44 }
45
46 -template <typename T> struct Vec {
46 +template <typename T>
47 +struct Vec {
48 constexpr static int get_elem_num() { return T::VEC_ELEM_NUM; }
49 };
50
@@ -55,12 +56,12 @@ struct FP16Vec8 : public Vec<FP16Vec8> {
56
57 __m128i reg;
58
58 - explicit FP16Vec8(const void *ptr)
59 - : reg((__m128i)_mm_loadu_si128((__m128i *)ptr)) {}
59 + explicit FP16Vec8(const void* ptr)
60 + : reg((__m128i)_mm_loadu_si128((__m128i*)ptr)) {}
61
61 - explicit FP16Vec8(const FP32Vec8 &);
62 + explicit FP16Vec8(const FP32Vec8&);
63
63 - void save(void *ptr) const { *reinterpret_cast<__m128i *>(ptr) = reg; }
64 + void save(void* ptr) const { *reinterpret_cast<__m128i*>(ptr) = reg; }
65 };
66
67 struct FP16Vec16 : public Vec<FP16Vec16> {
@@ -68,12 +69,12 @@ struct FP16Vec16 : public Vec<FP16Vec16> {
69
70 __m256i reg;
71
71 - explicit FP16Vec16(const void *ptr)
72 - : reg((__m256i)_mm256_loadu_si256((__m256i *)ptr)) {}
72 + explicit FP16Vec16(const void* ptr)
73 + : reg((__m256i)_mm256_loadu_si256((__m256i*)ptr)) {}
74
74 - explicit FP16Vec16(const FP32Vec16 &);
75 + explicit FP16Vec16(const FP32Vec16&);
76
76 - void save(void *ptr) const { *reinterpret_cast<__m256i *>(ptr) = reg; }
77 + void save(void* ptr) const { *reinterpret_cast<__m256i*>(ptr) = reg; }
78
79 void save(void* ptr, const int elem_num) const {
80 constexpr uint32_t M = 0xFFFFFFFF;
@@ -87,12 +88,12 @@ struct BF16Vec8 : public Vec<BF16Vec8> {
88
89 __m128i reg;
90
90 - explicit BF16Vec8(const void *ptr)
91 - : reg((__m128i)_mm_loadu_si128((__m128i *)ptr)) {}
91 + explicit BF16Vec8(const void* ptr)
92 + : reg((__m128i)_mm_loadu_si128((__m128i*)ptr)) {}
93
93 - explicit BF16Vec8(const FP32Vec8 &);
94 + explicit BF16Vec8(const FP32Vec8&);
95
95 - void save(void *ptr) const { *reinterpret_cast<__m128i *>(ptr) = reg; }
96 + void save(void* ptr) const { *reinterpret_cast<__m128i*>(ptr) = reg; }
97 };
98
99 struct BF16Vec16 : public Vec<BF16Vec16> {
@@ -100,12 +101,12 @@ struct BF16Vec16 : public Vec<BF16Vec16> {
101
102 __m256i reg;
103
103 - explicit BF16Vec16(const void *ptr)
104 - : reg((__m256i)_mm256_loadu_si256((__m256i *)ptr)) {}
104 + explicit BF16Vec16(const void* ptr)
105 + : reg((__m256i)_mm256_loadu_si256((__m256i*)ptr)) {}
106
106 - explicit BF16Vec16(const FP32Vec16 &);
107 + explicit BF16Vec16(const FP32Vec16&);
108
108 - void save(void *ptr) const { *reinterpret_cast<__m256i *>(ptr) = reg; }
109 + void save(void* ptr) const { *reinterpret_cast<__m256i*>(ptr) = reg; }
110
111 void save(void* ptr, const int elem_num) const {
112 constexpr uint32_t M = 0xFFFFFFFF;
@@ -120,11 +121,11 @@ struct BF16Vec32 : public Vec<BF16Vec32> {
121
122 __m512i reg;
123
123 - explicit BF16Vec32(const void *ptr) : reg((__m512i)_mm512_loadu_si512(ptr)) {}
124 + explicit BF16Vec32(const void* ptr) : reg((__m512i)_mm512_loadu_si512(ptr)) {}
125
126 explicit BF16Vec32(__m512i data) : reg(data) {}
127
127 - explicit BF16Vec32(BF16Vec8 &vec8_data)
128 + explicit BF16Vec32(BF16Vec8& vec8_data)
129 : reg((__m512i)_mm512_inserti32x4(
130 _mm512_inserti32x4(_mm512_inserti32x4(_mm512_castsi128_si512(
131 (__m128i)vec8_data.reg),
@@ -132,7 +133,7 @@ struct BF16Vec32 : public Vec<BF16Vec32> {
133 (__m128i)vec8_data.reg, 2),
134 (__m128i)vec8_data.reg, 3)) {}
135
135 - void save(void *ptr) const { *reinterpret_cast<__m512i *>(ptr) = reg; }
136 + void save(void* ptr) const { *reinterpret_cast<__m512i*>(ptr) = reg; }
137 };
138 #else
139 struct BF16Vec32 : public Vec<BF16Vec32> {
@@ -141,24 +142,24 @@ struct BF16Vec32 : public Vec<BF16Vec32> {
142 __m256i reg_low;
143 __m256i reg_high;
144
144 - explicit BF16Vec32(const void *ptr)
145 - : reg_low(_mm256_loadu_si256((__m256i const *)ptr)),
146 - reg_high(_mm256_loadu_si256((__m256i const *)ptr + 1)) {}
145 + explicit BF16Vec32(const void* ptr)
146 + : reg_low(_mm256_loadu_si256((__m256i const*)ptr)),
147 + reg_high(_mm256_loadu_si256((__m256i const*)ptr + 1)) {}
148
148 - explicit BF16Vec32(__m256i low, __m256i high) : reg_low(low),
149 - reg_high(high) {}
149 + explicit BF16Vec32(__m256i low, __m256i high)
150 + : reg_low(low), reg_high(high) {}
151
151 - explicit BF16Vec32(BF16Vec8 &vec8_data)
152 + explicit BF16Vec32(BF16Vec8& vec8_data)
153 : reg_low((__m256i)_mm256_inserti32x4(
153 - _mm256_castsi128_si256((__m128i)vec8_data.reg),
154 - (__m128i)vec8_data.reg, 1)),
154 + _mm256_castsi128_si256((__m128i)vec8_data.reg),
155 + (__m128i)vec8_data.reg, 1)),
156 reg_high((__m256i)_mm256_inserti32x4(
156 - _mm256_castsi128_si256((__m128i)vec8_data.reg),
157 - (__m128i)vec8_data.reg, 1)) {}
157 + _mm256_castsi128_si256((__m128i)vec8_data.reg),
158 + (__m128i)vec8_data.reg, 1)) {}
159
159 - void save(void *ptr) const {
160 - *reinterpret_cast<__m256i *>(ptr) = reg_low;
161 - *reinterpret_cast<__m256i *>((__m256i *)ptr + 1) = reg_high;
160 + void save(void* ptr) const {
161 + *reinterpret_cast<__m256i*>(ptr) = reg_low;
162 + *reinterpret_cast<__m256i*>((__m256i*)ptr + 1) = reg_high;
163 }
164 };
165 #endif
@@ -176,11 +177,11 @@ struct FP32Vec4 : public Vec<FP32Vec4> {
177
178 explicit FP32Vec4() : reg(_mm_set1_ps(0.0)) {}
179
179 - explicit FP32Vec4(const float *ptr) : reg(_mm_loadu_ps(ptr)) {}
180 + explicit FP32Vec4(const float* ptr) : reg(_mm_loadu_ps(ptr)) {}
181
182 explicit FP32Vec4(__m128 data) : reg(data) {}
183
183 - explicit FP32Vec4(const FP32Vec4 &data) : reg(data.reg) {}
184 + explicit FP32Vec4(const FP32Vec4& data) : reg(data.reg) {}
185 };
186
187 struct FP32Vec8 : public Vec<FP32Vec8> {
@@ -196,15 +197,15 @@ struct FP32Vec8 : public Vec<FP32Vec8> {
197
198 explicit FP32Vec8() : reg(_mm256_set1_ps(0.0)) {}
199
199 - explicit FP32Vec8(const float *ptr) : reg(_mm256_loadu_ps(ptr)) {}
200 + explicit FP32Vec8(const float* ptr) : reg(_mm256_loadu_ps(ptr)) {}
201
202 explicit FP32Vec8(__m256 data) : reg(data) {}
203
203 - explicit FP32Vec8(const FP32Vec8 &data) : reg(data.reg) {}
204 + explicit FP32Vec8(const FP32Vec8& data) : reg(data.reg) {}
205
205 - explicit FP32Vec8(const FP16Vec8 &v) : reg(_mm256_cvtph_ps(v.reg)) {}
206 + explicit FP32Vec8(const FP16Vec8& v) : reg(_mm256_cvtph_ps(v.reg)) {}
207
207 - explicit FP32Vec8(const BF16Vec8 &v)
208 + explicit FP32Vec8(const BF16Vec8& v)
209 : reg(_mm256_castsi256_ps(
210 _mm256_bslli_epi128(_mm256_cvtepu16_epi32(v.reg), 2))) {}
211
@@ -212,7 +213,8 @@ struct FP32Vec8 : public Vec<FP32Vec8> {
213 AliasReg ar;
214 ar.reg = reg;
215 float result = 0;
215 - unroll_loop<int, VEC_ELEM_NUM>([&result, &ar](int i) { result += ar.values[i]; });
216 + unroll_loop<int, VEC_ELEM_NUM>(
217 + [&result, &ar](int i) { result += ar.values[i]; });
218
219 return result;
220 }
@@ -244,27 +246,27 @@ struct FP32Vec8 : public Vec<FP32Vec8> {
246 erf(ar.values[1]), erf(ar.values[0])));
247 }
248
247 - FP32Vec8 operator*(const FP32Vec8 &b) const {
249 + FP32Vec8 operator*(const FP32Vec8& b) const {
250 return FP32Vec8(_mm256_mul_ps(reg, b.reg));
251 }
252
251 - FP32Vec8 operator+(const FP32Vec8 &b) const {
253 + FP32Vec8 operator+(const FP32Vec8& b) const {
254 return FP32Vec8(_mm256_add_ps(reg, b.reg));
255 }
256
255 - FP32Vec8 operator-(const FP32Vec8 &b) const {
257 + FP32Vec8 operator-(const FP32Vec8& b) const {
258 return FP32Vec8(_mm256_sub_ps(reg, b.reg));
259 }
260
259 - FP32Vec8 operator/(const FP32Vec8 &b) const {
261 + FP32Vec8 operator/(const FP32Vec8& b) const {
262 return FP32Vec8(_mm256_div_ps(reg, b.reg));
263 }
264
263 - void save(float *ptr) const { _mm256_storeu_ps(ptr, reg); }
265 + void save(float* ptr) const { _mm256_storeu_ps(ptr, reg); }
266 };
267
268 #ifdef __AVX512F__
267 -struct INT32Vec16: public Vec<INT32Vec16> {
269 +struct INT32Vec16 : public Vec<INT32Vec16> {
270 constexpr static int VEC_ELEM_NUM = 16;
271 union AliasReg {
272 __m512i reg;
@@ -272,12 +274,11 @@ struct INT32Vec16: public Vec<INT32Vec16> {
274 };
275
276 __m512i reg;
275 -
276 - explicit INT32Vec16(const void* data_ptr) : reg(_mm512_loadu_epi32(data_ptr)) {}
277
278 - void save(int32_t* ptr) const {
279 - _mm512_storeu_epi32(ptr, reg);
280 - }
278 + explicit INT32Vec16(const void* data_ptr)
279 + : reg(_mm512_loadu_epi32(data_ptr)) {}
280 +
281 + void save(int32_t* ptr) const { _mm512_storeu_epi32(ptr, reg); }
282
283 void save(int32_t* ptr, const int elem_num) const {
284 constexpr uint32_t M = 0xFFFFFFFF;
@@ -301,11 +302,11 @@ struct FP32Vec16 : public Vec<FP32Vec16> {
302
303 explicit FP32Vec16() : reg(_mm512_set1_ps(0.0)) {}
304
304 - explicit FP32Vec16(const float *ptr) : reg(_mm512_loadu_ps(ptr)) {}
305 + explicit FP32Vec16(const float* ptr) : reg(_mm512_loadu_ps(ptr)) {}
306
307 explicit FP32Vec16(__m512 data) : reg(data) {}
308
308 - explicit FP32Vec16(const FP32Vec4 &data)
309 + explicit FP32Vec16(const FP32Vec4& data)
310 : reg((__m512)_mm512_inserti32x4(
311 _mm512_inserti32x4(
312 _mm512_inserti32x4(_mm512_castsi128_si512((__m128i)data.reg),
@@ -313,36 +314,37 @@ struct FP32Vec16 : public Vec<FP32Vec16> {
314 (__m128i)data.reg, 2),
315 (__m128i)data.reg, 3)) {}
316
316 - explicit FP32Vec16(const FP32Vec8 &data)
317 + explicit FP32Vec16(const FP32Vec8& data)
318 : reg((__m512)_mm512_inserti32x8(
319 _mm512_castsi256_si512((__m256i)data.reg), (__m256i)data.reg, 1)) {}
320
320 - explicit FP32Vec16(const BF16Vec16 &v)
321 + explicit FP32Vec16(const BF16Vec16& v)
322 : reg(_mm512_castsi512_ps(
323 _mm512_bslli_epi128(_mm512_cvtepu16_epi32(v.reg), 2))) {}
324
324 - explicit FP32Vec16(const FP16Vec16 &v) : reg(_mm512_cvtph_ps(v.reg)) {}
325 + explicit FP32Vec16(const FP16Vec16& v) : reg(_mm512_cvtph_ps(v.reg)) {}
326
326 - explicit FP32Vec16(const FP16Vec8 &v) : FP32Vec16(FP32Vec8(v)) {}
327 + explicit FP32Vec16(const FP16Vec8& v) : FP32Vec16(FP32Vec8(v)) {}
328
328 - explicit FP32Vec16(const BF16Vec8 &v) : FP32Vec16(FP32Vec8(v)) {}
329 + explicit FP32Vec16(const BF16Vec8& v) : FP32Vec16(FP32Vec8(v)) {}
330
330 - explicit FP32Vec16(const INT32Vec16 &v)
331 - : reg(_mm512_cvt_roundepi32_ps(v.reg, _MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC)) {}
331 + explicit FP32Vec16(const INT32Vec16& v)
332 + : reg(_mm512_cvt_roundepi32_ps(
333 + v.reg, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC)) {}
334
333 - FP32Vec16 operator*(const FP32Vec16 &b) const {
335 + FP32Vec16 operator*(const FP32Vec16& b) const {
336 return FP32Vec16(_mm512_mul_ps(reg, b.reg));
337 }
338
337 - FP32Vec16 operator+(const FP32Vec16 &b) const {
339 + FP32Vec16 operator+(const FP32Vec16& b) const {
340 return FP32Vec16(_mm512_add_ps(reg, b.reg));
341 }
342
341 - FP32Vec16 operator-(const FP32Vec16 &b) const {
343 + FP32Vec16 operator-(const FP32Vec16& b) const {
344 return FP32Vec16(_mm512_sub_ps(reg, b.reg));
345 }
346
345 - FP32Vec16 operator/(const FP32Vec16 &b) const {
347 + FP32Vec16 operator/(const FP32Vec16& b) const {
348 return FP32Vec16(_mm512_div_ps(reg, b.reg));
349 }
350
@@ -370,9 +372,7 @@ struct FP32Vec16 : public Vec<FP32Vec16> {
372 return FP32Vec16(_mm512_mask_min_ps(reg, mask, reg, b.reg));
373 }
374
373 - FP32Vec16 abs() const {
374 - return FP32Vec16(_mm512_abs_ps(reg));
375 - }
375 + FP32Vec16 abs() const { return FP32Vec16(_mm512_abs_ps(reg)); }
376
377 float reduce_sum() const { return _mm512_reduce_add_ps(reg); }
378
@@ -380,14 +380,15 @@ struct FP32Vec16 : public Vec<FP32Vec16> {
380
381 float reduce_min() const { return _mm512_reduce_min_ps(reg); }
382
383 - template <int group_size> float reduce_sub_sum(int idx) {
383 + template <int group_size>
384 + float reduce_sub_sum(int idx) {
385 static_assert(VEC_ELEM_NUM % group_size == 0);
386 constexpr uint32_t base_mask = (0xFFFF >> (16 - group_size));
387 __mmask16 mask = _cvtu32_mask16(base_mask << (idx * group_size));
388 return _mm512_mask_reduce_add_ps(mask, reg);
389 }
390
390 - void save(float *ptr) const { _mm512_storeu_ps(ptr, reg); }
391 + void save(float* ptr) const { _mm512_storeu_ps(ptr, reg); }
392
393 void save(float* ptr, const int elem_num) const {
394 constexpr uint32_t M = 0xFFFFFFFF;
@@ -407,32 +408,30 @@ struct FP32Vec16 : public Vec<FP32Vec16> {
408 __m256 reg_low;
409 __m256 reg_high;
410
410 - explicit FP32Vec16(float v) : reg_low(_mm256_set1_ps(v)),
411 - reg_high(_mm256_set1_ps(v)) {}
411 + explicit FP32Vec16(float v)
412 + : reg_low(_mm256_set1_ps(v)), reg_high(_mm256_set1_ps(v)) {}
413
413 - explicit FP32Vec16() : reg_low(_mm256_set1_ps(0.0)),
414 - reg_high(_mm256_set1_ps(0.0)) {}
414 + explicit FP32Vec16()
415 + : reg_low(_mm256_set1_ps(0.0)), reg_high(_mm256_set1_ps(0.0)) {}
416
416 - explicit FP32Vec16(const float *ptr) : reg_low(_mm256_loadu_ps(ptr)),
417 - reg_high(_mm256_loadu_ps(ptr + 8)) {}
417 + explicit FP32Vec16(const float* ptr)
418 + : reg_low(_mm256_loadu_ps(ptr)), reg_high(_mm256_loadu_ps(ptr + 8)) {}
419
420 explicit FP32Vec16(__m256 low, __m256 high) : reg_low(low), reg_high(high) {}
421
421 - explicit FP32Vec16(const FP32Vec16 &data) : reg_low(data.reg_low),
422 - reg_high(data.reg_high) {}
422 + explicit FP32Vec16(const FP32Vec16& data)
423 + : reg_low(data.reg_low), reg_high(data.reg_high) {}
424
424 - explicit FP32Vec16(const FP32Vec4 &data)
425 + explicit FP32Vec16(const FP32Vec4& data)
426 : reg_low((__m256)_mm256_inserti128_si256(
426 - _mm256_castsi128_si256((__m128i)data.reg),
427 - (__m128i)data.reg, 1)),
427 + _mm256_castsi128_si256((__m128i)data.reg), (__m128i)data.reg, 1)),
428 reg_high((__m256)_mm256_inserti128_si256(
429 - _mm256_castsi128_si256((__m128i)data.reg),
430 - (__m128i)data.reg, 1)) {}
429 + _mm256_castsi128_si256((__m128i)data.reg), (__m128i)data.reg, 1)) {}
430
432 - explicit FP32Vec16(const FP32Vec8 &data)
431 + explicit FP32Vec16(const FP32Vec8& data)
432 : reg_low(data.reg), reg_high(data.reg) {}
433
435 - explicit FP32Vec16(const FP16Vec16 &v) {
434 + explicit FP32Vec16(const FP16Vec16& v) {
435 __m128i low = _mm256_extractf128_si256(v.reg, 0);
436 __m128i high = _mm256_extractf128_si256(v.reg, 1);
437
@@ -440,9 +439,9 @@ struct FP32Vec16 : public Vec<FP32Vec16> {
439 reg_high = _mm256_cvtph_ps(high);
440 }
441
443 - explicit FP32Vec16(const FP16Vec8 &v) : FP32Vec16(FP32Vec8(v)) {}
442 + explicit FP32Vec16(const FP16Vec8& v) : FP32Vec16(FP32Vec8(v)) {}
443
445 - explicit FP32Vec16(const BF16Vec16 &v) {
444 + explicit FP32Vec16(const BF16Vec16& v) {
445 __m128i low = _mm256_extractf128_si256(v.reg, 0);
446 __m128i high = _mm256_extractf128_si256(v.reg, 1);
447
@@ -456,24 +455,24 @@ struct FP32Vec16 : public Vec<FP32Vec16> {
455 reg_high = _mm256_castsi256_ps(v_high_shifted);
456 }
457
459 - explicit FP32Vec16(const BF16Vec8 &v) : FP32Vec16(FP32Vec8(v)) {}
458 + explicit FP32Vec16(const BF16Vec8& v) : FP32Vec16(FP32Vec8(v)) {}
459
461 - FP32Vec16 operator*(const FP32Vec16 &b) const {
460 + FP32Vec16 operator*(const FP32Vec16& b) const {
461 return FP32Vec16(_mm256_mul_ps(reg_low, b.reg_low),
462 _mm256_mul_ps(reg_high, b.reg_high));
463 }
464
466 - FP32Vec16 operator+(const FP32Vec16 &b) const {
465 + FP32Vec16 operator+(const FP32Vec16& b) const {
466 return FP32Vec16(_mm256_add_ps(reg_low, b.reg_low),
467 _mm256_add_ps(reg_high, b.reg_high));
468 }
469
471 - FP32Vec16 operator-(const FP32Vec16 &b) const {
470 + FP32Vec16 operator-(const FP32Vec16& b) const {
471 return FP32Vec16(_mm256_sub_ps(reg_low, b.reg_low),
472 _mm256_sub_ps(reg_high, b.reg_high));
473 }
474
476 - FP32Vec16 operator/(const FP32Vec16 &b) const {
475 + FP32Vec16 operator/(const FP32Vec16& b) const {
476 return FP32Vec16(_mm256_div_ps(reg_low, b.reg_low),
477 _mm256_div_ps(reg_high, b.reg_high));
478 }
@@ -484,7 +483,8 @@ struct FP32Vec16 : public Vec<FP32Vec16> {
483 return low.reduce_sum() + high.reduce_sum();
484 }
485
487 - template <int group_size> float reduce_sub_sum(int idx) {
486 + template <int group_size>
487 + float reduce_sub_sum(int idx) {
488 float sum = 0.0;
489 static_assert(VEC_ELEM_NUM % group_size == 0);
490 constexpr uint32_t base_mask = (0xFFFF >> (16 - group_size));
@@ -507,7 +507,7 @@ struct FP32Vec16 : public Vec<FP32Vec16> {
507 return sum;
508 }
509
510 - void save(float *ptr) const {
510 + void save(float* ptr) const {
511 _mm256_storeu_ps(ptr, reg_low);
512 _mm256_storeu_ps(ptr + 8, reg_high);
513 }
@@ -515,7 +515,7 @@ struct FP32Vec16 : public Vec<FP32Vec16> {
515 #endif
516
517 #ifdef __AVX512F__
518 -struct INT8Vec16: public Vec<INT8Vec16> {
518 +struct INT8Vec16 : public Vec<INT8Vec16> {
519 constexpr static int VEC_ELEM_NUM = 16;
520 union AliasReg {
521 __m128i reg;
@@ -523,14 +523,12 @@ struct INT8Vec16: public Vec<INT8Vec16> {
523 };
524
525 __m128i reg;
526 -
527 - explicit INT8Vec16(const FP32Vec16& vec) : reg(
528 - _mm512_cvtepi32_epi8(_mm512_cvt_roundps_epi32(vec.reg, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC))
529 - ) {}
526
531 - void save(int8_t* ptr) const {
532 - _mm_storeu_epi8(ptr, reg);
533 - }
527 + explicit INT8Vec16(const FP32Vec16& vec)
528 + : reg(_mm512_cvtepi32_epi8(_mm512_cvt_roundps_epi32(
529 + vec.reg, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC))) {}
530 +
531 + void save(int8_t* ptr) const { _mm_storeu_epi8(ptr, reg); }
532
533 void save(int8_t* ptr, const int elem_num) const {
534 constexpr uint32_t M = 0xFFFFFFFF;
@@ -540,71 +538,92 @@ struct INT8Vec16: public Vec<INT8Vec16> {
538 };
539 #endif
540
543 -template <typename T> struct VecType { using vec_type = void; };
541 +template <typename T>
542 +struct VecType {
543 + using vec_type = void;
544 +};
545
545 -template <typename T> using vec_t = typename VecType<T>::vec_type;
546 +template <typename T>
547 +using vec_t = typename VecType<T>::vec_type;
548
547 -template <> struct VecType<float> { using vec_type = FP32Vec8; };
549 +template <>
550 +struct VecType<float> {
551 + using vec_type = FP32Vec8;
552 +};
553
549 -template <> struct VecType<c10::Half> { using vec_type = FP16Vec8; };
554 +template <>
555 +struct VecType<c10::Half> {
556 + using vec_type = FP16Vec8;
557 +};
558
551 -template <> struct VecType<c10::BFloat16> { using vec_type = BF16Vec8; };
559 +template <>
560 +struct VecType<c10::BFloat16> {
561 + using vec_type = BF16Vec8;
562 +};
563
553 -template <typename T> void storeFP32(float v, T *ptr) { *ptr = v; }
564 +template <typename T>
565 +void storeFP32(float v, T* ptr) {
566 + *ptr = v;
567 +}
568
555 -inline void fma(FP32Vec16 &acc, FP32Vec16 &a, FP32Vec16 &b) {
569 +inline void fma(FP32Vec16& acc, FP32Vec16& a, FP32Vec16& b) {
570 acc = acc + a * b;
571 }
572
559 -template <> inline void storeFP32<c10::Half>(float v, c10::Half *ptr) {
560 - *reinterpret_cast<unsigned short *>(ptr) =
573 +template <>
574 +inline void storeFP32<c10::Half>(float v, c10::Half* ptr) {
575 + *reinterpret_cast<unsigned short*>(ptr) =
576 _cvtss_sh(v, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC);
577 }
578
564 -inline FP16Vec8::FP16Vec8(const FP32Vec8 &v)
579 +inline FP16Vec8::FP16Vec8(const FP32Vec8& v)
580 : reg(_mm256_cvtps_ph(v.reg,
581 _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC)) {}
582
583 #ifdef __AVX512F__
569 -inline FP16Vec16::FP16Vec16(const FP32Vec16 &v)
584 +inline FP16Vec16::FP16Vec16(const FP32Vec16& v)
585 : reg(_mm512_cvtps_ph(v.reg,
586 _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC)) {}
587 #else
573 -inline FP16Vec16::FP16Vec16(const FP32Vec16 &v)
574 - : reg(_mm256_insertf128_si256(_mm256_castsi128_si256(FP16Vec8(FP32Vec8(v.reg_low)).reg), FP16Vec8(FP32Vec8(v.reg_low)).reg, 1)) {}
588 +inline FP16Vec16::FP16Vec16(const FP32Vec16& v)
589 + : reg(_mm256_insertf128_si256(
590 + _mm256_castsi128_si256(FP16Vec8(FP32Vec8(v.reg_low)).reg),
591 + FP16Vec8(FP32Vec8(v.reg_low)).reg, 1)) {}
592 #endif
593
594 #ifdef __AVX512BF16__
578 -template <> inline void storeFP32<c10::BFloat16>(float v, c10::BFloat16 *ptr) {
579 - *reinterpret_cast<__bfloat16 *>(ptr) = _mm_cvtness_sbh(v);
595 +template <>
596 +inline void storeFP32<c10::BFloat16>(float v, c10::BFloat16* ptr) {
597 + *reinterpret_cast<__bfloat16*>(ptr) = _mm_cvtness_sbh(v);
598 }
599
582 -inline BF16Vec8::BF16Vec8(const FP32Vec8 &v)
600 +inline BF16Vec8::BF16Vec8(const FP32Vec8& v)
601 : reg((__m128i)_mm256_cvtneps_pbh(v.reg)) {}
602
585 -inline BF16Vec16::BF16Vec16(const FP32Vec16 &v)
603 +inline BF16Vec16::BF16Vec16(const FP32Vec16& v)
604 : reg((__m256i)_mm512_cvtneps_pbh(v.reg)) {}
605
588 -inline void fma(FP32Vec16 &acc, BF16Vec32 &a, BF16Vec32 &b) {
606 +inline void fma(FP32Vec16& acc, BF16Vec32& a, BF16Vec32& b) {
607 acc.reg = _mm512_dpbf16_ps(acc.reg, (__m512bh)a.reg, (__m512bh)b.reg);
608 }
609 #else
592 -template <> inline void storeFP32<c10::BFloat16>(float v, c10::BFloat16 *ptr) {
593 - c10::BFloat16 __attribute__((__may_alias__)) *v_ptr =
594 - reinterpret_cast<c10::BFloat16 *>(&v);
610 +template <>
611 +inline void storeFP32<c10::BFloat16>(float v, c10::BFloat16* ptr) {
612 + c10::BFloat16 __attribute__((__may_alias__))* v_ptr =
613 + reinterpret_cast<c10::BFloat16*>(&v);
614 *ptr = *(v_ptr + 1);
615 }
616
598 -#ifdef __AVX512F__
599 -inline BF16Vec8::BF16Vec8(const FP32Vec8 &v)
617 + #ifdef __AVX512F__
618 +inline BF16Vec8::BF16Vec8(const FP32Vec8& v)
619 : reg(_mm256_cvtepi32_epi16(
620 _mm256_bsrli_epi128(_mm256_castps_si256(v.reg), 2))) {}
621
603 -inline BF16Vec16::BF16Vec16(const FP32Vec16 &v)
622 +inline BF16Vec16::BF16Vec16(const FP32Vec16& v)
623 : reg(_mm512_cvtepi32_epi16(
624 _mm512_bsrli_epi128(_mm512_castps_si512(v.reg), 2))) {}
606 -#else
607 -namespace{
625 + #else
626 +namespace {
627 __m128i FP32Vec8_to_BF16Vec8_avx2(__m256 a) {
628 __m256i ai = _mm256_castps_si256(a);
629 ai = _mm256_srli_epi32(ai, 16);
@@ -612,21 +631,21 @@ __m128i FP32Vec8_to_BF16Vec8_avx2(__m256 a) {
631 ai = _mm256_permute4x64_epi64(ai, 0b00111001);
632 return _mm256_extracti128_si256(ai, 0);
633 }
615 -}
634 +} // namespace
635
617 -inline BF16Vec8::BF16Vec8(const FP32Vec8 &v)
636 +inline BF16Vec8::BF16Vec8(const FP32Vec8& v)
637 : reg(FP32Vec8_to_BF16Vec8_avx2(v.reg)) {}
638
620 -inline BF16Vec16::BF16Vec16(const FP32Vec16 &v) {
639 +inline BF16Vec16::BF16Vec16(const FP32Vec16& v) {
640 BF16Vec8 low = BF16Vec8(FP32Vec8(v.reg_low));
641 BF16Vec8 high = BF16Vec8(FP32Vec8(v.reg_high));
642 reg = _mm256_insertf128_si256(_mm256_castsi128_si256(low.reg), high.reg, 1);
643 }
625 -#endif // __AVX512F__
626 -#endif // __AVX512BF16__
644 + #endif // __AVX512F__
645 +#endif // __AVX512BF16__
646
628 -inline void prefetch(const void *addr) { _mm_prefetch(addr, _MM_HINT_T1); }
647 +inline void prefetch(const void* addr) { _mm_prefetch(addr, _MM_HINT_T1); }
648
630 -}; // namespace vec_op
649 +}; // namespace vec_op
650
651 #endif
csrc/cutlass_extensions/common.hpp
+1 -2
@@ -27,8 +27,7 @@
27 inline int get_cuda_max_shared_memory_per_block_opt_in(int const device) {
28 int max_shared_mem_per_block_opt_in = 0;
29 cudaDeviceGetAttribute(&max_shared_mem_per_block_opt_in,
30 - cudaDevAttrMaxSharedMemoryPerBlockOptin,
31 - device);
30 + cudaDevAttrMaxSharedMemoryPerBlockOptin, device);
31 return max_shared_mem_per_block_opt_in;
32 }
33
docs/source/contributing/overview.md
+8 -5
@@ -25,10 +25,12 @@ Check out the [building from source](#build-from-source) documentation for detai
25 ```bash
26 pip install -r requirements-dev.txt
27
28 -# linting and formatting
29 -bash format.sh
30 -# Static type checking
31 -mypy
28 +# Linting, formatting and static type checking
29 +pre-commit install
30 +
31 +# You can manually run pre-commit with
32 +pre-commit run --all-files
33 +
34 # Unit tests
35 pytest tests/
36 ```
@@ -88,7 +90,8 @@ If the PR spans more than one category, please include all relevant prefixes.
90 The PR needs to meet the following code quality standards:
91
92 - We adhere to [Google Python style guide](https://google.github.io/styleguide/pyguide.html) and [Google C++ style guide](https://google.github.io/styleguide/cppguide.html).
91 -- Pass all linter checks. Please use <gh-file:format.sh> to format your code.
93 +- Pass all linter checks. Please use `pre-commit` to format your code. See
94 + <https://pre-commit.com/#usage> if `pre-commit` is new to you.
95 - The code needs to be well-documented to ensure future contributors can easily
96 understand the code.
97 - Include sufficient tests to ensure the project stays correct and robust. This
format.sh deleted
-321
@@ -1,321 +0,0 @@
1 -#!/usr/bin/env bash
2 -# YAPF formatter, adapted from ray and skypilot.
3 -#
4 -# Usage:
5 -# # Do work and commit your work.
6 -
7 -# # Format files that differ from origin/main.
8 -# bash format.sh
9 -
10 -# # Commit changed files with message 'Run yapf and ruff'
11 -#
12 -#
13 -# YAPF + Clang formatter (if installed). This script formats all changed files from the last mergebase.
14 -# You are encouraged to run this locally before pushing changes for review.
15 -
16 -# Cause the script to exit if a single command fails
17 -set -eo pipefail
18 -
19 -# this stops git rev-parse from failing if we run this from the .git directory
20 -builtin cd "$(dirname "${BASH_SOURCE:-$0}")"
21 -ROOT="$(git rev-parse --show-toplevel)"
22 -builtin cd "$ROOT" || exit 1
23 -
24 -check_command() {
25 - if ! command -v "$1" &> /dev/null; then
26 - echo "❓❓$1 is not installed, please run \`pip install -r requirements-lint.txt\`"
27 - exit 1
28 - fi
29 -}
30 -
31 -check_command yapf
32 -check_command ruff
33 -check_command mypy
34 -check_command codespell
35 -check_command isort
36 -check_command clang-format
37 -
38 -YAPF_VERSION=$(yapf --version | awk '{print $2}')
39 -RUFF_VERSION=$(ruff --version | awk '{print $2}')
40 -MYPY_VERSION=$(mypy --version | awk '{print $2}')
41 -CODESPELL_VERSION=$(codespell --version)
42 -ISORT_VERSION=$(isort --vn)
43 -CLANGFORMAT_VERSION=$(clang-format --version | awk '{print $3}')
44 -PYMARKDOWNLNT_VERSION=$(pymarkdownlnt version | awk '{print $1}')
45 -
46 -# # params: tool name, tool version, required version
47 -tool_version_check() {
48 - expected=$(grep "$1" requirements-lint.txt | cut -d'=' -f3)
49 - if [[ "$2" != "$expected" ]]; then
50 - echo "❓❓Wrong $1 version installed: $expected is required, not $2."
51 - exit 1
52 - fi
53 -}
54 -
55 -tool_version_check "yapf" "$YAPF_VERSION"
56 -tool_version_check "ruff" "$RUFF_VERSION"
57 -tool_version_check "mypy" "$MYPY_VERSION"
58 -tool_version_check "isort" "$ISORT_VERSION"
59 -tool_version_check "codespell" "$CODESPELL_VERSION"
60 -tool_version_check "clang-format" "$CLANGFORMAT_VERSION"
61 -tool_version_check "pymarkdownlnt" "$PYMARKDOWNLNT_VERSION"
62 -
63 -YAPF_FLAGS=(
64 - '--recursive'
65 - '--parallel'
66 -)
67 -
68 -YAPF_EXCLUDES=(
69 - '--exclude' 'build/**'
70 -)
71 -
72 -# Format specified files
73 -format() {
74 - yapf --in-place "${YAPF_FLAGS[@]}" "$@"
75 -}
76 -
77 -# Format files that differ from main branch. Ignores dirs that are not slated
78 -# for autoformat yet.
79 -format_changed() {
80 - # The `if` guard ensures that the list of filenames is not empty, which
81 - # could cause yapf to receive 0 positional arguments, making it hang
82 - # waiting for STDIN.
83 - #
84 - # `diff-filter=ACM` and $MERGEBASE is to ensure we only format files that
85 - # exist on both branches.
86 - MERGEBASE="$(git merge-base origin/main HEAD)"
87 -
88 - if ! git diff --diff-filter=ACM --quiet --exit-code "$MERGEBASE" -- '*.py' '*.pyi' &>/dev/null; then
89 - git diff --name-only --diff-filter=ACM "$MERGEBASE" -- '*.py' '*.pyi' | xargs -P 5 \
90 - yapf --in-place "${YAPF_EXCLUDES[@]}" "${YAPF_FLAGS[@]}"
91 - fi
92 -
93 -}
94 -
95 -# Format all files
96 -format_all() {
97 - yapf --in-place "${YAPF_FLAGS[@]}" "${YAPF_EXCLUDES[@]}" .
98 -}
99 -
100 -## This flag formats individual files. --files *must* be the first command line
101 -## arg to use this option.
102 -if [[ "$1" == '--files' ]]; then
103 - format "${@:2}"
104 - # If `--all` is passed, then any further arguments are ignored and the
105 - # entire python directory is formatted.
106 -elif [[ "$1" == '--all' ]]; then
107 - format_all
108 -else
109 - # Format only the files that changed in last commit.
110 - format_changed
111 -fi
112 -echo 'vLLM yapf: Done'
113 -
114 -# Run mypy
115 -echo 'vLLM mypy:'
116 -tools/mypy.sh
117 -echo 'vLLM mypy: Done'
118 -
119 -
120 -# If git diff returns a file that is in the skip list, the file may be checked anyway:
121 -# https://github.com/codespell-project/codespell/issues/1915
122 -# Avoiding the "./" prefix and using "/**" globs for directories appears to solve the problem
123 -CODESPELL_EXCLUDES=(
124 - '--skip' 'tests/prompts/**,./benchmarks/sonnet.txt,*tests/lora/data/**,build/**'
125 -)
126 -
127 -# check spelling of specified files
128 -spell_check() {
129 - codespell "$@"
130 -}
131 -
132 -spell_check_all(){
133 - codespell --toml pyproject.toml "${CODESPELL_EXCLUDES[@]}"
134 -}
135 -
136 -# Spelling check of files that differ from main branch.
137 -spell_check_changed() {
138 - # The `if` guard ensures that the list of filenames is not empty, which
139 - # could cause ruff to receive 0 positional arguments, making it hang
140 - # waiting for STDIN.
141 - #
142 - # `diff-filter=ACM` and $MERGEBASE is to ensure we only lint files that
143 - # exist on both branches.
144 - MERGEBASE="$(git merge-base origin/main HEAD)"
145 - if ! git diff --diff-filter=ACM --quiet --exit-code "$MERGEBASE" -- '*.py' '*.pyi' &>/dev/null; then
146 - git diff --name-only --diff-filter=ACM "$MERGEBASE" -- '*.py' '*.pyi' | xargs \
147 - codespell "${CODESPELL_EXCLUDES[@]}"
148 - fi
149 -}
150 -
151 -# Run Codespell
152 -## This flag runs spell check of individual files. --files *must* be the first command line
153 -## arg to use this option.
154 -if [[ "$1" == '--files' ]]; then
155 - spell_check "${@:2}"
156 - # If `--all` is passed, then any further arguments are ignored and the
157 - # entire python directory is linted.
158 -elif [[ "$1" == '--all' ]]; then
159 - spell_check_all
160 -else
161 - # Check spelling only of the files that changed in last commit.
162 - spell_check_changed
163 -fi
164 -echo 'vLLM codespell: Done'
165 -
166 -
167 -# Lint specified files
168 -lint() {
169 - ruff check "$@"
170 -}
171 -
172 -# Lint files that differ from main branch. Ignores dirs that are not slated
173 -# for autolint yet.
174 -lint_changed() {
175 - # The `if` guard ensures that the list of filenames is not empty, which
176 - # could cause ruff to receive 0 positional arguments, making it hang
177 - # waiting for STDIN.
178 - #
179 - # `diff-filter=ACM` and $MERGEBASE is to ensure we only lint files that
180 - # exist on both branches.
181 - MERGEBASE="$(git merge-base origin/main HEAD)"
182 -
183 - if ! git diff --diff-filter=ACM --quiet --exit-code "$MERGEBASE" -- '*.py' '*.pyi' &>/dev/null; then
184 - git diff --name-only --diff-filter=ACM "$MERGEBASE" -- '*.py' '*.pyi' | xargs \
185 - ruff check
186 - fi
187 -
188 -}
189 -
190 -# Run Ruff
191 -### This flag lints individual files. --files *must* be the first command line
192 -### arg to use this option.
193 -if [[ "$1" == '--files' ]]; then
194 - lint "${@:2}"
195 - # If `--all` is passed, then any further arguments are ignored and the
196 - # entire python directory is linted.
197 -elif [[ "$1" == '--all' ]]; then
198 - lint vllm tests
199 -else
200 - # Format only the files that changed in last commit.
201 - lint_changed
202 -fi
203 -echo 'vLLM ruff: Done'
204 -
205 -# check spelling of specified files
206 -isort_check() {
207 - isort "$@"
208 -}
209 -
210 -isort_check_all(){
211 - isort .
212 -}
213 -
214 -# Spelling check of files that differ from main branch.
215 -isort_check_changed() {
216 - # The `if` guard ensures that the list of filenames is not empty, which
217 - # could cause ruff to receive 0 positional arguments, making it hang
218 - # waiting for STDIN.
219 - #
220 - # `diff-filter=ACM` and $MERGEBASE is to ensure we only lint files that
221 - # exist on both branches.
222 - MERGEBASE="$(git merge-base origin/main HEAD)"
223 -
224 - if ! git diff --diff-filter=ACM --quiet --exit-code "$MERGEBASE" -- '*.py' '*.pyi' &>/dev/null; then
225 - git diff --name-only --diff-filter=ACM "$MERGEBASE" -- '*.py' '*.pyi' | xargs \
226 - isort
227 - fi
228 -}
229 -
230 -# Run Isort
231 -# This flag runs spell check of individual files. --files *must* be the first command line
232 -# arg to use this option.
233 -if [[ "$1" == '--files' ]]; then
234 - isort_check "${@:2}"
235 - # If `--all` is passed, then any further arguments are ignored and the
236 - # entire python directory is linted.
237 -elif [[ "$1" == '--all' ]]; then
238 - isort_check_all
239 -else
240 - # Check spelling only of the files that changed in last commit.
241 - isort_check_changed
242 -fi
243 -echo 'vLLM isort: Done'
244 -
245 -# Clang-format section
246 -# Exclude some files for formatting because they are vendored
247 -# NOTE: Keep up to date with .github/workflows/clang-format.yml
248 -CLANG_FORMAT_EXCLUDES=(
249 - 'csrc/moe/topk_softmax_kernels.cu'
250 - 'csrc/quantization/gguf/ggml-common.h'
251 - 'csrc/quantization/gguf/dequantize.cuh'
252 - 'csrc/quantization/gguf/vecdotq.cuh'
253 - 'csrc/quantization/gguf/mmq.cuh'
254 - 'csrc/quantization/gguf/mmvq.cuh'
255 -)
256 -
257 -# Format specified files with clang-format
258 -clang_format() {
259 - clang-format -i "$@"
260 -}
261 -
262 -# Format files that differ from main branch with clang-format.
263 -clang_format_changed() {
264 - # The `if` guard ensures that the list of filenames is not empty, which
265 - # could cause clang-format to receive 0 positional arguments, making it hang
266 - # waiting for STDIN.
267 - #
268 - # `diff-filter=ACM` and $MERGEBASE is to ensure we only format files that
269 - # exist on both branches.
270 - MERGEBASE="$(git merge-base origin/main HEAD)"
271 -
272 - # Get the list of changed files, excluding the specified ones
273 - changed_files=$(git diff --name-only --diff-filter=ACM "$MERGEBASE" -- '*.h' '*.cpp' '*.cu' '*.cuh' | (grep -vFf <(printf "%s\n" "${CLANG_FORMAT_EXCLUDES[@]}") || echo -e))
274 - if [ -n "$changed_files" ]; then
275 - echo "$changed_files" | xargs -P 5 clang-format -i
276 - fi
277 -}
278 -
279 -# Format all files with clang-format
280 -clang_format_all() {
281 - find csrc/ \( -name '*.h' -o -name '*.cpp' -o -name '*.cu' -o -name '*.cuh' \) -print \
282 - | grep -vFf <(printf "%s\n" "${CLANG_FORMAT_EXCLUDES[@]}") \
283 - | xargs clang-format -i
284 -}
285 -
286 -# Run clang-format
287 -if [[ "$1" == '--files' ]]; then
288 - clang_format "${@:2}"
289 -elif [[ "$1" == '--all' ]]; then
290 - clang_format_all
291 -else
292 - clang_format_changed
293 -fi
294 -echo 'vLLM clang-format: Done'
295 -
296 -echo 'vLLM actionlint:'
297 -tools/actionlint.sh -color
298 -echo 'vLLM actionlint: Done'
299 -
300 -echo 'vLLM shellcheck:'
301 -tools/shellcheck.sh
302 -echo 'vLLM shellcheck: Done'
303 -
304 -echo 'excalidraw png check:'
305 -tools/png-lint.sh
306 -echo 'excalidraw png check: Done'
307 -
308 -if ! git diff --quiet &>/dev/null; then
309 - echo
310 - echo "🔍🔍There are files changed by the format checker or by you that are not added and committed:"
311 - git --no-pager diff --name-only
312 - echo "🔍🔍Format checker passed, but please add, commit and push all the files above to include changes made by the format checker."
313 -
314 - exit 1
315 -else
316 - echo "✨🎉 Format check passed! Congratulations! 🎉✨"
317 -fi
318 -
319 -echo 'vLLM doc-lint:'
320 -tools/doc-lint.sh
321 -echo 'vLLM doc-lint: Done'
pyproject.toml
+8
@@ -15,6 +15,11 @@ build-backend = "setuptools.build_meta"
15 [tool.setuptools_scm]
16 # version_file = "vllm/_version.py" # currently handled by `setup.py:get_version()`
17
18 +[tool.yapfignore]
19 +ignore_patterns = [
20 + "build/**",
21 +]
22 +
23 [tool.ruff]
24 # Allow lines to be as long as 80.
25 line-length = 80
@@ -52,6 +57,9 @@ ignore = [
57 "B007",
58 # f-string format
59 "UP032",
60 + # Python 3.8 typing
61 + "UP006", "UP035",
62 +
63 ]
64
65 [tool.mypy]
requirements-lint.txt
+1 -14
@@ -1,15 +1,2 @@
1 # formatting
2 -yapf==0.32.0
3 -toml==0.10.2
4 -tomli==2.0.2
5 -ruff==0.6.5
6 -codespell==2.3.0
7 -isort==5.13.2
8 -clang-format==18.1.5
9 -pymarkdownlnt==0.9.26
10 -
11 -# type checking
12 -mypy==1.11.1
13 -types-PyYAML
14 -types-requests
15 -types-setuptools
2 +pre-commit==4.0.1
tools/actionlint.sh deleted
-13
@@ -1,13 +0,0 @@
1 -#!/bin/bash
2 -
3 -if command -v actionlint &> /dev/null; then
4 - actionlint "$@"
5 - exit 0
6 -elif [ -x ./actionlint ]; then
7 - ./actionlint "$@"
8 - exit 0
9 -fi
10 -
11 -# download a binary to the current directory - v1.7.3
12 -bash <(curl https://raw.githubusercontent.com/rhysd/actionlint/aa0a7be8e566b096e64a5df8ff290ec24fa58fbc/scripts/download-actionlint.bash)
13 -./actionlint "$@"
tools/doc-lint.sh deleted
-3
@@ -1,3 +0,0 @@
1 -#!/bin/bash
2 -
3 -pymarkdownlnt scan docs -r