A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 08:15 UTC
[CI/Build] drop support for Python 3.8 EOL (#8464)
Signed-off-by: Aaron Pham <contact@aarnphm.xyz>
Aaron Pham committed
Nov 6, 2024 at 02:11 UTC
21063c11c7d340dbb01460e22d98d3619737cd4d
115 files changed
+240
-322
.buildkite/nightly-benchmarks/scripts/convert-results-json-to-markdown.py
+5
-5
@@ -56,7 +56,7 @@ serving_column_mapping = {
56
57
def read_markdown(file):
58
if os.path.exists(file):
59
- with open(file, "r") as f:
59
+ with open(file) as f:
60
return f.read() + "\n"
61
else:
62
return f"{file} not found.\n"
@@ -75,14 +75,14 @@ if __name__ == "__main__":
75
# collect results
76
for test_file in results_folder.glob("*.json"):
77
78
- with open(test_file, "r") as f:
78
+ with open(test_file) as f:
79
raw_result = json.loads(f.read())
80
81
if "serving" in str(test_file):
82
# this result is generated via `benchmark_serving.py`
83
84
# attach the benchmarking command to raw_result
85
- with open(test_file.with_suffix(".commands"), "r") as f:
85
+ with open(test_file.with_suffix(".commands")) as f:
86
command = json.loads(f.read())
87
raw_result.update(command)
88
@@ -97,7 +97,7 @@ if __name__ == "__main__":
97
# this result is generated via `benchmark_latency.py`
98
99
# attach the benchmarking command to raw_result
100
- with open(test_file.with_suffix(".commands"), "r") as f:
100
+ with open(test_file.with_suffix(".commands")) as f:
101
command = json.loads(f.read())
102
raw_result.update(command)
103
@@ -119,7 +119,7 @@ if __name__ == "__main__":
119
# this result is generated via `benchmark_throughput.py`
120
121
# attach the benchmarking command to raw_result
122
- with open(test_file.with_suffix(".commands"), "r") as f:
122
+ with open(test_file.with_suffix(".commands")) as f:
123
command = json.loads(f.read())
124
raw_result.update(command)
125
.buildkite/nightly-benchmarks/scripts/generate-nightly-markdown.py
+2
-2
@@ -72,7 +72,7 @@ def main(args):
72
73
# collect results
74
for test_file in results_folder.glob("*_nightly_results.json"):
75
- with open(test_file, "r") as f:
75
+ with open(test_file) as f:
76
results = results + json.loads(f.read())
77
78
# generate markdown table
@@ -80,7 +80,7 @@ def main(args):
80
81
md_table = tabulate(df, headers='keys', tablefmt='pipe', showindex=False)
82
83
- with open(args.description, "r") as f:
83
+ with open(args.description) as f:
84
description = f.read()
85
86
description = description.format(
.buildkite/nightly-benchmarks/scripts/summary-nightly-results.py
+2
-2
@@ -36,11 +36,11 @@ if __name__ == "__main__":
36
# collect results
37
for test_file in results_folder.glob("*.json"):
38
39
- with open(test_file, "r") as f:
39
+ with open(test_file) as f:
40
raw_result = json.loads(f.read())
41
42
# attach the benchmarking command to raw_result
43
- with open(test_file.with_suffix(".commands"), "r") as f:
43
+ with open(test_file.with_suffix(".commands")) as f:
44
command = json.loads(f.read())
45
raw_result.update(command)
46
.github/workflows/mypy.yaml
+1
-1
@@ -25,7 +25,7 @@ jobs:
25
runs-on: ubuntu-latest
26
strategy:
27
matrix:
28
- python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
28
+ python-version: ["3.9", "3.10", "3.11", "3.12"]
29
steps:
30
- uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
31
- name: Set up Python ${{ matrix.python-version }}
.github/workflows/publish.yml
+1
-1
@@ -48,7 +48,7 @@ jobs:
48
fail-fast: false
49
matrix:
50
os: ['ubuntu-20.04']
51
- python-version: ['3.8', '3.9', '3.10', '3.11', '3.12']
51
+ python-version: ['3.9', '3.10', '3.11', '3.12']
52
pytorch-version: ['2.4.0'] # Must be the most recent version that meets requirements-cuda.txt.
53
cuda-version: ['11.8', '12.1']
54
.github/workflows/ruff.yml
+16
-16
@@ -29,19 +29,19 @@ jobs:
29
matrix:
30
python-version: ["3.12"]
31
steps:
32
- - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
33
- - name: Set up Python ${{ matrix.python-version }}
34
- uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0
35
- with:
36
- python-version: ${{ matrix.python-version }}
37
- - name: Install dependencies
38
- run: |
39
- python -m pip install --upgrade pip
40
- pip install -r requirements-lint.txt
41
- - name: Analysing the code with ruff
42
- run: |
43
- echo "::add-matcher::.github/workflows/matchers/ruff.json"
44
- ruff check --output-format github .
45
- - name: Run isort
46
- run: |
47
- isort . --check-only
32
+ - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
33
+ - name: Set up Python ${{ matrix.python-version }}
34
+ uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0
35
+ with:
36
+ python-version: ${{ matrix.python-version }}
37
+ - name: Install dependencies
38
+ run: |
39
+ python -m pip install --upgrade pip
40
+ pip install -r requirements-lint.txt
41
+ - name: Analysing the code with ruff
42
+ run: |
43
+ echo "::add-matcher::.github/workflows/matchers/ruff.json"
44
+ ruff check --output-format github .
45
+ - name: Run isort
46
+ run: |
47
+ isort . --check-only
.github/workflows/yapf.yml
+13
-13
@@ -23,16 +23,16 @@ jobs:
23
matrix:
24
python-version: ["3.12"]
25
steps:
26
- - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
27
- - name: Set up Python ${{ matrix.python-version }}
28
- uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0
29
- with:
30
- python-version: ${{ matrix.python-version }}
31
- - name: Install dependencies
32
- run: |
33
- python -m pip install --upgrade pip
34
- pip install yapf==0.32.0
35
- pip install toml==0.10.2
36
- - name: Running yapf
37
- run: |
38
- yapf --diff --recursive .
26
+ - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
27
+ - name: Set up Python ${{ matrix.python-version }}
28
+ uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0
29
+ with:
30
+ python-version: ${{ matrix.python-version }}
31
+ - name: Install dependencies
32
+ run: |
33
+ python -m pip install --upgrade pip
34
+ pip install yapf==0.32.0
35
+ pip install toml==0.10.2
36
+ - name: Running yapf
37
+ run: |
38
+ yapf --diff --recursive .
.readthedocs.yaml
+5
-6
@@ -6,17 +6,16 @@ version: 2
6
build:
7
os: ubuntu-22.04
8
tools:
9
- python: "3.8"
9
+ python: '3.9'
10
11
sphinx:
12
- configuration: docs/source/conf.py
13
- fail_on_warning: true
12
+ configuration: docs/source/conf.py
13
+ fail_on_warning: true
14
15
# If using Sphinx, optionally build your docs in additional formats such as PDF
16
formats: []
17
18
# Optionally declare the Python requirements required to build your docs
19
python:
20
- install:
21
- - requirements: docs/requirements-docs.txt
22
-
20
+ install:
21
+ - requirements: docs/requirements-docs.txt
CMakeLists.txt
+18
-18
@@ -128,9 +128,9 @@ endif()
128
129
if(VLLM_GPU_LANG STREQUAL "CUDA")
130
#
131
- # For cuda we want to be able to control which architectures we compile for on
131
+ # For cuda we want to be able to control which architectures we compile for on
132
# a per-file basis in order to cut down on compile time. So here we extract
133
- # the set of architectures we want to compile for and remove the from the
133
+ # the set of architectures we want to compile for and remove the from the
134
# CMAKE_CUDA_FLAGS so that they are not applied globally.
135
#
136
clear_cuda_arches(CUDA_ARCH_FLAGS)
@@ -138,7 +138,7 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
138
message(STATUS "CUDA target architectures: ${CUDA_ARCHS}")
139
# Filter the target architectures by the supported supported archs
140
# since for some files we will build for all CUDA_ARCHS.
141
- cuda_archs_loose_intersection(CUDA_ARCHS
141
+ cuda_archs_loose_intersection(CUDA_ARCHS
142
"${CUDA_SUPPORTED_ARCHS}" "${CUDA_ARCHS}")
143
message(STATUS "CUDA supported target architectures: ${CUDA_ARCHS}")
144
else()
@@ -236,7 +236,7 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
236
# are not supported by Machete yet.
237
cuda_archs_loose_intersection(MARLIN_ARCHS "8.0;8.6;8.9;9.0" ${CUDA_ARCHS})
238
if (MARLIN_ARCHS)
239
- set(MARLIN_SRCS
239
+ set(MARLIN_SRCS
240
"csrc/quantization/fp8/fp8_marlin.cu"
241
"csrc/quantization/marlin/dense/marlin_cuda_kernel.cu"
242
"csrc/quantization/marlin/sparse/marlin_24_cuda_kernel.cu"
@@ -277,7 +277,7 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
277
"in CUDA target architectures")
278
endif()
279
280
- # clear SCALED_MM_3X_ARCHS so the scaled_mm_c2x kernels know we didn't
280
+ # clear SCALED_MM_3X_ARCHS so the scaled_mm_c2x kernels know we didn't
281
# build any 3x kernels
282
set(SCALED_MM_3X_ARCHS)
283
endif()
@@ -285,7 +285,7 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
285
#
286
# For the cutlass_scaled_mm kernels we want to build the c2x (CUTLASS 2.x)
287
# kernels for the remaining archs that are not already built for 3x.
288
- cuda_archs_loose_intersection(SCALED_MM_2X_ARCHS
288
+ cuda_archs_loose_intersection(SCALED_MM_2X_ARCHS
289
"7.5;8.0;8.6;8.9;9.0" "${CUDA_ARCHS}")
290
# subtract out the archs that are already built for 3x
291
list(REMOVE_ITEM SCALED_MM_2X_ARCHS ${SCALED_MM_3X_ARCHS})
@@ -316,10 +316,10 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
316
cuda_archs_loose_intersection(MACHETE_ARCHS "9.0a" "${CUDA_ARCHS}")
317
if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER 12.0 AND MACHETE_ARCHS)
318
#
319
- # For the Machete kernels we automatically generate sources for various
319
+ # For the Machete kernels we automatically generate sources for various
320
# preselected input type pairs and schedules.
321
# Generate sources:
322
- set(MACHETE_GEN_SCRIPT
322
+ set(MACHETE_GEN_SCRIPT
323
${CMAKE_CURRENT_SOURCE_DIR}/csrc/quantization/machete/generate.py)
324
file(MD5 ${MACHETE_GEN_SCRIPT} MACHETE_GEN_SCRIPT_HASH)
325
@@ -329,8 +329,8 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
329
if (NOT DEFINED CACHE{MACHETE_GEN_SCRIPT_HASH}
330
OR NOT $CACHE{MACHETE_GEN_SCRIPT_HASH} STREQUAL ${MACHETE_GEN_SCRIPT_HASH})
331
execute_process(
332
- COMMAND ${CMAKE_COMMAND} -E env
333
- PYTHONPATH=${CMAKE_CURRENT_SOURCE_DIR}/csrc/cutlass_extensions/:${CUTLASS_DIR}/python/:${VLLM_PYTHON_PATH}:$PYTHONPATH
332
+ COMMAND ${CMAKE_COMMAND} -E env
333
+ PYTHONPATH=${CMAKE_CURRENT_SOURCE_DIR}/csrc/cutlass_extensions/:${CUTLASS_DIR}/python/:${VLLM_PYTHON_PATH}:$PYTHONPATH
334
${Python_EXECUTABLE} ${MACHETE_GEN_SCRIPT}
335
RESULT_VARIABLE machete_generation_result
336
OUTPUT_VARIABLE machete_generation_output
@@ -340,11 +340,11 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
340
341
if (NOT machete_generation_result EQUAL 0)
342
message(FATAL_ERROR "Machete generation failed."
343
- " Result: \"${machete_generation_result}\""
343
+ " Result: \"${machete_generation_result}\""
344
"\nCheck the log for details: "
345
"${CMAKE_CURRENT_BINARY_DIR}/machete_generation.log")
346
else()
347
- set(MACHETE_GEN_SCRIPT_HASH ${MACHETE_GEN_SCRIPT_HASH}
347
+ set(MACHETE_GEN_SCRIPT_HASH ${MACHETE_GEN_SCRIPT_HASH}
348
CACHE STRING "Last run machete generate script hash" FORCE)
349
message(STATUS "Machete generation completed successfully.")
350
endif()
@@ -366,7 +366,7 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
366
367
message(STATUS "Building Machete kernels for archs: ${MACHETE_ARCHS}")
368
else()
369
- if (NOT ${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER 12.0
369
+ if (NOT ${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER 12.0
370
AND MACHETE_ARCHS)
371
message(STATUS "Not building Machete kernels as CUDA Compiler version is "
372
"not >= 12.0, we recommend upgrading to CUDA 12.0 or "
@@ -392,8 +392,8 @@ define_gpu_extension_target(
392
USE_SABI 3
393
WITH_SOABI)
394
395
-# If CUTLASS is compiled on NVCC >= 12.5, it by default uses
396
-# cudaGetDriverEntryPointByVersion as a wrapper to avoid directly calling the
395
+# If CUTLASS is compiled on NVCC >= 12.5, it by default uses
396
+# cudaGetDriverEntryPointByVersion as a wrapper to avoid directly calling the
397
# driver API. This causes problems when linking with earlier versions of CUDA.
398
# Setting this variable sidesteps the issue by calling the driver directly.
399
target_compile_definitions(_C PRIVATE CUTLASS_ENABLE_DIRECT_CUDA_DRIVER_CALL=1)
@@ -471,9 +471,9 @@ if (NOT VLLM_TARGET_DEVICE STREQUAL "cuda")
471
return()
472
endif ()
473
474
-# vLLM flash attention requires VLLM_GPU_ARCHES to contain the set of target
475
-# arches in the CMake syntax (75-real, 89-virtual, etc), since we clear the
476
-# arches in the CUDA case (and instead set the gencodes on a per file basis)
474
+# vLLM flash attention requires VLLM_GPU_ARCHES to contain the set of target
475
+# arches in the CMake syntax (75-real, 89-virtual, etc), since we clear the
476
+# arches in the CUDA case (and instead set the gencodes on a per file basis)
477
# we need to manually set VLLM_GPU_ARCHES here.
478
if(VLLM_GPU_LANG STREQUAL "CUDA")
479
foreach(_ARCH ${CUDA_ARCHS})
benchmarks/backend_request_func.py
+7
-15
@@ -79,7 +79,7 @@ async def async_request_tgi(
79
# any data, we should skip it.
80
if chunk_bytes.startswith(":"):
81
continue
82
- chunk = remove_prefix(chunk_bytes, "data:")
82
+ chunk = chunk_bytes.removeprefix("data:")
83
84
data = json.loads(chunk)
85
timestamp = time.perf_counter()
@@ -144,8 +144,8 @@ async def async_request_trt_llm(
144
if not chunk_bytes:
145
continue
146
147
- chunk = remove_prefix(chunk_bytes.decode("utf-8"),
148
- "data:")
147
+ chunk = chunk_bytes.decode("utf-8").removeprefix(
148
+ "data:")
149
150
data = json.loads(chunk)
151
output.generated_text += data["text_output"]
@@ -261,8 +261,8 @@ async def async_request_openai_completions(
261
if not chunk_bytes:
262
continue
263
264
- chunk = remove_prefix(chunk_bytes.decode("utf-8"),
265
- "data: ")
264
+ chunk = chunk_bytes.decode("utf-8").removeprefix(
265
+ "data: ")
266
if chunk == "[DONE]":
267
latency = time.perf_counter() - st
268
else:
@@ -349,8 +349,8 @@ async def async_request_openai_chat_completions(
349
if not chunk_bytes:
350
continue
351
352
- chunk = remove_prefix(chunk_bytes.decode("utf-8"),
353
- "data: ")
352
+ chunk = chunk_bytes.decode("utf-8").removeprefix(
353
+ "data: ")
354
if chunk == "[DONE]":
355
latency = time.perf_counter() - st
356
else:
@@ -389,14 +389,6 @@ async def async_request_openai_chat_completions(
389
return output
390
391
392
-# Since vllm must support Python 3.8, we can't use str.removeprefix(prefix)
393
-# introduced in Python 3.9
394
-def remove_prefix(text: str, prefix: str) -> str:
395
- if text.startswith(prefix):
396
- return text[len(prefix):]
397
- return text
398
-
399
-
392
def get_model(pretrained_model_name_or_path: str) -> str:
393
if os.getenv('VLLM_USE_MODELSCOPE', 'False').lower() == 'true':
394
from modelscope import snapshot_download
benchmarks/kernels/benchmark_machete.py
+3
-3
@@ -269,10 +269,10 @@ def run_square_bench(args):
269
270
271
def run_range_bench(args):
272
- m_start, k_start, n_start = [int(x) for x in args.dim_start.split(",")]
273
- m_end, k_end, n_end = [int(x) for x in args.dim_end.split(",")]
272
+ m_start, k_start, n_start = (int(x) for x in args.dim_start.split(","))
273
+ m_end, k_end, n_end = (int(x) for x in args.dim_end.split(","))
274
m_increment, k_increment, n_increment = \
275
- [int(x) for x in args.dim_increment.split(",")]
275
+ (int(x) for x in args.dim_increment.split(","))
276
Ms = list(range(m_start, m_end + 1, m_increment))
277
Ks = list(range(k_start, k_end + 1, k_increment))
278
Ns = list(range(n_start, n_end + 1, n_increment))
csrc/quantization/machete/generate.py
+4
-4
@@ -468,7 +468,7 @@ def generate():
468
impl_configs = []
469
470
GPTQ_kernel_type_configs = list(
471
- (TypeConfig(
471
+ TypeConfig(
472
element_a=element_a,
473
element_b=element_b,
474
element_b_scale=element_a,
@@ -476,7 +476,7 @@ def generate():
476
element_d=element_a,
477
accumulator=DataType.f32,
478
) for element_b in (VLLMDataType.u4b8, VLLMDataType.u8b128)
479
- for element_a in (DataType.f16, DataType.bf16)))
479
+ for element_a in (DataType.f16, DataType.bf16))
480
481
GPTQ_kernel_specializations = [
482
Specialization(with_C=False, with_zeropoints=False, with_scales=True)
@@ -490,7 +490,7 @@ def generate():
490
]
491
492
AWQ_kernel_type_configs = list(
493
- (TypeConfig(
493
+ TypeConfig(
494
element_a=element_a,
495
element_b=element_b,
496
element_b_scale=element_a,
@@ -498,7 +498,7 @@ def generate():
498
element_d=element_a,
499
accumulator=DataType.f32,
500
) for element_b in (DataType.u4, DataType.u8)
501
- for element_a in (DataType.f16, DataType.bf16)))
501
+ for element_a in (DataType.f16, DataType.bf16))
502
503
AWQ_kernel_specializations = [
504
Specialization(with_C=False, with_zeropoints=True, with_scales=True)
docs/source/getting_started/installation.rst
+5
-5
@@ -10,7 +10,7 @@ Requirements
10
============
11
12
* OS: Linux
13
-* Python: 3.8 - 3.12
13
+* Python: 3.9 -- 3.12
14
* GPU: compute capability 7.0 or higher (e.g., V100, T4, RTX20xx, A100, L4, H100, etc.)
15
16
Install released versions
@@ -148,7 +148,7 @@ If you want to modify C++ or CUDA code, you'll need to build vLLM from source. T
148
.. tip::
149
150
Building from source requires a lot of compilation. If you are building from source repeatedly, it's more efficient to cache the compilation results.
151
- For example, you can install `ccache <https://github.com/ccache/ccache>`_ using ``conda install ccache`` or ``apt install ccache`` .
151
+ For example, you can install `ccache <https://github.com/ccache/ccache>`_ using ``conda install ccache`` or ``apt install ccache`` .
152
As long as ``which ccache`` command can find the ``ccache`` binary, it will be used automatically by the build system. After the first build, subsequent builds will be much faster.
153
154
@@ -181,8 +181,8 @@ to be run simultaneously, via the environment variable ``MAX_JOBS``. For example
181
$ export MAX_JOBS=6
182
$ pip install -e .
183
184
-This is especially useful when you are building on less powerful machines. For example, when you use WSL it only `assigns 50% of the total memory by default <https://learn.microsoft.com/en-us/windows/wsl/wsl-config#main-wsl-settings>`_, so using ``export MAX_JOBS=1`` can avoid compiling multiple files simultaneously and running out of memory.
185
-A side effect is a much slower build process.
184
+This is especially useful when you are building on less powerful machines. For example, when you use WSL it only `assigns 50% of the total memory by default <https://learn.microsoft.com/en-us/windows/wsl/wsl-config#main-wsl-settings>`_, so using ``export MAX_JOBS=1`` can avoid compiling multiple files simultaneously and running out of memory.
185
+A side effect is a much slower build process.
186
187
Additionally, if you have trouble building vLLM, we recommend using the NVIDIA PyTorch Docker image.
188
@@ -209,7 +209,7 @@ Here is a sanity check to verify that the CUDA Toolkit is correctly installed:
209
Unsupported OS build
210
--------------------
211
212
-vLLM can fully run only on Linux but for development purposes, you can still build it on other systems (for example, macOS), allowing for imports and a more convenient development environment. The binaries will not be compiled and won't work on non-Linux systems.
212
+vLLM can fully run only on Linux but for development purposes, you can still build it on other systems (for example, macOS), allowing for imports and a more convenient development environment. The binaries will not be compiled and won't work on non-Linux systems.
213
214
Simply disable the ``VLLM_TARGET_DEVICE`` environment variable before installing:
215
pyproject.toml
+2
-2
@@ -34,7 +34,7 @@ select = [
34
# Pyflakes
35
"F",
36
# pyupgrade
37
- # "UP",
37
+ "UP",
38
# flake8-bugbear
39
"B",
40
# flake8-simplify
@@ -55,7 +55,7 @@ ignore = [
55
]
56
57
[tool.mypy]
58
-python_version = "3.8"
58
+python_version = "3.9"
59
60
ignore_missing_imports = true
61
check_untyped_defs = true
setup.py
+4
-5
@@ -1,5 +1,4 @@
1
import importlib.util
2
-import io
2
import logging
3
import os
4
import re
@@ -327,7 +326,7 @@ def get_neuronxcc_version():
326
"__init__.py")
327
328
# Check if the command was executed successfully
330
- with open(version_file, "rt") as fp:
329
+ with open(version_file) as fp:
330
content = fp.read()
331
332
# Extract the version using a regular expression
@@ -404,7 +403,8 @@ def read_readme() -> str:
403
"""Read the README file if present."""
404
p = get_path("README.md")
405
if os.path.isfile(p):
407
- return io.open(get_path("README.md"), "r", encoding="utf-8").read()
406
+ with open(get_path("README.md"), encoding="utf-8") as f:
407
+ return f.read()
408
else:
409
return ""
410
@@ -498,7 +498,6 @@ setup(
498
"Documentation": "https://vllm.readthedocs.io/en/latest/",
499
},
500
classifiers=[
501
- "Programming Language :: Python :: 3.8",
501
"Programming Language :: Python :: 3.9",
502
"Programming Language :: Python :: 3.10",
503
"Programming Language :: Python :: 3.11",
@@ -512,7 +511,7 @@ setup(
511
],
512
packages=find_packages(exclude=("benchmarks", "csrc", "docs", "examples",
513
"tests*")),
515
- python_requires=">=3.8",
514
+ python_requires=">=3.9",
515
install_requires=get_requirements(),
516
ext_modules=ext_modules,
517
extras_require={
tests/compile/piecewise/test_toy_llama.py
+2
-2
@@ -429,8 +429,8 @@ def benchmark():
429
# print in tabular format
430
print("batch size\teager mode\tfull cudagraph\tpiecewise cudagraph")
431
for b in cudagraph_sizes:
432
- print((f"{b}\t{eager_time[b]:.3f}\t{full_cudagraph_time[b]:.3f}"
433
- f"\t{piecewise_cudagraph_time[b]:.3f}"))
432
+ print(f"{b}\t{eager_time[b]:.3f}\t{full_cudagraph_time[b]:.3f}"
433
+ f"\t{piecewise_cudagraph_time[b]:.3f}")
434
435
436
if __name__ == "__main__":
tests/conftest.py
+8
-21
@@ -1,6 +1,5 @@
1
import json
2
import os
3
-import sys
3
import tempfile
4
from collections import UserList
5
from enum import Enum
@@ -52,7 +51,7 @@ PromptVideoInput = _PromptMultiModalInput[np.ndarray]
51
52
53
def _read_prompts(filename: str) -> List[str]:
55
- with open(filename, "r") as f:
54
+ with open(filename) as f:
55
prompts = f.readlines()
56
return prompts
57
@@ -62,14 +61,8 @@ class _ImageAssetPrompts(TypedDict):
61
cherry_blossom: str
62
63
65
-if sys.version_info < (3, 9):
66
- # UserList cannot be subscripted
67
- class _ImageAssetsBase(UserList):
68
- pass
69
-else:
70
-
71
- class _ImageAssetsBase(UserList[ImageAsset]):
72
- pass
64
+class _ImageAssetsBase(UserList[ImageAsset]):
65
+ pass
66
67
68
class _ImageAssets(_ImageAssetsBase):
@@ -94,14 +87,8 @@ class _VideoAssetPrompts(TypedDict):
87
sample_demo_1: str
88
89
97
-if sys.version_info < (3, 9):
98
- # UserList cannot be subscripted
99
- class _VideoAssetsBase(UserList):
100
- pass
101
-else:
102
-
103
- class _VideoAssetsBase(UserList[VideoAsset]):
104
- pass
90
+class _VideoAssetsBase(UserList[VideoAsset]):
91
+ pass
92
93
94
class _VideoAssets(_VideoAssetsBase):
@@ -958,7 +945,7 @@ def dummy_opt_path():
945
"*.msgpack"
946
])
947
assert os.path.exists(json_path)
961
- with open(json_path, "r") as f:
948
+ with open(json_path) as f:
949
config = json.load(f)
950
config["architectures"] = ["MyOPTForCausalLM"]
951
with open(json_path, "w") as f:
@@ -977,7 +964,7 @@ def dummy_llava_path():
964
"*.msgpack"
965
])
966
assert os.path.exists(json_path)
980
- with open(json_path, "r") as f:
967
+ with open(json_path) as f:
968
config = json.load(f)
969
config["architectures"] = ["MyLlava"]
970
with open(json_path, "w") as f:
@@ -996,7 +983,7 @@ def dummy_gemma2_embedding_path():
983
"*.msgpack"
984
])
985
assert os.path.exists(json_path)
999
- with open(json_path, "r") as f:
986
+ with open(json_path) as f:
987
config = json.load(f)
988
config["architectures"] = ["MyGemma2Embedding"]
989
with open(json_path, "w") as f:
tests/core/block/test_prefix_caching_block.py
+5
-7
@@ -99,13 +99,11 @@ class TestPrefixCachingBlock:
99
100
token_ids = [random.randint(0, 50_000) for _ in range(num_tokens)]
101
102
- first_chain, second_chain = [
103
- TestPrefixCachingBlock.create_chain(
104
- block_size=block_size,
105
- token_ids=token_ids,
106
- num_empty_trailing_blocks=num_empty_trailing_blocks)
107
- for _ in range(2)
108
- ]
102
+ first_chain, second_chain = (TestPrefixCachingBlock.create_chain(
103
+ block_size=block_size,
104
+ token_ids=token_ids,
105
+ num_empty_trailing_blocks=num_empty_trailing_blocks)
106
+ for _ in range(2))
107
108
for first_chain_block, second_chain_block in zip(
109
first_chain, second_chain):
tests/kernels/test_mamba_ssm.py
+1
-1
@@ -510,7 +510,7 @@ def test_selective_scan_varlen(with_padding, is_variable_B, is_variable_C,
510
for var in (u_ref, delta_ref, B_ref, C_ref, z_ref)
511
]
512
for i in range(len(seqlens[0])):
513
- u_s, delta_s, B_s, C_s, z_s = [v[i].unsqueeze(0) for v in splits]
513
+ u_s, delta_s, B_s, C_s, z_s = (v[i].unsqueeze(0) for v in splits)
514
if padded_state_indices[i] == PAD_SLOT_ID:
515
continue
516
out_ref_s, _ = selective_scan_ref(
tests/models/decoder_only/vision_language/mm_processor_kwargs/test_qwen.py
+1
-1
@@ -104,7 +104,7 @@ def test_input_mapper_valid_mm_data(input_mapper_for_qwen,
104
# Sad path tests for the multimodal input processor and mapper, respectively
105
@pytest.mark.parametrize("mm_data", [
106
{
107
- "image": torch.rand((5))
107
+ "image": torch.rand(5)
108
},
109
{
110
"image": torch.rand((5, 5, 5, 5, 5))
tests/samplers/test_rejection_sampler.py
+4
-6
@@ -413,12 +413,10 @@ class _CorrectnessTestHelper:
413
def generate_probs_for_test(
414
self, draft_and_target_probs_equal: bool
415
) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
416
- draft_probs, target_probs = [
417
- F.softmax(
418
- torch.rand(self.vocab_size, dtype=torch.float32),
419
- dim=-1,
420
- ) for _ in range(2)
421
- ]
416
+ draft_probs, target_probs = (F.softmax(
417
+ torch.rand(self.vocab_size, dtype=torch.float32),
418
+ dim=-1,
419
+ ) for _ in range(2))
420
421
num_reference_probs = 100
422
reference_probs = F.softmax(
tests/test_logger.py
+1
-1
@@ -29,7 +29,7 @@ def test_trace_function_call():
29
cur_dir = os.path.dirname(__file__)
30
enable_trace_function_call(path, cur_dir)
31
f1(1)
32
- with open(path, 'r') as f:
32
+ with open(path) as f:
33
content = f.read()
34
35
assert "f1" in content
tests/tokenization/test_detokenize.py
+2
-2
@@ -93,10 +93,10 @@ def test_mistral_edge_case(tokenizer, truth):
93
def skip_special_tokens(request, tokenizer_name) -> Generator[bool, Any, None]:
94
if "mistral" in tokenizer_name:
95
yield (
96
- bool(True) if request.param else
96
+ True if request.param else
97
pytest.skip("mistral doesn't support skip_special_tokens=False"))
98
else:
99
- yield bool(True) if request.param else bool(False)
99
+ yield bool(request.param)
100
101
102
@pytest.mark.parametrize("truth", TRUTH)
tools/profiler/print_layerwise_table.py
+1
-1
@@ -46,7 +46,7 @@ if __name__ == "__main__":
46
47
args = parser.parse_args()
48
49
- with open(args.json_trace, "r") as f:
49
+ with open(args.json_trace) as f:
50
profile_data = json.load(f)
51
52
if args.table == "summary":
tools/profiler/visualize_layerwise_profile.py
+1
-1
@@ -434,7 +434,7 @@ def main(
434
f"{', Sparsity ' + sparsity if sparsity else ''}")
435
436
profile_json = None
437
- with open(json_trace, "r") as f:
437
+ with open(json_trace) as f:
438
profile_json = json.load(f)
439
assert profile_json is not None
440
tools/report_build_time_ninja.py
+17
-15
@@ -81,7 +81,7 @@ class Target:
81
# Allow for modest floating-point errors
82
epsilon = 0.000002
83
if (self.weighted_duration > self.Duration() + epsilon):
84
- print('%s > %s?' % (self.weighted_duration, self.Duration()))
84
+ print('{} > {}?'.format(self.weighted_duration, self.Duration()))
85
assert (self.weighted_duration <= self.Duration() + epsilon)
86
return self.weighted_duration
87
@@ -104,7 +104,7 @@ def ReadTargets(log, show_all):
104
The result is a list of Target objects."""
105
header = log.readline()
106
assert header == '# ninja log v5\n', \
107
- 'unrecognized ninja log version %r' % header
107
+ 'unrecognized ninja log version {!r}'.format(header)
108
targets_dict = {}
109
last_end_seen = 0.0
110
for line in log:
@@ -254,8 +254,8 @@ def SummarizeEntries(entries, extra_step_types):
254
# Warn if the sum of weighted times is off by more than half a second.
255
if abs(length - weighted_total) > 500:
256
print('Warning: Possible corrupt ninja log, results may be '
257
- 'untrustworthy. Length = %.3f, weighted total = %.3f' %
258
- (length, weighted_total))
257
+ 'untrustworthy. Length = {:.3f}, weighted total = {:.3f}'.format(
258
+ length, weighted_total))
259
260
entries_by_ext = defaultdict(list)
261
for target in entries:
@@ -263,16 +263,17 @@ def SummarizeEntries(entries, extra_step_types):
263
entries_by_ext[extension].append(target)
264
265
for key, values in entries_by_ext.items():
266
- print(' Longest build steps for %s:' % key)
266
+ print(' Longest build steps for {}:'.format(key))
267
values.sort(key=lambda x: x.WeightedDuration())
268
for target in values[-long_count:]:
269
- print(' %8.1f weighted s to build %s (%.1f s elapsed time)' %
270
- (target.WeightedDuration(), target.DescribeTargets(),
271
- target.Duration()))
272
-
273
- print(' %.1f s weighted time (%.1f s elapsed time sum, %1.1fx '
274
- 'parallelism)' %
275
- (length, total_cpu_time, total_cpu_time * 1.0 / length))
269
+ print(
270
+ ' {:8.1f} weighted s to build {} ({:.1f} s elapsed time)'.
271
+ format(target.WeightedDuration(), target.DescribeTargets(),
272
+ target.Duration()))
273
+
274
+ print(' {:.1f} s weighted time ({:.1f} s elapsed time sum, {:1.1f}x '
275
+ 'parallelism)'.format(length, total_cpu_time,
276
+ total_cpu_time * 1.0 / length))
277
print(' %d build steps completed, average of %1.2f/s' %
278
(len(entries), len(entries) / (length)))
279
@@ -298,11 +299,12 @@ def main():
299
long_ext_count += len(args.step_types.split(';'))
300
301
try:
301
- with open(log_file, 'r') as log:
302
+ with open(log_file) as log:
303
entries = ReadTargets(log, False)
304
SummarizeEntries(entries, args.step_types)
304
- except IOError:
305
- print('Log file %r not found, no build summary created.' % log_file)
305
+ except OSError:
306
+ print('Log file {!r} not found, no build summary created.'.format(
307
+ log_file))
308
return errno.ENOENT
309
310
use_existing_torch.py
+1
-1
@@ -4,7 +4,7 @@ requires_files = glob.glob('requirements*.txt')
4
requires_files += ["pyproject.toml"]
5
for file in requires_files:
6
print(f">>> cleaning {file}")
7
- with open(file, 'r') as f:
7
+ with open(file) as f:
8
lines = f.readlines()
9
if "torch" in "".join(lines).lower():
10
print("removed:")
vllm/attention/ops/blocksparse_attention/interface.py
+2
-4
@@ -192,10 +192,8 @@ class LocalStridedBlockSparseAttn(torch.nn.Module):
192
attn_mask = self.dense_attn_mask[None, :, :maxlen, :maxlen]
193
194
q2 = self.transpose_and_pad(q, cu_seqlens, maxlen, 1)
195
- k2, v2 = [
196
- self.transpose_and_pad(x, cu_seqlens, maxlen, q_k_ratio)
197
- for x in [k, v]
198
- ]
195
+ k2, v2 = (self.transpose_and_pad(x, cu_seqlens, maxlen, q_k_ratio)
196
+ for x in [k, v])
197
spda_output = torch.nn.functional.scaled_dot_product_attention(
198
q2, k2, v2, attn_mask=attn_mask, scale=sm_scale)
199
return self.transpose_and_unpad(spda_output, cu_seqlens)
vllm/config.py
+4
-3
@@ -668,9 +668,10 @@ class ModelConfig:
668
@property
669
def is_encoder_decoder_model(self) -> bool:
670
"""Extract the HF encoder/decoder model flag."""
671
- return getattr(self.hf_config, "is_encoder_decoder", False) or (
672
- (hasattr(self.hf_config, "text_config") and getattr(
673
- self.hf_config.text_config, "is_encoder_decoder", False)))
671
+ return getattr(
672
+ self.hf_config, "is_encoder_decoder",
673
+ False) or (hasattr(self.hf_config, "text_config") and getattr(
674
+ self.hf_config.text_config, "is_encoder_decoder", False))
675
676
@property
677
def is_multimodal_model(self) -> bool:
vllm/core/evictor.py
+1
-1
@@ -52,7 +52,7 @@ class Evictor(ABC):
52
pass
53
54
55
-class BlockMetaData():
55
+class BlockMetaData:
56
"""Data structure for storing key data describe cached block, so that
57
evitor could use to make its decision which one to choose for eviction
58
vllm/distributed/device_communicators/custom_all_reduce_utils.py
+1
-1
@@ -240,7 +240,7 @@ def gpu_p2p_access_check(src: int, tgt: int) -> bool:
240
if is_distributed:
241
get_world_group().barrier()
242
logger.info("reading GPU P2P access cache from %s", path)
243
- with open(path, "r") as f:
243
+ with open(path) as f:
244
cache = json.load(f)
245
_gpu_p2p_access_cache = cache
246
return _gpu_p2p_access_cache[f"{src}->{tgt}"]
vllm/engine/async_llm_engine.py
+1
-1
@@ -812,7 +812,7 @@ class AsyncLLMEngine(EngineClient):
812
async def run_engine_loop(engine_ref: ReferenceType):
813
"""We use a weakref to the engine so that the running loop
814
doesn't prevent the engine being garbage collected."""
815
- engine: Optional["AsyncLLMEngine"] = engine_ref()
815
+ engine: Optional[AsyncLLMEngine] = engine_ref()
816
if not engine:
817
return
818
vllm/engine/llm_engine.py
+2
-2
@@ -1541,8 +1541,8 @@ class LLMEngine:
1541
seq_group.state.remaining_steps != ref_remaining_steps
1542
for seq_group in seq_group_metadata_list[1:]
1543
]):
1544
- raise AssertionError(("All running sequence groups should "
1545
- "have the same remaining steps."))
1544
+ raise AssertionError("All running sequence groups should "
1545
+ "have the same remaining steps.")
1546
1547
return ref_remaining_steps > 0
1548
vllm/engine/metrics_types.py
+1
-1
@@ -77,7 +77,7 @@ class StatLoggerBase(ABC):
77
self.num_generation_tokens: List[int] = []
78
self.last_local_log = time.time()
79
self.local_interval = local_interval
80
- self.spec_decode_metrics: Optional["SpecDecodeWorkerMetrics"] = None
80
+ self.spec_decode_metrics: Optional[SpecDecodeWorkerMetrics] = None
81
82
@abstractmethod
83
def log(self, stats: Stats) -> None:
vllm/engine/output_processor/multi_step.py
+1
-1
@@ -63,7 +63,7 @@ class MultiStepOutputProcessor(SequenceGroupOutputProcessor):
63
single_step_process_prompt_logprob(self, seq_group, output)
64
65
@staticmethod
66
- @functools.lru_cache()
66
+ @functools.lru_cache
67
def _log_prompt_logprob_unsupported_warning_once():
68
# Reminder: Please update docs/source/serving/compatibility_matrix.rst
69
# If the feature combo become valid
vllm/entrypoints/chat_utils.py
+1
-1
@@ -362,7 +362,7 @@ def load_chat_template(
362
if chat_template is None:
363
return None
364
try:
365
- with open(chat_template, "r") as f:
365
+ with open(chat_template) as f:
366
resolved_chat_template = f.read()
367
except OSError as e:
368
if isinstance(chat_template, Path):
vllm/entrypoints/openai/run_batch.py
+1
-1
@@ -120,7 +120,7 @@ async def read_file(path_or_url: str) -> str:
120
session.get(path_or_url) as resp:
121
return await resp.text()
122
else:
123
- with open(path_or_url, "r", encoding="utf-8") as f:
123
+ with open(path_or_url, encoding="utf-8") as f:
124
return f.read()
125
126
vllm/executor/ray_gpu_executor.py
+1
-1
@@ -32,7 +32,7 @@ class RayGPUExecutor(DistributedGPUExecutor):
32
uses_ray: bool = True
33
34
def _init_executor(self) -> None:
35
- self.forward_dag: Optional["ray.dag.CompiledDAG"] = None
35
+ self.forward_dag: Optional[ray.dag.CompiledDAG] = None
36
# If the env var is set, it uses the Ray's compiled DAG API
37
# which optimizes the control plane overhead.
38
# Run vLLM with VLLM_USE_RAY_COMPILED_DAG=1 to enable it.
vllm/logger.py
+1
-2
@@ -67,8 +67,7 @@ def _configure_vllm_root_logger() -> None:
67
raise RuntimeError(
68
"Could not load logging config. File does not exist: %s",
69
VLLM_LOGGING_CONFIG_PATH)
70
- with open(VLLM_LOGGING_CONFIG_PATH, encoding="utf-8",
71
- mode="r") as file:
70
+ with open(VLLM_LOGGING_CONFIG_PATH, encoding="utf-8") as file:
71
custom_config = json.loads(file.read())
72
73
if not isinstance(custom_config, dict):
vllm/lora/models.py
+2
-2
@@ -343,7 +343,7 @@ class LoRAModelManager(AdapterModelManager):
343
# text modules (e.g. ChatGLM)
344
and hasattr(self.model, "get_mm_mapping"))
345
self.packed_modules: Dict[str, List[str]] = {}
346
- self.modules: Dict[str, "BaseLayerWithLoRA"] = {}
346
+ self.modules: Dict[str, BaseLayerWithLoRA] = {}
347
# Dict instead of a Set for compatibility with LRUCache.
348
self._last_mapping: Optional[LoRAMapping] = None
349
self._create_lora_modules()
@@ -548,7 +548,7 @@ class LoRAModelManager(AdapterModelManager):
548
else:
549
parts = module_name.split(".")
550
replacements = self.packed_modules_mapping[parts[-1]]
551
- subloras: List[Optional["LoRALayerWeights"]] = []
551
+ subloras: List[Optional[LoRALayerWeights]] = []
552
for i, r in enumerate(replacements):
553
lora = LoRALayerWeights.create_dummy_lora_weights(
554
module_name + "." + r,
vllm/model_executor/custom_op.py
+1
-1
@@ -103,7 +103,7 @@ class CustomOp(nn.Module):
103
# On by default if VLLM_TORCH_COMPILE_LEVEL < CompilationLevel.PIECEWISE
104
# Specifying 'all' or 'none' in VLLM_CUSTOM_OPS takes precedence.
105
@staticmethod
106
- @lru_cache()
106
+ @lru_cache
107
def default_on() -> bool:
108
count_none = envs.VLLM_CUSTOM_OPS.count("none")
109
count_all = envs.VLLM_CUSTOM_OPS.count("all")
vllm/model_executor/layers/resampler.py
-1
@@ -1,4 +1,3 @@
1
-# coding=utf-8
1
# Adapted from
2
# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py
3
# https://huggingface.co/Qwen/Qwen-7B/blob/main/modeling_qwen.py
vllm/model_executor/layers/rotary_embedding.py
-1
@@ -1,4 +1,3 @@
1
-# coding=utf-8
1
# Adapted from
2
# https://github.com/huggingface/transformers/blob/v4.33.2/src/transformers/models/llama/modeling_llama.py
3
# Copyright 2023 The vLLM team.
vllm/model_executor/model_loader/loader.py
+1
-1
@@ -746,7 +746,7 @@ class BitsAndBytesModelLoader(BaseModelLoader):
746
747
config_file_path = self._get_config_file(qlora_adapter)
748
749
- with open(config_file_path, "r") as f:
749
+ with open(config_file_path) as f:
750
config = json.load(f)
751
self.target_modules = config["target_modules"]
752
vllm/model_executor/model_loader/openvino.py
+1
-1
@@ -190,7 +190,7 @@ def get_model(
190
kv_cache_dtype: ov.Type,
191
**kwargs,
192
) -> torch.nn.Module:
193
- lora_config = kwargs.get("lora_config", None)
193
+ lora_config = kwargs.get("lora_config")
194
ov_core = kwargs.get("ov_core")
195
if lora_config:
196
raise ValueError(
vllm/model_executor/model_loader/tensorizer.py
+2
-3
@@ -280,7 +280,7 @@ class TensorizerAgent:
280
self.tensorizer_args = (
281
self.tensorizer_config._construct_tensorizer_args())
282
self.extra_kwargs = extra_kwargs
283
- if extra_kwargs.get("quant_config", None) is not None:
283
+ if extra_kwargs.get("quant_config") is not None:
284
self.quant_config = extra_kwargs["quant_config"]
285
else:
286
self.quant_config = quant_config
@@ -380,8 +380,7 @@ def tensorizer_weights_iterator(
380
stream = open_stream(tensorizer_args.tensorizer_uri, **stream_params)
381
with TensorDeserializer(stream, **deserializer_args,
382
device="cpu") as state:
383
- for name, param in state.items():
384
- yield name, param
383
+ yield from state.items()
384
del state
385
386
vllm/model_executor/model_loader/weight_utils.py
+4
-5
@@ -188,7 +188,7 @@ def get_quant_config(model_config: ModelConfig,
188
f"{quant_config_files}")
189
190
quant_config_file = quant_config_files[0]
191
- with open(quant_config_file, "r") as f:
191
+ with open(quant_config_file) as f:
192
config = json.load(f)
193
194
if model_config.quantization == "bitsandbytes":
@@ -306,7 +306,7 @@ def filter_duplicate_safetensors_files(hf_weights_files: List[str],
306
307
# Iterate through the weight_map (weight_name: safetensors files)
308
# to identify weights that we should use.
309
- with open(index_file_name, "r") as f:
309
+ with open(index_file_name) as f:
310
weight_map = json.load(f)["weight_map"]
311
weight_files_in_index = set()
312
for weight_name in weight_map:
@@ -382,7 +382,7 @@ def np_cache_weights_iterator(
382
with open(weight_names_file, "w") as f:
383
json.dump(weight_names, f)
384
385
- with open(weight_names_file, "r") as f:
385
+ with open(weight_names_file) as f:
386
weight_names = json.load(f)
387
388
for name in weight_names:
@@ -423,8 +423,7 @@ def pt_weights_iterator(
423
bar_format=_BAR_FORMAT,
424
):
425
state = torch.load(bin_file, map_location="cpu")
426
- for name, param in state.items():
427
- yield name, param
426
+ yield from state.items()
427
del state
428
torch.cuda.empty_cache()
429
vllm/model_executor/models/arctic.py
+2
-2
@@ -48,7 +48,7 @@ class ArcticMLP(nn.Module):
48
is_residual_mlp: bool = False,
49
quant_config: Optional[QuantizationConfig] = None,
50
reduce_results: bool = True):
51
- super(ArcticMLP, self).__init__()
51
+ super().__init__()
52
self.hidden_size = config.hidden_size
53
self.expert_id = expert_id
54
self.layer_id = layer_id
@@ -89,7 +89,7 @@ class ArcticMoE(nn.Module):
89
params_dtype: Optional[torch.dtype] = None,
90
quant_config: Optional[QuantizationConfig] = None,
91
reduce_results: bool = True):
92
- super(ArcticMoE, self).__init__()
92
+ super().__init__()
93
94
self.tp_size = tp_size or get_tensor_model_parallel_world_size()
95
self.hidden_size = config.hidden_size
vllm/model_executor/models/baichuan.py
-1
@@ -1,4 +1,3 @@
1
-# coding=utf-8
1
# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
2
#
3
# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
vllm/model_executor/models/bloom.py
-1
@@ -1,4 +1,3 @@
1
-# coding=utf-8
1
# Adapted from
2
# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/bloom/modeling_bloom.py
3
# Copyright 2023 The vLLM team.
vllm/model_executor/models/chatglm.py
-1
@@ -1,4 +1,3 @@
1
-# coding=utf-8
1
# Adapted from
2
# https://github.com/THUDM/GLM-4
3
"""Inference-only ChatGLM model compatible with THUDM weights."""
vllm/model_executor/models/commandr.py
-1
@@ -1,4 +1,3 @@
1
-# coding=utf-8
1
# Copyright 2024 Cohere and the HuggingFace Inc. team. All rights reserved.
2
#
3
# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
vllm/model_executor/models/dbrx.py
-1
@@ -1,4 +1,3 @@
1
-# coding=utf-8
1
from typing import Iterable, List, Optional, Tuple, Union
2
3
import torch
vllm/model_executor/models/decilm.py
-1
@@ -1,4 +1,3 @@
1
-# coding=utf-8
1
# Adapted from
2
# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py
3
# Copyright 2023 DeciAI Research Team. All rights reserved.
vllm/model_executor/models/deepseek.py
-1
@@ -1,4 +1,3 @@
1
-# coding=utf-8
1
# Adapted from
2
# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py
3
# Copyright 2023 The vLLM team.
vllm/model_executor/models/deepseek_v2.py
-1
@@ -1,4 +1,3 @@
1
-# coding=utf-8
1
# Adapted from
2
# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py
3
# Copyright 2023 The vLLM team.
vllm/model_executor/models/exaone.py
-1
@@ -1,4 +1,3 @@
1
-# coding=utf-8
1
# Adapted from
2
# https://huggingface.co/LGAI-EXAONE/EXAONE-3.0-7.8B-Instruct/blob/main/modeling_exaone.py
3
# Copyright 2024 The LG U+ CTO AI Tech Lab.
vllm/model_executor/models/falcon.py
-1
@@ -1,4 +1,3 @@
1
-# coding=utf-8
1
# Adapted from
2
# https://github.com/huggingface/transformers/blob/a5cc30d72ae2dc19af534e4b35c986cc28db1275/src/transformers/models/falcon/modeling_falcon.py
3
# Copyright 2023 The vLLM team.
vllm/model_executor/models/fuyu.py
-1
@@ -1,4 +1,3 @@
1
-# coding=utf-8
1
# adapted from https://github.com/huggingface/transformers/blob/v4.39.3/src/transformers/models/fuyu/modeling_fuyu.py
2
# Copyright 2023 The vLLM team.
3
# Copyright 2023 HuggingFace Inc. team. All rights reserved.
vllm/model_executor/models/gemma.py
-1
@@ -1,4 +1,3 @@
1
-# coding=utf-8
1
# Copyright 2023 The vLLM team.
2
# Copyright (c) Google Inc.
3
#
vllm/model_executor/models/gemma2.py
-1
@@ -1,4 +1,3 @@
1
-# coding=utf-8
1
# Copyright 2024 The vLLM team.
2
# Copyright 2024 Google Inc. HuggingFace Inc. team. All rights reserved.
3
#
vllm/model_executor/models/glm4_vision_encoder.py
-1
@@ -1,4 +1,3 @@
1
-# coding=utf-8
1
# Adapted from
2
# https://github.com/THUDM/GLM-4
3
"""Inference-only GLM-4v model visual encoder compatible with THUDM weights."""
vllm/model_executor/models/gpt2.py
-1
@@ -1,4 +1,3 @@
1
-# coding=utf-8
1
# Adapted from
2
# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/gpt2/modeling_gpt2.py
3
# Copyright 2023 The vLLM team.
vllm/model_executor/models/gpt_bigcode.py
-1
@@ -1,4 +1,3 @@
1
-# coding=utf-8
1
# Adapted from
2
# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/gpt2/modeling_gpt2.py
3
# Copyright 2023 The vLLM team.
vllm/model_executor/models/gpt_j.py
-1
@@ -1,4 +1,3 @@
1
-# coding=utf-8
1
# Adapted from
2
# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/gptj/modeling_gptj.py
3
# Copyright 2023 The vLLM team.
vllm/model_executor/models/gpt_neox.py
-1
@@ -1,4 +1,3 @@
1
-# coding=utf-8
1
# Adapted from
2
# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/gpt_neox/modeling_gpt_neox.py
3
# Copyright 2023 The vLLM team.
vllm/model_executor/models/granite.py
-1
@@ -1,4 +1,3 @@
1
-# coding=utf-8
1
# Adapted from
2
# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py
3
# Copyright 2023 The vLLM team.
vllm/model_executor/models/granitemoe.py
-1
@@ -1,4 +1,3 @@
1
-# coding=utf-8
1
# Adapted from
2
# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py
3
# Copyright 2023 The vLLM team.
vllm/model_executor/models/idefics2_vision_model.py
-2
@@ -1,5 +1,3 @@
1
-# coding=utf-8
2
-
1
# adapted from https://github.com/huggingface/transformers/blob/v4.43.2/src/transformers/models/idefics2/modeling_idefics2.py
2
# Copyright 2024 The vLLM team.
3
# Copyright 2024 the HuggingFace Inc. team. All rights reserved.
vllm/model_executor/models/internlm2.py
-1
@@ -1,4 +1,3 @@
1
-# -*- coding: utf-8 -*-
1
from functools import partial
2
from typing import Any, Dict, Iterable, List, Optional, Tuple, Union
3
vllm/model_executor/models/internlm2_ve.py
-1
@@ -1,4 +1,3 @@
1
-# -*- coding: utf-8 -*-
1
from typing import List, Optional, Tuple, Union
2
3
import torch
vllm/model_executor/models/jais.py
-1
@@ -1,4 +1,3 @@
1
-# coding=utf-8
1
# Adapted from
2
# https://huggingface.co/inceptionai/jais-30b-chat-v3/blob/main/modeling_jais.py
3
# Copyright 2023 The vLLM team.
vllm/model_executor/models/jamba.py
-1
@@ -1,4 +1,3 @@
1
-# coding=utf-8
1
"""Inference-only Jamba model."""
2
from typing import Iterable, List, Optional, Tuple
3
vllm/model_executor/models/llama.py
-1
@@ -1,4 +1,3 @@
1
-# coding=utf-8
1
# Adapted from
2
# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py
3
# Copyright 2023 The vLLM team.
vllm/model_executor/models/mamba.py
-1
@@ -1,4 +1,3 @@
1
-# coding=utf-8
1
"""PyTorch MAMBA model."""
2
from typing import Iterable, List, Optional, Tuple
3
vllm/model_executor/models/minicpm.py
-1
@@ -1,4 +1,3 @@
1
-# coding=utf-8
1
# Adapted from
2
# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py
3
# Copyright 2023 The vLLM team.
vllm/model_executor/models/minicpm3.py
-1
@@ -1,4 +1,3 @@
1
-# coding=utf-8
1
# Adapted from
2
# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py
3
# Copyright 2024 The ModelBest team.
vllm/model_executor/models/minicpmv.py
-1
@@ -1,4 +1,3 @@
1
-# coding=utf-8
1
# Adapted from
2
# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py
3
# Copyright 2023 The vLLM team.
vllm/model_executor/models/mixtral.py
-1
@@ -1,4 +1,3 @@
1
-# coding=utf-8
1
# Adapted from
2
# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py
3
# Copyright 2023 The vLLM team.
vllm/model_executor/models/mixtral_quant.py
-1
@@ -1,4 +1,3 @@
1
-# coding=utf-8
1
# Adapted from
2
# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py
3
# Copyright 2023 The vLLM team.
vllm/model_executor/models/mllama.py
-1
@@ -1,4 +1,3 @@
1
-# coding=utf-8
1
# Copyright 2024 the HuggingFace Inc. team. All rights reserved.
2
#
3
# Licensed under the Apache License, Version 2.0 (the "License");
vllm/model_executor/models/mlp_speculator.py
+1
-1
@@ -37,7 +37,7 @@ class MLPSpeculatorLayerNorm(nn.Module):
37
eps=1e-06,
38
elementwise_scale_and_shift=True,
39
):
40
- super(MLPSpeculatorLayerNorm, self).__init__()
40
+ super().__init__()
41
self.elementwise_scale_and_shift = elementwise_scale_and_shift
42
if self.elementwise_scale_and_shift:
43
self.weight = nn.Parameter(torch.empty(normalized_shape))
vllm/model_executor/models/molmo.py
+3
-3
@@ -1121,9 +1121,9 @@ class MolmoForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
1121
batch_size * num_image * num_patch, -1).contiguous()
1122
1123
image_input_idx = image_input_idx * valid.to(image_input_idx.dtype)
1124
- offset = torch.cat(
1125
- [seq_len.new_zeros(
1126
- (1)), seq_len.cumsum(dim=0)[:-1]], dim=0)[:, None]
1124
+ offset = torch.cat([seq_len.new_zeros(1),
1125
+ seq_len.cumsum(dim=0)[:-1]],
1126
+ dim=0)[:, None]
1127
image_input_idx = image_input_idx + offset.to(image_input_idx.dtype)
1128
image_input_idx = image_input_idx.flatten()[:, None]
1129
mat = image_input_idx == torch.arange(
vllm/model_executor/models/mpt.py
-1
@@ -1,4 +1,3 @@
1
-# coding=utf-8
1
# Adapted from https://huggingface.co/mosaicml/mpt-7b/tree/main
2
import math
3
from typing import Iterable, List, Optional, Tuple, Union
vllm/model_executor/models/nemotron.py
-1
@@ -1,4 +1,3 @@
1
-# coding=utf-8
1
# Adapted from
2
# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py
3
# Copyright 2023 The vLLM team.
vllm/model_executor/models/olmo.py
-1
@@ -1,4 +1,3 @@
1
-# coding=utf-8
1
# Adapted from
2
# https://github.com/huggingface/transformers/blob/v4.40.1/src/transformers/models/olmo/modeling_olmo.py
3
# Copyright 2024 The vLLM team.
vllm/model_executor/models/opt.py
-1
@@ -1,4 +1,3 @@
1
-# coding=utf-8
1
# Adapted from
2
# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/opt/modeling_opt.py
3
# Copyright 2023 The vLLM team.
vllm/model_executor/models/orion.py
-1
@@ -1,4 +1,3 @@
1
-# coding=utf-8
1
# Adapted from
2
# https://huggingface.co/OrionStarAI/Orion-14B-Base/blob/main/modeling_orion.py
3
# Copyright (c) OrionStar Inc.
vllm/model_executor/models/persimmon.py
-1
@@ -1,4 +1,3 @@
1
-# coding=utf-8
1
# adapted from https://github.com/huggingface/transformers/blob/v4.39.3/src/transformers/models/persimmon/modeling_persimmon.py
2
# Copyright 2023 The vLLM team.
3
# Copyright 2023 EleutherAI and the HuggingFace Inc. team. All rights reserved.
vllm/model_executor/models/phi.py
-1
@@ -1,4 +1,3 @@
1
-# coding=utf-8
1
# Adapted from
2
# https://huggingface.co/microsoft/phi-1_5/blob/main/modeling_phi.py
3
# Copyright 2023 The vLLM team.
vllm/model_executor/models/phi3.py
-1
@@ -1,4 +1,3 @@
1
-# coding=utf-8
1
# Adapted from llama.py
2
"""Inference-only Phi3 model code inherit from Llama.py"""
3
vllm/model_executor/models/phi3v.py
-1
@@ -1,4 +1,3 @@
1
-# coding=utf-8
1
# Copyright 2024 The vLLM team.
2
# Copyright 2024 Microsoft and the HuggingFace Inc. team. All rights reserved.
3
#
vllm/model_executor/models/phimoe.py
-1
@@ -1,4 +1,3 @@
1
-# coding=utf-8
1
# Adapted from
2
# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py
3
# Copyright 2023 The vLLM team.
vllm/model_executor/models/pixtral.py
+5
-5
@@ -136,11 +136,11 @@ def input_processor_for_pixtral(ctx: InputContext, inputs: DecoderOnlyInputs):
136
137
if image_token_id not in inputs['prompt_token_ids']:
138
raise ValueError(
139
- (f"You've passed {inputs=} without {image_token_id=}"
140
- " Make sure to process your input via mistral_common's"
141
- " tokenizer or pass a chat completion request. For more"
142
- " For more info, see: "
143
- "https://github.com/vllm-project/vllm/issues/8411."))
139
+ f"You've passed {inputs=} without {image_token_id=}"
140
+ " Make sure to process your input via mistral_common's"
141
+ " tokenizer or pass a chat completion request. For more"
142
+ " For more info, see: "
143
+ "https://github.com/vllm-project/vllm/issues/8411.")
144
145
return inputs
146
vllm/model_executor/models/qwen.py
-1
@@ -1,4 +1,3 @@
1
-# coding=utf-8
1
# Adapted from
2
# https://huggingface.co/Qwen/Qwen-7B/blob/main/modeling_qwen.py
3
# Copyright (c) Alibaba Cloud.
vllm/model_executor/models/qwen2.py
+3
-4
@@ -1,4 +1,3 @@
1
-# coding=utf-8
1
# Adapted from
2
# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/qwen2/modeling_qwen2.py
3
# Copyright 2024 The Qwen team.
@@ -417,9 +416,9 @@ class Qwen2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
416
and hasattr(config, "max_window_layers")):
417
raise ValueError("Sliding window for some but all layers is not "
418
"supported. This model uses sliding window "
420
- "but `max_window_layers` = %s is less than "
421
- "`num_hidden_layers` = %s. Please open an issue "
422
- "to discuss this feature." % (
419
+ "but `max_window_layers` = {} is less than "
420
+ "`num_hidden_layers` = {}. Please open an issue "
421
+ "to discuss this feature.".format(
422
config.max_window_layers,
423
config.num_hidden_layers,
424
))
vllm/model_executor/models/qwen2_audio.py
-1
@@ -1,4 +1,3 @@
1
-# coding=utf-8
1
# Copyright 2024 The Qwen team.
2
# Copyright 2023 The vLLM team.
3
# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
vllm/model_executor/models/qwen2_cls.py
+3
-4
@@ -1,4 +1,3 @@
1
-# coding=utf-8
1
# Adapted from
2
# https://huggingface.co/Qwen/Qwen2.5-Math-RM-72B/blob/main/modeling_qwen2_rm.py
3
# Copyright 2024 Kakao Corp. (Kanana-X Team)
@@ -60,9 +59,9 @@ class Qwen2ForSequenceClassification(nn.Module):
59
and hasattr(config, "max_window_layers")):
60
raise ValueError("Sliding window for some but all layers is not "
61
"supported. This model uses sliding window "
63
- "but `max_window_layers` = %s is less than "
64
- "`num_hidden_layers` = %s. Please open an issue "
65
- "to discuss this feature." % (
62
+ "but `max_window_layers` = {} is less than "
63
+ "`num_hidden_layers` = {}. Please open an issue "
64
+ "to discuss this feature.".format(
65
config.max_window_layers,
66
config.num_hidden_layers,
67
))
vllm/model_executor/models/qwen2_moe.py
-1
@@ -1,4 +1,3 @@
1
-# coding=utf-8
1
# Adapted from
2
# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/qwen2_moe/modeling_qwen2_moe.py
3
# Copyright 2024 The Qwen team.
vllm/model_executor/models/qwen2_rm.py
+3
-4
@@ -1,4 +1,3 @@
1
-# coding=utf-8
1
# Adapted from
2
# https://huggingface.co/Qwen/Qwen2.5-Math-RM-72B/blob/main/modeling_qwen2_rm.py
3
# Copyright 2024 The Qwen team.
@@ -71,9 +70,9 @@ class Qwen2ForRewardModel(nn.Module, SupportsPP):
70
and hasattr(config, "max_window_layers")):
71
raise ValueError("Sliding window for some but all layers is not "
72
"supported. This model uses sliding window "
74
- "but `max_window_layers` = %s is less than "
75
- "`num_hidden_layers` = %s. Please open an issue "
76
- "to discuss this feature." % (
73
+ "but `max_window_layers` = {} is less than "
74
+ "`num_hidden_layers` = {}. Please open an issue "
75
+ "to discuss this feature.".format(
76
config.max_window_layers,
77
config.num_hidden_layers,
78
))
vllm/model_executor/models/qwen2_vl.py
+4
-6
@@ -1,4 +1,3 @@
1
-# coding=utf-8
1
# Adapted from
2
# https://github.com/huggingface/transformers/blob/19e6e80e10118f855137b90740936c0b11ac397f/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py
3
# Copyright 2024 The Qwen team.
@@ -246,9 +245,8 @@ class Qwen2VisionAttention(nn.Module):
245
q, k, v = dist_utils.split_tensor_along_last_dim(x, 3)
246
batch_size = q.shape[1]
247
249
- q, k, v = [
250
- rearrange(x, "s b ... -> b s ...").contiguous() for x in (q, k, v)
251
- ]
248
+ q, k, v = (rearrange(x, "s b ... -> b s ...").contiguous()
249
+ for x in (q, k, v))
250
if rotary_pos_emb is not None:
251
q = apply_rotary_pos_emb_vision(q, rotary_pos_emb)
252
k = apply_rotary_pos_emb_vision(k, rotary_pos_emb)
@@ -258,7 +256,7 @@ class Qwen2VisionAttention(nn.Module):
256
# flash_attn_varlen_func)
257
from flash_attn import flash_attn_varlen_func
258
261
- q, k, v = [rearrange(x, "b s ... -> (b s) ...") for x in [q, k, v]]
259
+ q, k, v = (rearrange(x, "b s ... -> (b s) ...") for x in [q, k, v])
260
261
max_seqlen = (cu_seqlens[1:] - cu_seqlens[:-1]).max().item()
262
output = flash_attn_varlen_func(q,
@@ -276,7 +274,7 @@ class Qwen2VisionAttention(nn.Module):
274
b=batch_size)
275
elif self.attn_backend == _Backend.TORCH_SDPA:
276
seq_length = q.size(1)
279
- q, k, v = [rearrange(x, "b s h d -> b h s d") for x in [q, k, v]]
277
+ q, k, v = (rearrange(x, "b s h d -> b h s d") for x in [q, k, v])
278
attention_mask = torch.zeros([1, seq_length, seq_length],
279
device=q.device,
280
dtype=torch.bool)
vllm/model_executor/models/solar.py
-1
@@ -1,4 +1,3 @@
1
-# coding=utf-8
1
# Adapted from
2
# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py
3
# Copyright 2023 The vLLM team.
vllm/model_executor/models/stablelm.py
-1
@@ -1,4 +1,3 @@
1
-# coding=utf-8
1
# Copyright 2023 Stability AI, EleutherAI, and The HuggingFace Inc. team.
2
# All rights reserved.
3
#
vllm/model_executor/models/starcoder2.py
-1
@@ -1,4 +1,3 @@
1
-# coding=utf-8
1
# Copyright 2024 BigCode and the HuggingFace Inc. team. All rights reserved.
2
#
3
# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
vllm/model_executor/models/xverse.py
-1
@@ -1,4 +1,3 @@
1
-# coding=utf-8
1
# Adapted from
2
# https://huggingface.co/xverse/XVERSE-7B/blob/main/modeling_xverse.py
3
# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
vllm/multimodal/base.py
+39
-27
@@ -1,4 +1,3 @@
1
-import sys
1
from abc import ABC, abstractmethod
2
from collections import UserDict, defaultdict
3
from typing import (TYPE_CHECKING, Any, Callable, Dict, List, Mapping,
@@ -34,14 +33,9 @@ A dictionary containing nested tensors which have been batched via
33
:meth:`MultiModalInputs.batch`.
34
"""
35
37
-if sys.version_info < (3, 9):
38
- # UserDict cannot be subscripted
39
- class _MultiModalInputsBase(UserDict):
40
- pass
41
-else:
36
43
- class _MultiModalInputsBase(UserDict[str, NestedTensors]):
44
- pass
37
+class _MultiModalInputsBase(UserDict[str, NestedTensors]):
38
+ pass
39
40
41
class MultiModalInputs(_MultiModalInputsBase):
@@ -262,18 +256,23 @@ class MultiModalPlugin(ABC):
256
logger.warning(
257
"Model class %s already has an input mapper "
258
"registered to %s. It is overwritten by the new one.",
265
- model_cls, self)
259
+ model_cls,
260
+ self,
261
+ )
262
267
- self._input_mappers[model_cls] = mapper \
268
- or self._default_input_mapper
263
+ self._input_mappers[model_cls] = (mapper
264
+ or self._default_input_mapper)
265
266
return model_cls
267
268
return wrapper
269
274
- def map_input(self, model_config: "ModelConfig",
275
- data: MultiModalData[object],
276
- mm_processor_kwargs: Dict[str, Any]) -> MultiModalInputs:
270
+ def map_input(
271
+ self,
272
+ model_config: "ModelConfig",
273
+ data: MultiModalData[object],
274
+ mm_processor_kwargs: Dict[str, Any],
275
+ ) -> MultiModalInputs:
276
"""
277
Transform the data into a dictionary of model inputs using the
278
input mapper registered for that model.
@@ -348,13 +347,15 @@ class MultiModalPlugin(ABC):
347
logger.warning(
348
"Model class %s already calculates maximum number of "
349
"tokens in %s. It is overwritten by the new one.",
351
- model_cls, self)
350
+ model_cls,
351
+ self,
352
+ )
353
354
if isinstance(max_mm_tokens, int):
355
self._validate_max_multimodal_tokens(max_mm_tokens)
356
356
- self._max_mm_tokens[model_cls] = max_mm_tokens \
357
- or self._default_max_multimodal_tokens
357
+ self._max_mm_tokens[model_cls] = (
358
+ max_mm_tokens or self._default_max_multimodal_tokens)
359
360
return model_cls
361
@@ -482,8 +483,10 @@ class MultiModalPlaceholderMap:
483
placeholder_maps: Dict[str, MultiModalPlaceholderMap] = defaultdict(
484
MultiModalPlaceholderMap)
485
485
- for modality, placeholders in seq_group.multi_modal_placeholders.items(
486
- ):
486
+ for (
487
+ modality,
488
+ placeholders,
489
+ ) in seq_group.multi_modal_placeholders.items():
490
mm_items = mm_data.pop(modality)
491
if not isinstance(mm_items, list):
492
mm_items = [mm_items]
@@ -499,8 +502,11 @@ class MultiModalPlaceholderMap:
502
return mm_data, placeholder_maps
503
504
def append_items_from_seq_group(
502
- self, positions: range, multi_modal_items: List[_T],
503
- multi_modal_placeholders: List[PlaceholderRange]) -> List[_T]:
505
+ self,
506
+ positions: range,
507
+ multi_modal_items: List[_T],
508
+ multi_modal_placeholders: List[PlaceholderRange],
509
+ ) -> List[_T]:
510
"""
511
Adds the multi-modal items that intersect ```positions`` to this
512
placeholder map and returns the intersecting items.
@@ -515,20 +521,26 @@ class MultiModalPlaceholderMap:
521
multi_modal_items):
522
placeholder = range(
523
placeholder_dict["offset"],
518
- placeholder_dict["offset"] + placeholder_dict["length"])
519
- intersection = range(max(positions.start, placeholder.start),
520
- min(positions.stop, placeholder.stop))
524
+ placeholder_dict["offset"] + placeholder_dict["length"],
525
+ )
526
+ intersection = range(
527
+ max(positions.start, placeholder.start),
528
+ min(positions.stop, placeholder.stop),
529
+ )
530
531
if not intersection:
532
# Skip this multi-modal item.
533
continue
534
526
- token_embedding_range = range(intersection.start - positions.start,
527
- intersection.stop - positions.start)
535
+ token_embedding_range = range(
536
+ intersection.start - positions.start,
537
+ intersection.stop - positions.start,
538
+ )
539
540
multimodal_embedding_range = range(
541
intersection.start - placeholder.start + self.src_len,
531
- intersection.stop - placeholder.start + self.src_len)
542
+ intersection.stop - placeholder.start + self.src_len,
543
+ )
544
545
intersecting_items.append(mm_item)
546
self.dest_ranges.append(token_embedding_range)
vllm/prompt_adapter/utils.py
+8
-9
@@ -37,9 +37,8 @@ def load_peft_weights(model_id: str,
37
Additional arguments to pass to the `hf_hub_download` method when
38
loading from the HuggingFace Hub.
39
"""
40
- path = (os.path.join(model_id, hf_hub_download_kwargs["subfolder"])
41
- if hf_hub_download_kwargs.get("subfolder", None) is not None else
42
- model_id)
40
+ path = (os.path.join(model_id, hf_hub_download_kwargs["subfolder"]) if
41
+ hf_hub_download_kwargs.get("subfolder") is not None else model_id)
42
43
if device is None:
44
device = infer_device()
@@ -51,19 +50,19 @@ def load_peft_weights(model_id: str,
50
filename = os.path.join(path, WEIGHTS_NAME)
51
use_safetensors = False
52
else:
54
- token = hf_hub_download_kwargs.get("token", None)
53
+ token = hf_hub_download_kwargs.get("token")
54
if token is None:
56
- token = hf_hub_download_kwargs.get("use_auth_token", None)
55
+ token = hf_hub_download_kwargs.get("use_auth_token")
56
57
hub_filename = (os.path.join(hf_hub_download_kwargs["subfolder"],
58
SAFETENSORS_WEIGHTS_NAME)
60
- if hf_hub_download_kwargs.get("subfolder", None)
61
- is not None else SAFETENSORS_WEIGHTS_NAME)
59
+ if hf_hub_download_kwargs.get("subfolder") is not None
60
+ else SAFETENSORS_WEIGHTS_NAME)
61
has_remote_safetensors_file = file_exists(
62
repo_id=model_id,
63
filename=hub_filename,
65
- revision=hf_hub_download_kwargs.get("revision", None),
66
- repo_type=hf_hub_download_kwargs.get("repo_type", None),
64
+ revision=hf_hub_download_kwargs.get("revision"),
65
+ repo_type=hf_hub_download_kwargs.get("repo_type"),
66
token=token,
67
)
68
use_safetensors = has_remote_safetensors_file
vllm/transformers_utils/config.py
+1
-1
@@ -308,7 +308,7 @@ def load_params_config(model, revision) -> PretrainedConfig:
308
config_path = Path(
309
hf_hub_download(model, config_file_name, revision=revision))
310
311
- with open(config_path, "r") as file:
311
+ with open(config_path) as file:
312
config_dict = json.load(file)
313
314
config_mapping = {
vllm/transformers_utils/configs/chatglm.py
-1
@@ -1,4 +1,3 @@
1
-# coding=utf-8
1
# Adapted from
2
# https://github.com/THUDM/ChatGLM2-6B
3
from transformers import PretrainedConfig
vllm/transformers_utils/configs/exaone.py
-1
@@ -1,4 +1,3 @@
1
-# coding=utf-8
1
# Copied from
2
# https://huggingface.co/LGAI-EXAONE/EXAONE-3.0-7.8B-Instruct/blob/main/configuration_exaone.py
3
# Copyright 2021 The LG AI Research EXAONE Lab. All rights reserved.
vllm/transformers_utils/configs/jais.py
-1
@@ -1,4 +1,3 @@
1
-# coding=utf-8
1
# Copyright 2023 The OpenAI Team Authors and HuggingFace Inc. team.
2
# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved.
3
# Copyright 2023 Cerebras Systems.
vllm/transformers_utils/configs/mpt.py
+3
-4
@@ -1,4 +1,3 @@
1
-# coding=utf-8
1
# Copied from
2
# https://huggingface.co/mosaicml/mpt-7b/blob/main/configuration_mpt.py
3
"""A HuggingFace-style model configuration."""
@@ -117,10 +116,10 @@ class MPTConfig(PretrainedConfig):
116
init_config_defaults)
117
if self.d_model % self.n_heads != 0:
118
raise ValueError('d_model must be divisible by n_heads')
120
- if any((
119
+ if any(
120
prob < 0 or prob > 1 for prob in
122
- [self.attn_config['attn_pdrop'], self.resid_pdrop, self.emb_pdrop]
123
- )):
121
+ [self.attn_config['attn_pdrop'], self.resid_pdrop, self.emb_pdrop
122
+ ]):
123
raise ValueError(
124
"self.attn_config['attn_pdrop'], resid_pdrop, emb_pdrop are "
125
"probabilities and must be between 0 and 1")
vllm/transformers_utils/configs/nemotron.py
+3
-4
@@ -1,4 +1,3 @@
1
-# coding=utf-8
1
# Copyright 2024 HuggingFace Inc. team. All rights reserved.
2
# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
3
#
@@ -144,7 +143,7 @@ class NemotronConfig(PretrainedConfig):
143
self.intermediate_size = intermediate_size
144
self.num_hidden_layers = num_hidden_layers
145
self.num_attention_heads = num_attention_heads
147
- head_dim = head_dim or kwargs.get("kv_channels", None)
146
+ head_dim = head_dim or kwargs.get("kv_channels")
147
self.head_dim = head_dim if head_dim is not None else (
148
hidden_size // num_attention_heads)
149
@@ -160,8 +159,8 @@ class NemotronConfig(PretrainedConfig):
159
self.rope_theta = rope_theta
160
self.rope_scaling = rope_scaling
161
# for backward compatibility
163
- partial_rotary_factor = kwargs.get("rope_percent", None) or kwargs.get(
164
- "rope_percentage", None) or partial_rotary_factor
162
+ partial_rotary_factor = kwargs.get("rope_percent") or kwargs.get(
163
+ "rope_percentage") or partial_rotary_factor
164
self.partial_rotary_factor = partial_rotary_factor
165
self._rope_scaling_validation()
166
self.attention_bias = attention_bias
vllm/transformers_utils/configs/solar.py
-1
@@ -1,4 +1,3 @@
1
-# coding=utf-8
1
# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
2
#
3
# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
vllm/utils.py
+2
-2
@@ -1153,7 +1153,7 @@ class SortedHelpFormatter(argparse.HelpFormatter):
1153
1154
def add_arguments(self, actions):
1155
actions = sorted(actions, key=lambda x: x.option_strings)
1156
- super(SortedHelpFormatter, self).add_arguments(actions)
1156
+ super().add_arguments(actions)
1157
1158
1159
class FlexibleArgumentParser(argparse.ArgumentParser):
@@ -1279,7 +1279,7 @@ class FlexibleArgumentParser(argparse.ArgumentParser):
1279
1280
config: Dict[str, Union[int, str]] = {}
1281
try:
1282
- with open(file_path, 'r') as config_file:
1282
+ with open(file_path) as config_file:
1283
config = yaml.safe_load(config_file)
1284
except Exception as ex:
1285
logger.error(