@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 21063c11c7
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 08:15 UTC

[CI/Build] drop support for Python 3.8 EOL (#8464)

Signed-off-by: Aaron Pham <contact@aarnphm.xyz>

Aaron Pham committed Nov 6, 2024 at 02:11 UTC 21063c11c7d340dbb01460e22d98d3619737cd4d
115 files changed +240 -322
.buildkite/nightly-benchmarks/scripts/convert-results-json-to-markdown.py
+5 -5
@@ -56,7 +56,7 @@ serving_column_mapping = {
56
57 def read_markdown(file):
58 if os.path.exists(file):
59 - with open(file, "r") as f:
59 + with open(file) as f:
60 return f.read() + "\n"
61 else:
62 return f"{file} not found.\n"
@@ -75,14 +75,14 @@ if __name__ == "__main__":
75 # collect results
76 for test_file in results_folder.glob("*.json"):
77
78 - with open(test_file, "r") as f:
78 + with open(test_file) as f:
79 raw_result = json.loads(f.read())
80
81 if "serving" in str(test_file):
82 # this result is generated via `benchmark_serving.py`
83
84 # attach the benchmarking command to raw_result
85 - with open(test_file.with_suffix(".commands"), "r") as f:
85 + with open(test_file.with_suffix(".commands")) as f:
86 command = json.loads(f.read())
87 raw_result.update(command)
88
@@ -97,7 +97,7 @@ if __name__ == "__main__":
97 # this result is generated via `benchmark_latency.py`
98
99 # attach the benchmarking command to raw_result
100 - with open(test_file.with_suffix(".commands"), "r") as f:
100 + with open(test_file.with_suffix(".commands")) as f:
101 command = json.loads(f.read())
102 raw_result.update(command)
103
@@ -119,7 +119,7 @@ if __name__ == "__main__":
119 # this result is generated via `benchmark_throughput.py`
120
121 # attach the benchmarking command to raw_result
122 - with open(test_file.with_suffix(".commands"), "r") as f:
122 + with open(test_file.with_suffix(".commands")) as f:
123 command = json.loads(f.read())
124 raw_result.update(command)
125
.buildkite/nightly-benchmarks/scripts/generate-nightly-markdown.py
+2 -2
@@ -72,7 +72,7 @@ def main(args):
72
73 # collect results
74 for test_file in results_folder.glob("*_nightly_results.json"):
75 - with open(test_file, "r") as f:
75 + with open(test_file) as f:
76 results = results + json.loads(f.read())
77
78 # generate markdown table
@@ -80,7 +80,7 @@ def main(args):
80
81 md_table = tabulate(df, headers='keys', tablefmt='pipe', showindex=False)
82
83 - with open(args.description, "r") as f:
83 + with open(args.description) as f:
84 description = f.read()
85
86 description = description.format(
.buildkite/nightly-benchmarks/scripts/summary-nightly-results.py
+2 -2
@@ -36,11 +36,11 @@ if __name__ == "__main__":
36 # collect results
37 for test_file in results_folder.glob("*.json"):
38
39 - with open(test_file, "r") as f:
39 + with open(test_file) as f:
40 raw_result = json.loads(f.read())
41
42 # attach the benchmarking command to raw_result
43 - with open(test_file.with_suffix(".commands"), "r") as f:
43 + with open(test_file.with_suffix(".commands")) as f:
44 command = json.loads(f.read())
45 raw_result.update(command)
46
.github/workflows/mypy.yaml
+1 -1
@@ -25,7 +25,7 @@ jobs:
25 runs-on: ubuntu-latest
26 strategy:
27 matrix:
28 - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
28 + python-version: ["3.9", "3.10", "3.11", "3.12"]
29 steps:
30 - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
31 - name: Set up Python ${{ matrix.python-version }}
.github/workflows/publish.yml
+1 -1
@@ -48,7 +48,7 @@ jobs:
48 fail-fast: false
49 matrix:
50 os: ['ubuntu-20.04']
51 - python-version: ['3.8', '3.9', '3.10', '3.11', '3.12']
51 + python-version: ['3.9', '3.10', '3.11', '3.12']
52 pytorch-version: ['2.4.0'] # Must be the most recent version that meets requirements-cuda.txt.
53 cuda-version: ['11.8', '12.1']
54
.github/workflows/ruff.yml
+16 -16
@@ -29,19 +29,19 @@ jobs:
29 matrix:
30 python-version: ["3.12"]
31 steps:
32 - - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
33 - - name: Set up Python ${{ matrix.python-version }}
34 - uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0
35 - with:
36 - python-version: ${{ matrix.python-version }}
37 - - name: Install dependencies
38 - run: |
39 - python -m pip install --upgrade pip
40 - pip install -r requirements-lint.txt
41 - - name: Analysing the code with ruff
42 - run: |
43 - echo "::add-matcher::.github/workflows/matchers/ruff.json"
44 - ruff check --output-format github .
45 - - name: Run isort
46 - run: |
47 - isort . --check-only
32 + - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
33 + - name: Set up Python ${{ matrix.python-version }}
34 + uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0
35 + with:
36 + python-version: ${{ matrix.python-version }}
37 + - name: Install dependencies
38 + run: |
39 + python -m pip install --upgrade pip
40 + pip install -r requirements-lint.txt
41 + - name: Analysing the code with ruff
42 + run: |
43 + echo "::add-matcher::.github/workflows/matchers/ruff.json"
44 + ruff check --output-format github .
45 + - name: Run isort
46 + run: |
47 + isort . --check-only
.github/workflows/yapf.yml
+13 -13
@@ -23,16 +23,16 @@ jobs:
23 matrix:
24 python-version: ["3.12"]
25 steps:
26 - - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
27 - - name: Set up Python ${{ matrix.python-version }}
28 - uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0
29 - with:
30 - python-version: ${{ matrix.python-version }}
31 - - name: Install dependencies
32 - run: |
33 - python -m pip install --upgrade pip
34 - pip install yapf==0.32.0
35 - pip install toml==0.10.2
36 - - name: Running yapf
37 - run: |
38 - yapf --diff --recursive .
26 + - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
27 + - name: Set up Python ${{ matrix.python-version }}
28 + uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0
29 + with:
30 + python-version: ${{ matrix.python-version }}
31 + - name: Install dependencies
32 + run: |
33 + python -m pip install --upgrade pip
34 + pip install yapf==0.32.0
35 + pip install toml==0.10.2
36 + - name: Running yapf
37 + run: |
38 + yapf --diff --recursive .
.readthedocs.yaml
+5 -6
@@ -6,17 +6,16 @@ version: 2
6 build:
7 os: ubuntu-22.04
8 tools:
9 - python: "3.8"
9 + python: '3.9'
10
11 sphinx:
12 - configuration: docs/source/conf.py
13 - fail_on_warning: true
12 + configuration: docs/source/conf.py
13 + fail_on_warning: true
14
15 # If using Sphinx, optionally build your docs in additional formats such as PDF
16 formats: []
17
18 # Optionally declare the Python requirements required to build your docs
19 python:
20 - install:
21 - - requirements: docs/requirements-docs.txt
22 -
20 + install:
21 + - requirements: docs/requirements-docs.txt
CMakeLists.txt
+18 -18
@@ -128,9 +128,9 @@ endif()
128
129 if(VLLM_GPU_LANG STREQUAL "CUDA")
130 #
131 - # For cuda we want to be able to control which architectures we compile for on
131 + # For cuda we want to be able to control which architectures we compile for on
132 # a per-file basis in order to cut down on compile time. So here we extract
133 - # the set of architectures we want to compile for and remove the from the
133 + # the set of architectures we want to compile for and remove the from the
134 # CMAKE_CUDA_FLAGS so that they are not applied globally.
135 #
136 clear_cuda_arches(CUDA_ARCH_FLAGS)
@@ -138,7 +138,7 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
138 message(STATUS "CUDA target architectures: ${CUDA_ARCHS}")
139 # Filter the target architectures by the supported supported archs
140 # since for some files we will build for all CUDA_ARCHS.
141 - cuda_archs_loose_intersection(CUDA_ARCHS
141 + cuda_archs_loose_intersection(CUDA_ARCHS
142 "${CUDA_SUPPORTED_ARCHS}" "${CUDA_ARCHS}")
143 message(STATUS "CUDA supported target architectures: ${CUDA_ARCHS}")
144 else()
@@ -236,7 +236,7 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
236 # are not supported by Machete yet.
237 cuda_archs_loose_intersection(MARLIN_ARCHS "8.0;8.6;8.9;9.0" ${CUDA_ARCHS})
238 if (MARLIN_ARCHS)
239 - set(MARLIN_SRCS
239 + set(MARLIN_SRCS
240 "csrc/quantization/fp8/fp8_marlin.cu"
241 "csrc/quantization/marlin/dense/marlin_cuda_kernel.cu"
242 "csrc/quantization/marlin/sparse/marlin_24_cuda_kernel.cu"
@@ -277,7 +277,7 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
277 "in CUDA target architectures")
278 endif()
279
280 - # clear SCALED_MM_3X_ARCHS so the scaled_mm_c2x kernels know we didn't
280 + # clear SCALED_MM_3X_ARCHS so the scaled_mm_c2x kernels know we didn't
281 # build any 3x kernels
282 set(SCALED_MM_3X_ARCHS)
283 endif()
@@ -285,7 +285,7 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
285 #
286 # For the cutlass_scaled_mm kernels we want to build the c2x (CUTLASS 2.x)
287 # kernels for the remaining archs that are not already built for 3x.
288 - cuda_archs_loose_intersection(SCALED_MM_2X_ARCHS
288 + cuda_archs_loose_intersection(SCALED_MM_2X_ARCHS
289 "7.5;8.0;8.6;8.9;9.0" "${CUDA_ARCHS}")
290 # subtract out the archs that are already built for 3x
291 list(REMOVE_ITEM SCALED_MM_2X_ARCHS ${SCALED_MM_3X_ARCHS})
@@ -316,10 +316,10 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
316 cuda_archs_loose_intersection(MACHETE_ARCHS "9.0a" "${CUDA_ARCHS}")
317 if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER 12.0 AND MACHETE_ARCHS)
318 #
319 - # For the Machete kernels we automatically generate sources for various
319 + # For the Machete kernels we automatically generate sources for various
320 # preselected input type pairs and schedules.
321 # Generate sources:
322 - set(MACHETE_GEN_SCRIPT
322 + set(MACHETE_GEN_SCRIPT
323 ${CMAKE_CURRENT_SOURCE_DIR}/csrc/quantization/machete/generate.py)
324 file(MD5 ${MACHETE_GEN_SCRIPT} MACHETE_GEN_SCRIPT_HASH)
325
@@ -329,8 +329,8 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
329 if (NOT DEFINED CACHE{MACHETE_GEN_SCRIPT_HASH}
330 OR NOT $CACHE{MACHETE_GEN_SCRIPT_HASH} STREQUAL ${MACHETE_GEN_SCRIPT_HASH})
331 execute_process(
332 - COMMAND ${CMAKE_COMMAND} -E env
333 - PYTHONPATH=${CMAKE_CURRENT_SOURCE_DIR}/csrc/cutlass_extensions/:${CUTLASS_DIR}/python/:${VLLM_PYTHON_PATH}:$PYTHONPATH
332 + COMMAND ${CMAKE_COMMAND} -E env
333 + PYTHONPATH=${CMAKE_CURRENT_SOURCE_DIR}/csrc/cutlass_extensions/:${CUTLASS_DIR}/python/:${VLLM_PYTHON_PATH}:$PYTHONPATH
334 ${Python_EXECUTABLE} ${MACHETE_GEN_SCRIPT}
335 RESULT_VARIABLE machete_generation_result
336 OUTPUT_VARIABLE machete_generation_output
@@ -340,11 +340,11 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
340
341 if (NOT machete_generation_result EQUAL 0)
342 message(FATAL_ERROR "Machete generation failed."
343 - " Result: \"${machete_generation_result}\""
343 + " Result: \"${machete_generation_result}\""
344 "\nCheck the log for details: "
345 "${CMAKE_CURRENT_BINARY_DIR}/machete_generation.log")
346 else()
347 - set(MACHETE_GEN_SCRIPT_HASH ${MACHETE_GEN_SCRIPT_HASH}
347 + set(MACHETE_GEN_SCRIPT_HASH ${MACHETE_GEN_SCRIPT_HASH}
348 CACHE STRING "Last run machete generate script hash" FORCE)
349 message(STATUS "Machete generation completed successfully.")
350 endif()
@@ -366,7 +366,7 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
366
367 message(STATUS "Building Machete kernels for archs: ${MACHETE_ARCHS}")
368 else()
369 - if (NOT ${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER 12.0
369 + if (NOT ${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER 12.0
370 AND MACHETE_ARCHS)
371 message(STATUS "Not building Machete kernels as CUDA Compiler version is "
372 "not >= 12.0, we recommend upgrading to CUDA 12.0 or "
@@ -392,8 +392,8 @@ define_gpu_extension_target(
392 USE_SABI 3
393 WITH_SOABI)
394
395 -# If CUTLASS is compiled on NVCC >= 12.5, it by default uses
396 -# cudaGetDriverEntryPointByVersion as a wrapper to avoid directly calling the
395 +# If CUTLASS is compiled on NVCC >= 12.5, it by default uses
396 +# cudaGetDriverEntryPointByVersion as a wrapper to avoid directly calling the
397 # driver API. This causes problems when linking with earlier versions of CUDA.
398 # Setting this variable sidesteps the issue by calling the driver directly.
399 target_compile_definitions(_C PRIVATE CUTLASS_ENABLE_DIRECT_CUDA_DRIVER_CALL=1)
@@ -471,9 +471,9 @@ if (NOT VLLM_TARGET_DEVICE STREQUAL "cuda")
471 return()
472 endif ()
473
474 -# vLLM flash attention requires VLLM_GPU_ARCHES to contain the set of target
475 -# arches in the CMake syntax (75-real, 89-virtual, etc), since we clear the
476 -# arches in the CUDA case (and instead set the gencodes on a per file basis)
474 +# vLLM flash attention requires VLLM_GPU_ARCHES to contain the set of target
475 +# arches in the CMake syntax (75-real, 89-virtual, etc), since we clear the
476 +# arches in the CUDA case (and instead set the gencodes on a per file basis)
477 # we need to manually set VLLM_GPU_ARCHES here.
478 if(VLLM_GPU_LANG STREQUAL "CUDA")
479 foreach(_ARCH ${CUDA_ARCHS})
benchmarks/backend_request_func.py
+7 -15
@@ -79,7 +79,7 @@ async def async_request_tgi(
79 # any data, we should skip it.
80 if chunk_bytes.startswith(":"):
81 continue
82 - chunk = remove_prefix(chunk_bytes, "data:")
82 + chunk = chunk_bytes.removeprefix("data:")
83
84 data = json.loads(chunk)
85 timestamp = time.perf_counter()
@@ -144,8 +144,8 @@ async def async_request_trt_llm(
144 if not chunk_bytes:
145 continue
146
147 - chunk = remove_prefix(chunk_bytes.decode("utf-8"),
148 - "data:")
147 + chunk = chunk_bytes.decode("utf-8").removeprefix(
148 + "data:")
149
150 data = json.loads(chunk)
151 output.generated_text += data["text_output"]
@@ -261,8 +261,8 @@ async def async_request_openai_completions(
261 if not chunk_bytes:
262 continue
263
264 - chunk = remove_prefix(chunk_bytes.decode("utf-8"),
265 - "data: ")
264 + chunk = chunk_bytes.decode("utf-8").removeprefix(
265 + "data: ")
266 if chunk == "[DONE]":
267 latency = time.perf_counter() - st
268 else:
@@ -349,8 +349,8 @@ async def async_request_openai_chat_completions(
349 if not chunk_bytes:
350 continue
351
352 - chunk = remove_prefix(chunk_bytes.decode("utf-8"),
353 - "data: ")
352 + chunk = chunk_bytes.decode("utf-8").removeprefix(
353 + "data: ")
354 if chunk == "[DONE]":
355 latency = time.perf_counter() - st
356 else:
@@ -389,14 +389,6 @@ async def async_request_openai_chat_completions(
389 return output
390
391
392 -# Since vllm must support Python 3.8, we can't use str.removeprefix(prefix)
393 -# introduced in Python 3.9
394 -def remove_prefix(text: str, prefix: str) -> str:
395 - if text.startswith(prefix):
396 - return text[len(prefix):]
397 - return text
398 -
399 -
392 def get_model(pretrained_model_name_or_path: str) -> str:
393 if os.getenv('VLLM_USE_MODELSCOPE', 'False').lower() == 'true':
394 from modelscope import snapshot_download
benchmarks/kernels/benchmark_machete.py
+3 -3
@@ -269,10 +269,10 @@ def run_square_bench(args):
269
270
271 def run_range_bench(args):
272 - m_start, k_start, n_start = [int(x) for x in args.dim_start.split(",")]
273 - m_end, k_end, n_end = [int(x) for x in args.dim_end.split(",")]
272 + m_start, k_start, n_start = (int(x) for x in args.dim_start.split(","))
273 + m_end, k_end, n_end = (int(x) for x in args.dim_end.split(","))
274 m_increment, k_increment, n_increment = \
275 - [int(x) for x in args.dim_increment.split(",")]
275 + (int(x) for x in args.dim_increment.split(","))
276 Ms = list(range(m_start, m_end + 1, m_increment))
277 Ks = list(range(k_start, k_end + 1, k_increment))
278 Ns = list(range(n_start, n_end + 1, n_increment))
csrc/quantization/machete/generate.py
+4 -4
@@ -468,7 +468,7 @@ def generate():
468 impl_configs = []
469
470 GPTQ_kernel_type_configs = list(
471 - (TypeConfig(
471 + TypeConfig(
472 element_a=element_a,
473 element_b=element_b,
474 element_b_scale=element_a,
@@ -476,7 +476,7 @@ def generate():
476 element_d=element_a,
477 accumulator=DataType.f32,
478 ) for element_b in (VLLMDataType.u4b8, VLLMDataType.u8b128)
479 - for element_a in (DataType.f16, DataType.bf16)))
479 + for element_a in (DataType.f16, DataType.bf16))
480
481 GPTQ_kernel_specializations = [
482 Specialization(with_C=False, with_zeropoints=False, with_scales=True)
@@ -490,7 +490,7 @@ def generate():
490 ]
491
492 AWQ_kernel_type_configs = list(
493 - (TypeConfig(
493 + TypeConfig(
494 element_a=element_a,
495 element_b=element_b,
496 element_b_scale=element_a,
@@ -498,7 +498,7 @@ def generate():
498 element_d=element_a,
499 accumulator=DataType.f32,
500 ) for element_b in (DataType.u4, DataType.u8)
501 - for element_a in (DataType.f16, DataType.bf16)))
501 + for element_a in (DataType.f16, DataType.bf16))
502
503 AWQ_kernel_specializations = [
504 Specialization(with_C=False, with_zeropoints=True, with_scales=True)
docs/source/getting_started/installation.rst
+5 -5
@@ -10,7 +10,7 @@ Requirements
10 ============
11
12 * OS: Linux
13 -* Python: 3.8 - 3.12
13 +* Python: 3.9 -- 3.12
14 * GPU: compute capability 7.0 or higher (e.g., V100, T4, RTX20xx, A100, L4, H100, etc.)
15
16 Install released versions
@@ -148,7 +148,7 @@ If you want to modify C++ or CUDA code, you'll need to build vLLM from source. T
148 .. tip::
149
150 Building from source requires a lot of compilation. If you are building from source repeatedly, it's more efficient to cache the compilation results.
151 - For example, you can install `ccache <https://github.com/ccache/ccache>`_ using ``conda install ccache`` or ``apt install ccache`` .
151 + For example, you can install `ccache <https://github.com/ccache/ccache>`_ using ``conda install ccache`` or ``apt install ccache`` .
152 As long as ``which ccache`` command can find the ``ccache`` binary, it will be used automatically by the build system. After the first build, subsequent builds will be much faster.
153
154
@@ -181,8 +181,8 @@ to be run simultaneously, via the environment variable ``MAX_JOBS``. For example
181 $ export MAX_JOBS=6
182 $ pip install -e .
183
184 -This is especially useful when you are building on less powerful machines. For example, when you use WSL it only `assigns 50% of the total memory by default <https://learn.microsoft.com/en-us/windows/wsl/wsl-config#main-wsl-settings>`_, so using ``export MAX_JOBS=1`` can avoid compiling multiple files simultaneously and running out of memory.
185 -A side effect is a much slower build process.
184 +This is especially useful when you are building on less powerful machines. For example, when you use WSL it only `assigns 50% of the total memory by default <https://learn.microsoft.com/en-us/windows/wsl/wsl-config#main-wsl-settings>`_, so using ``export MAX_JOBS=1`` can avoid compiling multiple files simultaneously and running out of memory.
185 +A side effect is a much slower build process.
186
187 Additionally, if you have trouble building vLLM, we recommend using the NVIDIA PyTorch Docker image.
188
@@ -209,7 +209,7 @@ Here is a sanity check to verify that the CUDA Toolkit is correctly installed:
209 Unsupported OS build
210 --------------------
211
212 -vLLM can fully run only on Linux but for development purposes, you can still build it on other systems (for example, macOS), allowing for imports and a more convenient development environment. The binaries will not be compiled and won't work on non-Linux systems.
212 +vLLM can fully run only on Linux but for development purposes, you can still build it on other systems (for example, macOS), allowing for imports and a more convenient development environment. The binaries will not be compiled and won't work on non-Linux systems.
213
214 Simply disable the ``VLLM_TARGET_DEVICE`` environment variable before installing:
215
pyproject.toml
+2 -2
@@ -34,7 +34,7 @@ select = [
34 # Pyflakes
35 "F",
36 # pyupgrade
37 - # "UP",
37 + "UP",
38 # flake8-bugbear
39 "B",
40 # flake8-simplify
@@ -55,7 +55,7 @@ ignore = [
55 ]
56
57 [tool.mypy]
58 -python_version = "3.8"
58 +python_version = "3.9"
59
60 ignore_missing_imports = true
61 check_untyped_defs = true
setup.py
+4 -5
@@ -1,5 +1,4 @@
1 import importlib.util
2 -import io
2 import logging
3 import os
4 import re
@@ -327,7 +326,7 @@ def get_neuronxcc_version():
326 "__init__.py")
327
328 # Check if the command was executed successfully
330 - with open(version_file, "rt") as fp:
329 + with open(version_file) as fp:
330 content = fp.read()
331
332 # Extract the version using a regular expression
@@ -404,7 +403,8 @@ def read_readme() -> str:
403 """Read the README file if present."""
404 p = get_path("README.md")
405 if os.path.isfile(p):
407 - return io.open(get_path("README.md"), "r", encoding="utf-8").read()
406 + with open(get_path("README.md"), encoding="utf-8") as f:
407 + return f.read()
408 else:
409 return ""
410
@@ -498,7 +498,6 @@ setup(
498 "Documentation": "https://vllm.readthedocs.io/en/latest/",
499 },
500 classifiers=[
501 - "Programming Language :: Python :: 3.8",
501 "Programming Language :: Python :: 3.9",
502 "Programming Language :: Python :: 3.10",
503 "Programming Language :: Python :: 3.11",
@@ -512,7 +511,7 @@ setup(
511 ],
512 packages=find_packages(exclude=("benchmarks", "csrc", "docs", "examples",
513 "tests*")),
515 - python_requires=">=3.8",
514 + python_requires=">=3.9",
515 install_requires=get_requirements(),
516 ext_modules=ext_modules,
517 extras_require={
tests/compile/piecewise/test_toy_llama.py
+2 -2
@@ -429,8 +429,8 @@ def benchmark():
429 # print in tabular format
430 print("batch size\teager mode\tfull cudagraph\tpiecewise cudagraph")
431 for b in cudagraph_sizes:
432 - print((f"{b}\t{eager_time[b]:.3f}\t{full_cudagraph_time[b]:.3f}"
433 - f"\t{piecewise_cudagraph_time[b]:.3f}"))
432 + print(f"{b}\t{eager_time[b]:.3f}\t{full_cudagraph_time[b]:.3f}"
433 + f"\t{piecewise_cudagraph_time[b]:.3f}")
434
435
436 if __name__ == "__main__":
tests/conftest.py
+8 -21
@@ -1,6 +1,5 @@
1 import json
2 import os
3 -import sys
3 import tempfile
4 from collections import UserList
5 from enum import Enum
@@ -52,7 +51,7 @@ PromptVideoInput = _PromptMultiModalInput[np.ndarray]
51
52
53 def _read_prompts(filename: str) -> List[str]:
55 - with open(filename, "r") as f:
54 + with open(filename) as f:
55 prompts = f.readlines()
56 return prompts
57
@@ -62,14 +61,8 @@ class _ImageAssetPrompts(TypedDict):
61 cherry_blossom: str
62
63
65 -if sys.version_info < (3, 9):
66 - # UserList cannot be subscripted
67 - class _ImageAssetsBase(UserList):
68 - pass
69 -else:
70 -
71 - class _ImageAssetsBase(UserList[ImageAsset]):
72 - pass
64 +class _ImageAssetsBase(UserList[ImageAsset]):
65 + pass
66
67
68 class _ImageAssets(_ImageAssetsBase):
@@ -94,14 +87,8 @@ class _VideoAssetPrompts(TypedDict):
87 sample_demo_1: str
88
89
97 -if sys.version_info < (3, 9):
98 - # UserList cannot be subscripted
99 - class _VideoAssetsBase(UserList):
100 - pass
101 -else:
102 -
103 - class _VideoAssetsBase(UserList[VideoAsset]):
104 - pass
90 +class _VideoAssetsBase(UserList[VideoAsset]):
91 + pass
92
93
94 class _VideoAssets(_VideoAssetsBase):
@@ -958,7 +945,7 @@ def dummy_opt_path():
945 "*.msgpack"
946 ])
947 assert os.path.exists(json_path)
961 - with open(json_path, "r") as f:
948 + with open(json_path) as f:
949 config = json.load(f)
950 config["architectures"] = ["MyOPTForCausalLM"]
951 with open(json_path, "w") as f:
@@ -977,7 +964,7 @@ def dummy_llava_path():
964 "*.msgpack"
965 ])
966 assert os.path.exists(json_path)
980 - with open(json_path, "r") as f:
967 + with open(json_path) as f:
968 config = json.load(f)
969 config["architectures"] = ["MyLlava"]
970 with open(json_path, "w") as f:
@@ -996,7 +983,7 @@ def dummy_gemma2_embedding_path():
983 "*.msgpack"
984 ])
985 assert os.path.exists(json_path)
999 - with open(json_path, "r") as f:
986 + with open(json_path) as f:
987 config = json.load(f)
988 config["architectures"] = ["MyGemma2Embedding"]
989 with open(json_path, "w") as f:
tests/core/block/test_prefix_caching_block.py
+5 -7
@@ -99,13 +99,11 @@ class TestPrefixCachingBlock:
99
100 token_ids = [random.randint(0, 50_000) for _ in range(num_tokens)]
101
102 - first_chain, second_chain = [
103 - TestPrefixCachingBlock.create_chain(
104 - block_size=block_size,
105 - token_ids=token_ids,
106 - num_empty_trailing_blocks=num_empty_trailing_blocks)
107 - for _ in range(2)
108 - ]
102 + first_chain, second_chain = (TestPrefixCachingBlock.create_chain(
103 + block_size=block_size,
104 + token_ids=token_ids,
105 + num_empty_trailing_blocks=num_empty_trailing_blocks)
106 + for _ in range(2))
107
108 for first_chain_block, second_chain_block in zip(
109 first_chain, second_chain):
tests/kernels/test_mamba_ssm.py
+1 -1
@@ -510,7 +510,7 @@ def test_selective_scan_varlen(with_padding, is_variable_B, is_variable_C,
510 for var in (u_ref, delta_ref, B_ref, C_ref, z_ref)
511 ]
512 for i in range(len(seqlens[0])):
513 - u_s, delta_s, B_s, C_s, z_s = [v[i].unsqueeze(0) for v in splits]
513 + u_s, delta_s, B_s, C_s, z_s = (v[i].unsqueeze(0) for v in splits)
514 if padded_state_indices[i] == PAD_SLOT_ID:
515 continue
516 out_ref_s, _ = selective_scan_ref(
tests/models/decoder_only/vision_language/mm_processor_kwargs/test_qwen.py
+1 -1
@@ -104,7 +104,7 @@ def test_input_mapper_valid_mm_data(input_mapper_for_qwen,
104 # Sad path tests for the multimodal input processor and mapper, respectively
105 @pytest.mark.parametrize("mm_data", [
106 {
107 - "image": torch.rand((5))
107 + "image": torch.rand(5)
108 },
109 {
110 "image": torch.rand((5, 5, 5, 5, 5))
tests/samplers/test_rejection_sampler.py
+4 -6
@@ -413,12 +413,10 @@ class _CorrectnessTestHelper:
413 def generate_probs_for_test(
414 self, draft_and_target_probs_equal: bool
415 ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
416 - draft_probs, target_probs = [
417 - F.softmax(
418 - torch.rand(self.vocab_size, dtype=torch.float32),
419 - dim=-1,
420 - ) for _ in range(2)
421 - ]
416 + draft_probs, target_probs = (F.softmax(
417 + torch.rand(self.vocab_size, dtype=torch.float32),
418 + dim=-1,
419 + ) for _ in range(2))
420
421 num_reference_probs = 100
422 reference_probs = F.softmax(
tests/test_logger.py
+1 -1
@@ -29,7 +29,7 @@ def test_trace_function_call():
29 cur_dir = os.path.dirname(__file__)
30 enable_trace_function_call(path, cur_dir)
31 f1(1)
32 - with open(path, 'r') as f:
32 + with open(path) as f:
33 content = f.read()
34
35 assert "f1" in content
tests/tokenization/test_detokenize.py
+2 -2
@@ -93,10 +93,10 @@ def test_mistral_edge_case(tokenizer, truth):
93 def skip_special_tokens(request, tokenizer_name) -> Generator[bool, Any, None]:
94 if "mistral" in tokenizer_name:
95 yield (
96 - bool(True) if request.param else
96 + True if request.param else
97 pytest.skip("mistral doesn't support skip_special_tokens=False"))
98 else:
99 - yield bool(True) if request.param else bool(False)
99 + yield bool(request.param)
100
101
102 @pytest.mark.parametrize("truth", TRUTH)
tools/profiler/print_layerwise_table.py
+1 -1
@@ -46,7 +46,7 @@ if __name__ == "__main__":
46
47 args = parser.parse_args()
48
49 - with open(args.json_trace, "r") as f:
49 + with open(args.json_trace) as f:
50 profile_data = json.load(f)
51
52 if args.table == "summary":
tools/profiler/visualize_layerwise_profile.py
+1 -1
@@ -434,7 +434,7 @@ def main(
434 f"{', Sparsity ' + sparsity if sparsity else ''}")
435
436 profile_json = None
437 - with open(json_trace, "r") as f:
437 + with open(json_trace) as f:
438 profile_json = json.load(f)
439 assert profile_json is not None
440
tools/report_build_time_ninja.py
+17 -15
@@ -81,7 +81,7 @@ class Target:
81 # Allow for modest floating-point errors
82 epsilon = 0.000002
83 if (self.weighted_duration > self.Duration() + epsilon):
84 - print('%s > %s?' % (self.weighted_duration, self.Duration()))
84 + print('{} > {}?'.format(self.weighted_duration, self.Duration()))
85 assert (self.weighted_duration <= self.Duration() + epsilon)
86 return self.weighted_duration
87
@@ -104,7 +104,7 @@ def ReadTargets(log, show_all):
104 The result is a list of Target objects."""
105 header = log.readline()
106 assert header == '# ninja log v5\n', \
107 - 'unrecognized ninja log version %r' % header
107 + 'unrecognized ninja log version {!r}'.format(header)
108 targets_dict = {}
109 last_end_seen = 0.0
110 for line in log:
@@ -254,8 +254,8 @@ def SummarizeEntries(entries, extra_step_types):
254 # Warn if the sum of weighted times is off by more than half a second.
255 if abs(length - weighted_total) > 500:
256 print('Warning: Possible corrupt ninja log, results may be '
257 - 'untrustworthy. Length = %.3f, weighted total = %.3f' %
258 - (length, weighted_total))
257 + 'untrustworthy. Length = {:.3f}, weighted total = {:.3f}'.format(
258 + length, weighted_total))
259
260 entries_by_ext = defaultdict(list)
261 for target in entries:
@@ -263,16 +263,17 @@ def SummarizeEntries(entries, extra_step_types):
263 entries_by_ext[extension].append(target)
264
265 for key, values in entries_by_ext.items():
266 - print(' Longest build steps for %s:' % key)
266 + print(' Longest build steps for {}:'.format(key))
267 values.sort(key=lambda x: x.WeightedDuration())
268 for target in values[-long_count:]:
269 - print(' %8.1f weighted s to build %s (%.1f s elapsed time)' %
270 - (target.WeightedDuration(), target.DescribeTargets(),
271 - target.Duration()))
272 -
273 - print(' %.1f s weighted time (%.1f s elapsed time sum, %1.1fx '
274 - 'parallelism)' %
275 - (length, total_cpu_time, total_cpu_time * 1.0 / length))
269 + print(
270 + ' {:8.1f} weighted s to build {} ({:.1f} s elapsed time)'.
271 + format(target.WeightedDuration(), target.DescribeTargets(),
272 + target.Duration()))
273 +
274 + print(' {:.1f} s weighted time ({:.1f} s elapsed time sum, {:1.1f}x '
275 + 'parallelism)'.format(length, total_cpu_time,
276 + total_cpu_time * 1.0 / length))
277 print(' %d build steps completed, average of %1.2f/s' %
278 (len(entries), len(entries) / (length)))
279
@@ -298,11 +299,12 @@ def main():
299 long_ext_count += len(args.step_types.split(';'))
300
301 try:
301 - with open(log_file, 'r') as log:
302 + with open(log_file) as log:
303 entries = ReadTargets(log, False)
304 SummarizeEntries(entries, args.step_types)
304 - except IOError:
305 - print('Log file %r not found, no build summary created.' % log_file)
305 + except OSError:
306 + print('Log file {!r} not found, no build summary created.'.format(
307 + log_file))
308 return errno.ENOENT
309
310
use_existing_torch.py
+1 -1
@@ -4,7 +4,7 @@ requires_files = glob.glob('requirements*.txt')
4 requires_files += ["pyproject.toml"]
5 for file in requires_files:
6 print(f">>> cleaning {file}")
7 - with open(file, 'r') as f:
7 + with open(file) as f:
8 lines = f.readlines()
9 if "torch" in "".join(lines).lower():
10 print("removed:")
vllm/attention/ops/blocksparse_attention/interface.py
+2 -4
@@ -192,10 +192,8 @@ class LocalStridedBlockSparseAttn(torch.nn.Module):
192 attn_mask = self.dense_attn_mask[None, :, :maxlen, :maxlen]
193
194 q2 = self.transpose_and_pad(q, cu_seqlens, maxlen, 1)
195 - k2, v2 = [
196 - self.transpose_and_pad(x, cu_seqlens, maxlen, q_k_ratio)
197 - for x in [k, v]
198 - ]
195 + k2, v2 = (self.transpose_and_pad(x, cu_seqlens, maxlen, q_k_ratio)
196 + for x in [k, v])
197 spda_output = torch.nn.functional.scaled_dot_product_attention(
198 q2, k2, v2, attn_mask=attn_mask, scale=sm_scale)
199 return self.transpose_and_unpad(spda_output, cu_seqlens)
vllm/config.py
+4 -3
@@ -668,9 +668,10 @@ class ModelConfig:
668 @property
669 def is_encoder_decoder_model(self) -> bool:
670 """Extract the HF encoder/decoder model flag."""
671 - return getattr(self.hf_config, "is_encoder_decoder", False) or (
672 - (hasattr(self.hf_config, "text_config") and getattr(
673 - self.hf_config.text_config, "is_encoder_decoder", False)))
671 + return getattr(
672 + self.hf_config, "is_encoder_decoder",
673 + False) or (hasattr(self.hf_config, "text_config") and getattr(
674 + self.hf_config.text_config, "is_encoder_decoder", False))
675
676 @property
677 def is_multimodal_model(self) -> bool:
vllm/core/evictor.py
+1 -1
@@ -52,7 +52,7 @@ class Evictor(ABC):
52 pass
53
54
55 -class BlockMetaData():
55 +class BlockMetaData:
56 """Data structure for storing key data describe cached block, so that
57 evitor could use to make its decision which one to choose for eviction
58
vllm/distributed/device_communicators/custom_all_reduce_utils.py
+1 -1
@@ -240,7 +240,7 @@ def gpu_p2p_access_check(src: int, tgt: int) -> bool:
240 if is_distributed:
241 get_world_group().barrier()
242 logger.info("reading GPU P2P access cache from %s", path)
243 - with open(path, "r") as f:
243 + with open(path) as f:
244 cache = json.load(f)
245 _gpu_p2p_access_cache = cache
246 return _gpu_p2p_access_cache[f"{src}->{tgt}"]
vllm/engine/async_llm_engine.py
+1 -1
@@ -812,7 +812,7 @@ class AsyncLLMEngine(EngineClient):
812 async def run_engine_loop(engine_ref: ReferenceType):
813 """We use a weakref to the engine so that the running loop
814 doesn't prevent the engine being garbage collected."""
815 - engine: Optional["AsyncLLMEngine"] = engine_ref()
815 + engine: Optional[AsyncLLMEngine] = engine_ref()
816 if not engine:
817 return
818
vllm/engine/llm_engine.py
+2 -2
@@ -1541,8 +1541,8 @@ class LLMEngine:
1541 seq_group.state.remaining_steps != ref_remaining_steps
1542 for seq_group in seq_group_metadata_list[1:]
1543 ]):
1544 - raise AssertionError(("All running sequence groups should "
1545 - "have the same remaining steps."))
1544 + raise AssertionError("All running sequence groups should "
1545 + "have the same remaining steps.")
1546
1547 return ref_remaining_steps > 0
1548
vllm/engine/metrics_types.py
+1 -1
@@ -77,7 +77,7 @@ class StatLoggerBase(ABC):
77 self.num_generation_tokens: List[int] = []
78 self.last_local_log = time.time()
79 self.local_interval = local_interval
80 - self.spec_decode_metrics: Optional["SpecDecodeWorkerMetrics"] = None
80 + self.spec_decode_metrics: Optional[SpecDecodeWorkerMetrics] = None
81
82 @abstractmethod
83 def log(self, stats: Stats) -> None:
vllm/engine/output_processor/multi_step.py
+1 -1
@@ -63,7 +63,7 @@ class MultiStepOutputProcessor(SequenceGroupOutputProcessor):
63 single_step_process_prompt_logprob(self, seq_group, output)
64
65 @staticmethod
66 - @functools.lru_cache()
66 + @functools.lru_cache
67 def _log_prompt_logprob_unsupported_warning_once():
68 # Reminder: Please update docs/source/serving/compatibility_matrix.rst
69 # If the feature combo become valid
vllm/entrypoints/chat_utils.py
+1 -1
@@ -362,7 +362,7 @@ def load_chat_template(
362 if chat_template is None:
363 return None
364 try:
365 - with open(chat_template, "r") as f:
365 + with open(chat_template) as f:
366 resolved_chat_template = f.read()
367 except OSError as e:
368 if isinstance(chat_template, Path):
vllm/entrypoints/openai/run_batch.py
+1 -1
@@ -120,7 +120,7 @@ async def read_file(path_or_url: str) -> str:
120 session.get(path_or_url) as resp:
121 return await resp.text()
122 else:
123 - with open(path_or_url, "r", encoding="utf-8") as f:
123 + with open(path_or_url, encoding="utf-8") as f:
124 return f.read()
125
126
vllm/executor/ray_gpu_executor.py
+1 -1
@@ -32,7 +32,7 @@ class RayGPUExecutor(DistributedGPUExecutor):
32 uses_ray: bool = True
33
34 def _init_executor(self) -> None:
35 - self.forward_dag: Optional["ray.dag.CompiledDAG"] = None
35 + self.forward_dag: Optional[ray.dag.CompiledDAG] = None
36 # If the env var is set, it uses the Ray's compiled DAG API
37 # which optimizes the control plane overhead.
38 # Run vLLM with VLLM_USE_RAY_COMPILED_DAG=1 to enable it.
vllm/logger.py
+1 -2
@@ -67,8 +67,7 @@ def _configure_vllm_root_logger() -> None:
67 raise RuntimeError(
68 "Could not load logging config. File does not exist: %s",
69 VLLM_LOGGING_CONFIG_PATH)
70 - with open(VLLM_LOGGING_CONFIG_PATH, encoding="utf-8",
71 - mode="r") as file:
70 + with open(VLLM_LOGGING_CONFIG_PATH, encoding="utf-8") as file:
71 custom_config = json.loads(file.read())
72
73 if not isinstance(custom_config, dict):
vllm/lora/models.py
+2 -2
@@ -343,7 +343,7 @@ class LoRAModelManager(AdapterModelManager):
343 # text modules (e.g. ChatGLM)
344 and hasattr(self.model, "get_mm_mapping"))
345 self.packed_modules: Dict[str, List[str]] = {}
346 - self.modules: Dict[str, "BaseLayerWithLoRA"] = {}
346 + self.modules: Dict[str, BaseLayerWithLoRA] = {}
347 # Dict instead of a Set for compatibility with LRUCache.
348 self._last_mapping: Optional[LoRAMapping] = None
349 self._create_lora_modules()
@@ -548,7 +548,7 @@ class LoRAModelManager(AdapterModelManager):
548 else:
549 parts = module_name.split(".")
550 replacements = self.packed_modules_mapping[parts[-1]]
551 - subloras: List[Optional["LoRALayerWeights"]] = []
551 + subloras: List[Optional[LoRALayerWeights]] = []
552 for i, r in enumerate(replacements):
553 lora = LoRALayerWeights.create_dummy_lora_weights(
554 module_name + "." + r,
vllm/model_executor/custom_op.py
+1 -1
@@ -103,7 +103,7 @@ class CustomOp(nn.Module):
103 # On by default if VLLM_TORCH_COMPILE_LEVEL < CompilationLevel.PIECEWISE
104 # Specifying 'all' or 'none' in VLLM_CUSTOM_OPS takes precedence.
105 @staticmethod
106 - @lru_cache()
106 + @lru_cache
107 def default_on() -> bool:
108 count_none = envs.VLLM_CUSTOM_OPS.count("none")
109 count_all = envs.VLLM_CUSTOM_OPS.count("all")
vllm/model_executor/layers/resampler.py
-1
@@ -1,4 +1,3 @@
1 -# coding=utf-8
1 # Adapted from
2 # https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py
3 # https://huggingface.co/Qwen/Qwen-7B/blob/main/modeling_qwen.py
vllm/model_executor/layers/rotary_embedding.py
-1
@@ -1,4 +1,3 @@
1 -# coding=utf-8
1 # Adapted from
2 # https://github.com/huggingface/transformers/blob/v4.33.2/src/transformers/models/llama/modeling_llama.py
3 # Copyright 2023 The vLLM team.
vllm/model_executor/model_loader/loader.py
+1 -1
@@ -746,7 +746,7 @@ class BitsAndBytesModelLoader(BaseModelLoader):
746
747 config_file_path = self._get_config_file(qlora_adapter)
748
749 - with open(config_file_path, "r") as f:
749 + with open(config_file_path) as f:
750 config = json.load(f)
751 self.target_modules = config["target_modules"]
752
vllm/model_executor/model_loader/openvino.py
+1 -1
@@ -190,7 +190,7 @@ def get_model(
190 kv_cache_dtype: ov.Type,
191 **kwargs,
192 ) -> torch.nn.Module:
193 - lora_config = kwargs.get("lora_config", None)
193 + lora_config = kwargs.get("lora_config")
194 ov_core = kwargs.get("ov_core")
195 if lora_config:
196 raise ValueError(
vllm/model_executor/model_loader/tensorizer.py
+2 -3
@@ -280,7 +280,7 @@ class TensorizerAgent:
280 self.tensorizer_args = (
281 self.tensorizer_config._construct_tensorizer_args())
282 self.extra_kwargs = extra_kwargs
283 - if extra_kwargs.get("quant_config", None) is not None:
283 + if extra_kwargs.get("quant_config") is not None:
284 self.quant_config = extra_kwargs["quant_config"]
285 else:
286 self.quant_config = quant_config
@@ -380,8 +380,7 @@ def tensorizer_weights_iterator(
380 stream = open_stream(tensorizer_args.tensorizer_uri, **stream_params)
381 with TensorDeserializer(stream, **deserializer_args,
382 device="cpu") as state:
383 - for name, param in state.items():
384 - yield name, param
383 + yield from state.items()
384 del state
385
386
vllm/model_executor/model_loader/weight_utils.py
+4 -5
@@ -188,7 +188,7 @@ def get_quant_config(model_config: ModelConfig,
188 f"{quant_config_files}")
189
190 quant_config_file = quant_config_files[0]
191 - with open(quant_config_file, "r") as f:
191 + with open(quant_config_file) as f:
192 config = json.load(f)
193
194 if model_config.quantization == "bitsandbytes":
@@ -306,7 +306,7 @@ def filter_duplicate_safetensors_files(hf_weights_files: List[str],
306
307 # Iterate through the weight_map (weight_name: safetensors files)
308 # to identify weights that we should use.
309 - with open(index_file_name, "r") as f:
309 + with open(index_file_name) as f:
310 weight_map = json.load(f)["weight_map"]
311 weight_files_in_index = set()
312 for weight_name in weight_map:
@@ -382,7 +382,7 @@ def np_cache_weights_iterator(
382 with open(weight_names_file, "w") as f:
383 json.dump(weight_names, f)
384
385 - with open(weight_names_file, "r") as f:
385 + with open(weight_names_file) as f:
386 weight_names = json.load(f)
387
388 for name in weight_names:
@@ -423,8 +423,7 @@ def pt_weights_iterator(
423 bar_format=_BAR_FORMAT,
424 ):
425 state = torch.load(bin_file, map_location="cpu")
426 - for name, param in state.items():
427 - yield name, param
426 + yield from state.items()
427 del state
428 torch.cuda.empty_cache()
429
vllm/model_executor/models/arctic.py
+2 -2
@@ -48,7 +48,7 @@ class ArcticMLP(nn.Module):
48 is_residual_mlp: bool = False,
49 quant_config: Optional[QuantizationConfig] = None,
50 reduce_results: bool = True):
51 - super(ArcticMLP, self).__init__()
51 + super().__init__()
52 self.hidden_size = config.hidden_size
53 self.expert_id = expert_id
54 self.layer_id = layer_id
@@ -89,7 +89,7 @@ class ArcticMoE(nn.Module):
89 params_dtype: Optional[torch.dtype] = None,
90 quant_config: Optional[QuantizationConfig] = None,
91 reduce_results: bool = True):
92 - super(ArcticMoE, self).__init__()
92 + super().__init__()
93
94 self.tp_size = tp_size or get_tensor_model_parallel_world_size()
95 self.hidden_size = config.hidden_size
vllm/model_executor/models/baichuan.py
-1
@@ -1,4 +1,3 @@
1 -# coding=utf-8
1 # Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
2 #
3 # This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
vllm/model_executor/models/bloom.py
-1
@@ -1,4 +1,3 @@
1 -# coding=utf-8
1 # Adapted from
2 # https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/bloom/modeling_bloom.py
3 # Copyright 2023 The vLLM team.
vllm/model_executor/models/chatglm.py
-1
@@ -1,4 +1,3 @@
1 -# coding=utf-8
1 # Adapted from
2 # https://github.com/THUDM/GLM-4
3 """Inference-only ChatGLM model compatible with THUDM weights."""
vllm/model_executor/models/commandr.py
-1
@@ -1,4 +1,3 @@
1 -# coding=utf-8
1 # Copyright 2024 Cohere and the HuggingFace Inc. team. All rights reserved.
2 #
3 # This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
vllm/model_executor/models/dbrx.py
-1
@@ -1,4 +1,3 @@
1 -# coding=utf-8
1 from typing import Iterable, List, Optional, Tuple, Union
2
3 import torch
vllm/model_executor/models/decilm.py
-1
@@ -1,4 +1,3 @@
1 -# coding=utf-8
1 # Adapted from
2 # https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py
3 # Copyright 2023 DeciAI Research Team. All rights reserved.
vllm/model_executor/models/deepseek.py
-1
@@ -1,4 +1,3 @@
1 -# coding=utf-8
1 # Adapted from
2 # https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py
3 # Copyright 2023 The vLLM team.
vllm/model_executor/models/deepseek_v2.py
-1
@@ -1,4 +1,3 @@
1 -# coding=utf-8
1 # Adapted from
2 # https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py
3 # Copyright 2023 The vLLM team.
vllm/model_executor/models/exaone.py
-1
@@ -1,4 +1,3 @@
1 -# coding=utf-8
1 # Adapted from
2 # https://huggingface.co/LGAI-EXAONE/EXAONE-3.0-7.8B-Instruct/blob/main/modeling_exaone.py
3 # Copyright 2024 The LG U+ CTO AI Tech Lab.
vllm/model_executor/models/falcon.py
-1
@@ -1,4 +1,3 @@
1 -# coding=utf-8
1 # Adapted from
2 # https://github.com/huggingface/transformers/blob/a5cc30d72ae2dc19af534e4b35c986cc28db1275/src/transformers/models/falcon/modeling_falcon.py
3 # Copyright 2023 The vLLM team.
vllm/model_executor/models/fuyu.py
-1
@@ -1,4 +1,3 @@
1 -# coding=utf-8
1 # adapted from https://github.com/huggingface/transformers/blob/v4.39.3/src/transformers/models/fuyu/modeling_fuyu.py
2 # Copyright 2023 The vLLM team.
3 # Copyright 2023 HuggingFace Inc. team. All rights reserved.
vllm/model_executor/models/gemma.py
-1
@@ -1,4 +1,3 @@
1 -# coding=utf-8
1 # Copyright 2023 The vLLM team.
2 # Copyright (c) Google Inc.
3 #
vllm/model_executor/models/gemma2.py
-1
@@ -1,4 +1,3 @@
1 -# coding=utf-8
1 # Copyright 2024 The vLLM team.
2 # Copyright 2024 Google Inc. HuggingFace Inc. team. All rights reserved.
3 #
vllm/model_executor/models/glm4_vision_encoder.py
-1
@@ -1,4 +1,3 @@
1 -# coding=utf-8
1 # Adapted from
2 # https://github.com/THUDM/GLM-4
3 """Inference-only GLM-4v model visual encoder compatible with THUDM weights."""
vllm/model_executor/models/gpt2.py
-1
@@ -1,4 +1,3 @@
1 -# coding=utf-8
1 # Adapted from
2 # https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/gpt2/modeling_gpt2.py
3 # Copyright 2023 The vLLM team.
vllm/model_executor/models/gpt_bigcode.py
-1
@@ -1,4 +1,3 @@
1 -# coding=utf-8
1 # Adapted from
2 # https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/gpt2/modeling_gpt2.py
3 # Copyright 2023 The vLLM team.
vllm/model_executor/models/gpt_j.py
-1
@@ -1,4 +1,3 @@
1 -# coding=utf-8
1 # Adapted from
2 # https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/gptj/modeling_gptj.py
3 # Copyright 2023 The vLLM team.
vllm/model_executor/models/gpt_neox.py
-1
@@ -1,4 +1,3 @@
1 -# coding=utf-8
1 # Adapted from
2 # https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/gpt_neox/modeling_gpt_neox.py
3 # Copyright 2023 The vLLM team.
vllm/model_executor/models/granite.py
-1
@@ -1,4 +1,3 @@
1 -# coding=utf-8
1 # Adapted from
2 # https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py
3 # Copyright 2023 The vLLM team.
vllm/model_executor/models/granitemoe.py
-1
@@ -1,4 +1,3 @@
1 -# coding=utf-8
1 # Adapted from
2 # https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py
3 # Copyright 2023 The vLLM team.
vllm/model_executor/models/idefics2_vision_model.py
-2
@@ -1,5 +1,3 @@
1 -# coding=utf-8
2 -
1 # adapted from https://github.com/huggingface/transformers/blob/v4.43.2/src/transformers/models/idefics2/modeling_idefics2.py
2 # Copyright 2024 The vLLM team.
3 # Copyright 2024 the HuggingFace Inc. team. All rights reserved.
vllm/model_executor/models/internlm2.py
-1
@@ -1,4 +1,3 @@
1 -# -*- coding: utf-8 -*-
1 from functools import partial
2 from typing import Any, Dict, Iterable, List, Optional, Tuple, Union
3
vllm/model_executor/models/internlm2_ve.py
-1
@@ -1,4 +1,3 @@
1 -# -*- coding: utf-8 -*-
1 from typing import List, Optional, Tuple, Union
2
3 import torch
vllm/model_executor/models/jais.py
-1
@@ -1,4 +1,3 @@
1 -# coding=utf-8
1 # Adapted from
2 # https://huggingface.co/inceptionai/jais-30b-chat-v3/blob/main/modeling_jais.py
3 # Copyright 2023 The vLLM team.
vllm/model_executor/models/jamba.py
-1
@@ -1,4 +1,3 @@
1 -# coding=utf-8
1 """Inference-only Jamba model."""
2 from typing import Iterable, List, Optional, Tuple
3
vllm/model_executor/models/llama.py
-1
@@ -1,4 +1,3 @@
1 -# coding=utf-8
1 # Adapted from
2 # https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py
3 # Copyright 2023 The vLLM team.
vllm/model_executor/models/mamba.py
-1
@@ -1,4 +1,3 @@
1 -# coding=utf-8
1 """PyTorch MAMBA model."""
2 from typing import Iterable, List, Optional, Tuple
3
vllm/model_executor/models/minicpm.py
-1
@@ -1,4 +1,3 @@
1 -# coding=utf-8
1 # Adapted from
2 # https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py
3 # Copyright 2023 The vLLM team.
vllm/model_executor/models/minicpm3.py
-1
@@ -1,4 +1,3 @@
1 -# coding=utf-8
1 # Adapted from
2 # https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py
3 # Copyright 2024 The ModelBest team.
vllm/model_executor/models/minicpmv.py
-1
@@ -1,4 +1,3 @@
1 -# coding=utf-8
1 # Adapted from
2 # https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py
3 # Copyright 2023 The vLLM team.
vllm/model_executor/models/mixtral.py
-1
@@ -1,4 +1,3 @@
1 -# coding=utf-8
1 # Adapted from
2 # https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py
3 # Copyright 2023 The vLLM team.
vllm/model_executor/models/mixtral_quant.py
-1
@@ -1,4 +1,3 @@
1 -# coding=utf-8
1 # Adapted from
2 # https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py
3 # Copyright 2023 The vLLM team.
vllm/model_executor/models/mllama.py
-1
@@ -1,4 +1,3 @@
1 -# coding=utf-8
1 # Copyright 2024 the HuggingFace Inc. team. All rights reserved.
2 #
3 # Licensed under the Apache License, Version 2.0 (the "License");
vllm/model_executor/models/mlp_speculator.py
+1 -1
@@ -37,7 +37,7 @@ class MLPSpeculatorLayerNorm(nn.Module):
37 eps=1e-06,
38 elementwise_scale_and_shift=True,
39 ):
40 - super(MLPSpeculatorLayerNorm, self).__init__()
40 + super().__init__()
41 self.elementwise_scale_and_shift = elementwise_scale_and_shift
42 if self.elementwise_scale_and_shift:
43 self.weight = nn.Parameter(torch.empty(normalized_shape))
vllm/model_executor/models/molmo.py
+3 -3
@@ -1121,9 +1121,9 @@ class MolmoForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
1121 batch_size * num_image * num_patch, -1).contiguous()
1122
1123 image_input_idx = image_input_idx * valid.to(image_input_idx.dtype)
1124 - offset = torch.cat(
1125 - [seq_len.new_zeros(
1126 - (1)), seq_len.cumsum(dim=0)[:-1]], dim=0)[:, None]
1124 + offset = torch.cat([seq_len.new_zeros(1),
1125 + seq_len.cumsum(dim=0)[:-1]],
1126 + dim=0)[:, None]
1127 image_input_idx = image_input_idx + offset.to(image_input_idx.dtype)
1128 image_input_idx = image_input_idx.flatten()[:, None]
1129 mat = image_input_idx == torch.arange(
vllm/model_executor/models/mpt.py
-1
@@ -1,4 +1,3 @@
1 -# coding=utf-8
1 # Adapted from https://huggingface.co/mosaicml/mpt-7b/tree/main
2 import math
3 from typing import Iterable, List, Optional, Tuple, Union
vllm/model_executor/models/nemotron.py
-1
@@ -1,4 +1,3 @@
1 -# coding=utf-8
1 # Adapted from
2 # https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py
3 # Copyright 2023 The vLLM team.
vllm/model_executor/models/olmo.py
-1
@@ -1,4 +1,3 @@
1 -# coding=utf-8
1 # Adapted from
2 # https://github.com/huggingface/transformers/blob/v4.40.1/src/transformers/models/olmo/modeling_olmo.py
3 # Copyright 2024 The vLLM team.
vllm/model_executor/models/opt.py
-1
@@ -1,4 +1,3 @@
1 -# coding=utf-8
1 # Adapted from
2 # https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/opt/modeling_opt.py
3 # Copyright 2023 The vLLM team.
vllm/model_executor/models/orion.py
-1
@@ -1,4 +1,3 @@
1 -# coding=utf-8
1 # Adapted from
2 # https://huggingface.co/OrionStarAI/Orion-14B-Base/blob/main/modeling_orion.py
3 # Copyright (c) OrionStar Inc.
vllm/model_executor/models/persimmon.py
-1
@@ -1,4 +1,3 @@
1 -# coding=utf-8
1 # adapted from https://github.com/huggingface/transformers/blob/v4.39.3/src/transformers/models/persimmon/modeling_persimmon.py
2 # Copyright 2023 The vLLM team.
3 # Copyright 2023 EleutherAI and the HuggingFace Inc. team. All rights reserved.
vllm/model_executor/models/phi.py
-1
@@ -1,4 +1,3 @@
1 -# coding=utf-8
1 # Adapted from
2 # https://huggingface.co/microsoft/phi-1_5/blob/main/modeling_phi.py
3 # Copyright 2023 The vLLM team.
vllm/model_executor/models/phi3.py
-1
@@ -1,4 +1,3 @@
1 -# coding=utf-8
1 # Adapted from llama.py
2 """Inference-only Phi3 model code inherit from Llama.py"""
3
vllm/model_executor/models/phi3v.py
-1
@@ -1,4 +1,3 @@
1 -# coding=utf-8
1 # Copyright 2024 The vLLM team.
2 # Copyright 2024 Microsoft and the HuggingFace Inc. team. All rights reserved.
3 #
vllm/model_executor/models/phimoe.py
-1
@@ -1,4 +1,3 @@
1 -# coding=utf-8
1 # Adapted from
2 # https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py
3 # Copyright 2023 The vLLM team.
vllm/model_executor/models/pixtral.py
+5 -5
@@ -136,11 +136,11 @@ def input_processor_for_pixtral(ctx: InputContext, inputs: DecoderOnlyInputs):
136
137 if image_token_id not in inputs['prompt_token_ids']:
138 raise ValueError(
139 - (f"You've passed {inputs=} without {image_token_id=}"
140 - " Make sure to process your input via mistral_common's"
141 - " tokenizer or pass a chat completion request. For more"
142 - " For more info, see: "
143 - "https://github.com/vllm-project/vllm/issues/8411."))
139 + f"You've passed {inputs=} without {image_token_id=}"
140 + " Make sure to process your input via mistral_common's"
141 + " tokenizer or pass a chat completion request. For more"
142 + " For more info, see: "
143 + "https://github.com/vllm-project/vllm/issues/8411.")
144
145 return inputs
146
vllm/model_executor/models/qwen.py
-1
@@ -1,4 +1,3 @@
1 -# coding=utf-8
1 # Adapted from
2 # https://huggingface.co/Qwen/Qwen-7B/blob/main/modeling_qwen.py
3 # Copyright (c) Alibaba Cloud.
vllm/model_executor/models/qwen2.py
+3 -4
@@ -1,4 +1,3 @@
1 -# coding=utf-8
1 # Adapted from
2 # https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/qwen2/modeling_qwen2.py
3 # Copyright 2024 The Qwen team.
@@ -417,9 +416,9 @@ class Qwen2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
416 and hasattr(config, "max_window_layers")):
417 raise ValueError("Sliding window for some but all layers is not "
418 "supported. This model uses sliding window "
420 - "but `max_window_layers` = %s is less than "
421 - "`num_hidden_layers` = %s. Please open an issue "
422 - "to discuss this feature." % (
419 + "but `max_window_layers` = {} is less than "
420 + "`num_hidden_layers` = {}. Please open an issue "
421 + "to discuss this feature.".format(
422 config.max_window_layers,
423 config.num_hidden_layers,
424 ))
vllm/model_executor/models/qwen2_audio.py
-1
@@ -1,4 +1,3 @@
1 -# coding=utf-8
1 # Copyright 2024 The Qwen team.
2 # Copyright 2023 The vLLM team.
3 # Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
vllm/model_executor/models/qwen2_cls.py
+3 -4
@@ -1,4 +1,3 @@
1 -# coding=utf-8
1 # Adapted from
2 # https://huggingface.co/Qwen/Qwen2.5-Math-RM-72B/blob/main/modeling_qwen2_rm.py
3 # Copyright 2024 Kakao Corp. (Kanana-X Team)
@@ -60,9 +59,9 @@ class Qwen2ForSequenceClassification(nn.Module):
59 and hasattr(config, "max_window_layers")):
60 raise ValueError("Sliding window for some but all layers is not "
61 "supported. This model uses sliding window "
63 - "but `max_window_layers` = %s is less than "
64 - "`num_hidden_layers` = %s. Please open an issue "
65 - "to discuss this feature." % (
62 + "but `max_window_layers` = {} is less than "
63 + "`num_hidden_layers` = {}. Please open an issue "
64 + "to discuss this feature.".format(
65 config.max_window_layers,
66 config.num_hidden_layers,
67 ))
vllm/model_executor/models/qwen2_moe.py
-1
@@ -1,4 +1,3 @@
1 -# coding=utf-8
1 # Adapted from
2 # https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/qwen2_moe/modeling_qwen2_moe.py
3 # Copyright 2024 The Qwen team.
vllm/model_executor/models/qwen2_rm.py
+3 -4
@@ -1,4 +1,3 @@
1 -# coding=utf-8
1 # Adapted from
2 # https://huggingface.co/Qwen/Qwen2.5-Math-RM-72B/blob/main/modeling_qwen2_rm.py
3 # Copyright 2024 The Qwen team.
@@ -71,9 +70,9 @@ class Qwen2ForRewardModel(nn.Module, SupportsPP):
70 and hasattr(config, "max_window_layers")):
71 raise ValueError("Sliding window for some but all layers is not "
72 "supported. This model uses sliding window "
74 - "but `max_window_layers` = %s is less than "
75 - "`num_hidden_layers` = %s. Please open an issue "
76 - "to discuss this feature." % (
73 + "but `max_window_layers` = {} is less than "
74 + "`num_hidden_layers` = {}. Please open an issue "
75 + "to discuss this feature.".format(
76 config.max_window_layers,
77 config.num_hidden_layers,
78 ))
vllm/model_executor/models/qwen2_vl.py
+4 -6
@@ -1,4 +1,3 @@
1 -# coding=utf-8
1 # Adapted from
2 # https://github.com/huggingface/transformers/blob/19e6e80e10118f855137b90740936c0b11ac397f/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py
3 # Copyright 2024 The Qwen team.
@@ -246,9 +245,8 @@ class Qwen2VisionAttention(nn.Module):
245 q, k, v = dist_utils.split_tensor_along_last_dim(x, 3)
246 batch_size = q.shape[1]
247
249 - q, k, v = [
250 - rearrange(x, "s b ... -> b s ...").contiguous() for x in (q, k, v)
251 - ]
248 + q, k, v = (rearrange(x, "s b ... -> b s ...").contiguous()
249 + for x in (q, k, v))
250 if rotary_pos_emb is not None:
251 q = apply_rotary_pos_emb_vision(q, rotary_pos_emb)
252 k = apply_rotary_pos_emb_vision(k, rotary_pos_emb)
@@ -258,7 +256,7 @@ class Qwen2VisionAttention(nn.Module):
256 # flash_attn_varlen_func)
257 from flash_attn import flash_attn_varlen_func
258
261 - q, k, v = [rearrange(x, "b s ... -> (b s) ...") for x in [q, k, v]]
259 + q, k, v = (rearrange(x, "b s ... -> (b s) ...") for x in [q, k, v])
260
261 max_seqlen = (cu_seqlens[1:] - cu_seqlens[:-1]).max().item()
262 output = flash_attn_varlen_func(q,
@@ -276,7 +274,7 @@ class Qwen2VisionAttention(nn.Module):
274 b=batch_size)
275 elif self.attn_backend == _Backend.TORCH_SDPA:
276 seq_length = q.size(1)
279 - q, k, v = [rearrange(x, "b s h d -> b h s d") for x in [q, k, v]]
277 + q, k, v = (rearrange(x, "b s h d -> b h s d") for x in [q, k, v])
278 attention_mask = torch.zeros([1, seq_length, seq_length],
279 device=q.device,
280 dtype=torch.bool)
vllm/model_executor/models/solar.py
-1
@@ -1,4 +1,3 @@
1 -# coding=utf-8
1 # Adapted from
2 # https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py
3 # Copyright 2023 The vLLM team.
vllm/model_executor/models/stablelm.py
-1
@@ -1,4 +1,3 @@
1 -# coding=utf-8
1 # Copyright 2023 Stability AI, EleutherAI, and The HuggingFace Inc. team.
2 # All rights reserved.
3 #
vllm/model_executor/models/starcoder2.py
-1
@@ -1,4 +1,3 @@
1 -# coding=utf-8
1 # Copyright 2024 BigCode and the HuggingFace Inc. team. All rights reserved.
2 #
3 # This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
vllm/model_executor/models/xverse.py
-1
@@ -1,4 +1,3 @@
1 -# coding=utf-8
1 # Adapted from
2 # https://huggingface.co/xverse/XVERSE-7B/blob/main/modeling_xverse.py
3 # Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
vllm/multimodal/base.py
+39 -27
@@ -1,4 +1,3 @@
1 -import sys
1 from abc import ABC, abstractmethod
2 from collections import UserDict, defaultdict
3 from typing import (TYPE_CHECKING, Any, Callable, Dict, List, Mapping,
@@ -34,14 +33,9 @@ A dictionary containing nested tensors which have been batched via
33 :meth:`MultiModalInputs.batch`.
34 """
35
37 -if sys.version_info < (3, 9):
38 - # UserDict cannot be subscripted
39 - class _MultiModalInputsBase(UserDict):
40 - pass
41 -else:
36
43 - class _MultiModalInputsBase(UserDict[str, NestedTensors]):
44 - pass
37 +class _MultiModalInputsBase(UserDict[str, NestedTensors]):
38 + pass
39
40
41 class MultiModalInputs(_MultiModalInputsBase):
@@ -262,18 +256,23 @@ class MultiModalPlugin(ABC):
256 logger.warning(
257 "Model class %s already has an input mapper "
258 "registered to %s. It is overwritten by the new one.",
265 - model_cls, self)
259 + model_cls,
260 + self,
261 + )
262
267 - self._input_mappers[model_cls] = mapper \
268 - or self._default_input_mapper
263 + self._input_mappers[model_cls] = (mapper
264 + or self._default_input_mapper)
265
266 return model_cls
267
268 return wrapper
269
274 - def map_input(self, model_config: "ModelConfig",
275 - data: MultiModalData[object],
276 - mm_processor_kwargs: Dict[str, Any]) -> MultiModalInputs:
270 + def map_input(
271 + self,
272 + model_config: "ModelConfig",
273 + data: MultiModalData[object],
274 + mm_processor_kwargs: Dict[str, Any],
275 + ) -> MultiModalInputs:
276 """
277 Transform the data into a dictionary of model inputs using the
278 input mapper registered for that model.
@@ -348,13 +347,15 @@ class MultiModalPlugin(ABC):
347 logger.warning(
348 "Model class %s already calculates maximum number of "
349 "tokens in %s. It is overwritten by the new one.",
351 - model_cls, self)
350 + model_cls,
351 + self,
352 + )
353
354 if isinstance(max_mm_tokens, int):
355 self._validate_max_multimodal_tokens(max_mm_tokens)
356
356 - self._max_mm_tokens[model_cls] = max_mm_tokens \
357 - or self._default_max_multimodal_tokens
357 + self._max_mm_tokens[model_cls] = (
358 + max_mm_tokens or self._default_max_multimodal_tokens)
359
360 return model_cls
361
@@ -482,8 +483,10 @@ class MultiModalPlaceholderMap:
483 placeholder_maps: Dict[str, MultiModalPlaceholderMap] = defaultdict(
484 MultiModalPlaceholderMap)
485
485 - for modality, placeholders in seq_group.multi_modal_placeholders.items(
486 - ):
486 + for (
487 + modality,
488 + placeholders,
489 + ) in seq_group.multi_modal_placeholders.items():
490 mm_items = mm_data.pop(modality)
491 if not isinstance(mm_items, list):
492 mm_items = [mm_items]
@@ -499,8 +502,11 @@ class MultiModalPlaceholderMap:
502 return mm_data, placeholder_maps
503
504 def append_items_from_seq_group(
502 - self, positions: range, multi_modal_items: List[_T],
503 - multi_modal_placeholders: List[PlaceholderRange]) -> List[_T]:
505 + self,
506 + positions: range,
507 + multi_modal_items: List[_T],
508 + multi_modal_placeholders: List[PlaceholderRange],
509 + ) -> List[_T]:
510 """
511 Adds the multi-modal items that intersect ```positions`` to this
512 placeholder map and returns the intersecting items.
@@ -515,20 +521,26 @@ class MultiModalPlaceholderMap:
521 multi_modal_items):
522 placeholder = range(
523 placeholder_dict["offset"],
518 - placeholder_dict["offset"] + placeholder_dict["length"])
519 - intersection = range(max(positions.start, placeholder.start),
520 - min(positions.stop, placeholder.stop))
524 + placeholder_dict["offset"] + placeholder_dict["length"],
525 + )
526 + intersection = range(
527 + max(positions.start, placeholder.start),
528 + min(positions.stop, placeholder.stop),
529 + )
530
531 if not intersection:
532 # Skip this multi-modal item.
533 continue
534
526 - token_embedding_range = range(intersection.start - positions.start,
527 - intersection.stop - positions.start)
535 + token_embedding_range = range(
536 + intersection.start - positions.start,
537 + intersection.stop - positions.start,
538 + )
539
540 multimodal_embedding_range = range(
541 intersection.start - placeholder.start + self.src_len,
531 - intersection.stop - placeholder.start + self.src_len)
542 + intersection.stop - placeholder.start + self.src_len,
543 + )
544
545 intersecting_items.append(mm_item)
546 self.dest_ranges.append(token_embedding_range)
vllm/prompt_adapter/utils.py
+8 -9
@@ -37,9 +37,8 @@ def load_peft_weights(model_id: str,
37 Additional arguments to pass to the `hf_hub_download` method when
38 loading from the HuggingFace Hub.
39 """
40 - path = (os.path.join(model_id, hf_hub_download_kwargs["subfolder"])
41 - if hf_hub_download_kwargs.get("subfolder", None) is not None else
42 - model_id)
40 + path = (os.path.join(model_id, hf_hub_download_kwargs["subfolder"]) if
41 + hf_hub_download_kwargs.get("subfolder") is not None else model_id)
42
43 if device is None:
44 device = infer_device()
@@ -51,19 +50,19 @@ def load_peft_weights(model_id: str,
50 filename = os.path.join(path, WEIGHTS_NAME)
51 use_safetensors = False
52 else:
54 - token = hf_hub_download_kwargs.get("token", None)
53 + token = hf_hub_download_kwargs.get("token")
54 if token is None:
56 - token = hf_hub_download_kwargs.get("use_auth_token", None)
55 + token = hf_hub_download_kwargs.get("use_auth_token")
56
57 hub_filename = (os.path.join(hf_hub_download_kwargs["subfolder"],
58 SAFETENSORS_WEIGHTS_NAME)
60 - if hf_hub_download_kwargs.get("subfolder", None)
61 - is not None else SAFETENSORS_WEIGHTS_NAME)
59 + if hf_hub_download_kwargs.get("subfolder") is not None
60 + else SAFETENSORS_WEIGHTS_NAME)
61 has_remote_safetensors_file = file_exists(
62 repo_id=model_id,
63 filename=hub_filename,
65 - revision=hf_hub_download_kwargs.get("revision", None),
66 - repo_type=hf_hub_download_kwargs.get("repo_type", None),
64 + revision=hf_hub_download_kwargs.get("revision"),
65 + repo_type=hf_hub_download_kwargs.get("repo_type"),
66 token=token,
67 )
68 use_safetensors = has_remote_safetensors_file
vllm/transformers_utils/config.py
+1 -1
@@ -308,7 +308,7 @@ def load_params_config(model, revision) -> PretrainedConfig:
308 config_path = Path(
309 hf_hub_download(model, config_file_name, revision=revision))
310
311 - with open(config_path, "r") as file:
311 + with open(config_path) as file:
312 config_dict = json.load(file)
313
314 config_mapping = {
vllm/transformers_utils/configs/chatglm.py
-1
@@ -1,4 +1,3 @@
1 -# coding=utf-8
1 # Adapted from
2 # https://github.com/THUDM/ChatGLM2-6B
3 from transformers import PretrainedConfig
vllm/transformers_utils/configs/exaone.py
-1
@@ -1,4 +1,3 @@
1 -# coding=utf-8
1 # Copied from
2 # https://huggingface.co/LGAI-EXAONE/EXAONE-3.0-7.8B-Instruct/blob/main/configuration_exaone.py
3 # Copyright 2021 The LG AI Research EXAONE Lab. All rights reserved.
vllm/transformers_utils/configs/jais.py
-1
@@ -1,4 +1,3 @@
1 -# coding=utf-8
1 # Copyright 2023 The OpenAI Team Authors and HuggingFace Inc. team.
2 # Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved.
3 # Copyright 2023 Cerebras Systems.
vllm/transformers_utils/configs/mpt.py
+3 -4
@@ -1,4 +1,3 @@
1 -# coding=utf-8
1 # Copied from
2 # https://huggingface.co/mosaicml/mpt-7b/blob/main/configuration_mpt.py
3 """A HuggingFace-style model configuration."""
@@ -117,10 +116,10 @@ class MPTConfig(PretrainedConfig):
116 init_config_defaults)
117 if self.d_model % self.n_heads != 0:
118 raise ValueError('d_model must be divisible by n_heads')
120 - if any((
119 + if any(
120 prob < 0 or prob > 1 for prob in
122 - [self.attn_config['attn_pdrop'], self.resid_pdrop, self.emb_pdrop]
123 - )):
121 + [self.attn_config['attn_pdrop'], self.resid_pdrop, self.emb_pdrop
122 + ]):
123 raise ValueError(
124 "self.attn_config['attn_pdrop'], resid_pdrop, emb_pdrop are "
125 "probabilities and must be between 0 and 1")
vllm/transformers_utils/configs/nemotron.py
+3 -4
@@ -1,4 +1,3 @@
1 -# coding=utf-8
1 # Copyright 2024 HuggingFace Inc. team. All rights reserved.
2 # Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
3 #
@@ -144,7 +143,7 @@ class NemotronConfig(PretrainedConfig):
143 self.intermediate_size = intermediate_size
144 self.num_hidden_layers = num_hidden_layers
145 self.num_attention_heads = num_attention_heads
147 - head_dim = head_dim or kwargs.get("kv_channels", None)
146 + head_dim = head_dim or kwargs.get("kv_channels")
147 self.head_dim = head_dim if head_dim is not None else (
148 hidden_size // num_attention_heads)
149
@@ -160,8 +159,8 @@ class NemotronConfig(PretrainedConfig):
159 self.rope_theta = rope_theta
160 self.rope_scaling = rope_scaling
161 # for backward compatibility
163 - partial_rotary_factor = kwargs.get("rope_percent", None) or kwargs.get(
164 - "rope_percentage", None) or partial_rotary_factor
162 + partial_rotary_factor = kwargs.get("rope_percent") or kwargs.get(
163 + "rope_percentage") or partial_rotary_factor
164 self.partial_rotary_factor = partial_rotary_factor
165 self._rope_scaling_validation()
166 self.attention_bias = attention_bias
vllm/transformers_utils/configs/solar.py
-1
@@ -1,4 +1,3 @@
1 -# coding=utf-8
1 # Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
2 #
3 # This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
vllm/utils.py
+2 -2
@@ -1153,7 +1153,7 @@ class SortedHelpFormatter(argparse.HelpFormatter):
1153
1154 def add_arguments(self, actions):
1155 actions = sorted(actions, key=lambda x: x.option_strings)
1156 - super(SortedHelpFormatter, self).add_arguments(actions)
1156 + super().add_arguments(actions)
1157
1158
1159 class FlexibleArgumentParser(argparse.ArgumentParser):
@@ -1279,7 +1279,7 @@ class FlexibleArgumentParser(argparse.ArgumentParser):
1279
1280 config: Dict[str, Union[int, str]] = {}
1281 try:
1282 - with open(file_path, 'r') as config_file:
1282 + with open(file_path) as config_file:
1283 config = yaml.safe_load(config_file)
1284 except Exception as ex:
1285 logger.error(