A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
Re-enable the 80 char line width limit (#3305)
Zhuohan Li committed
Mar 10, 2024 at 19:49 UTC
2f8844ba08d77af8a64784317055b03a475f6051
67 files changed
+557
-528
pyproject.toml
+4
-2
@@ -9,6 +9,10 @@ requires = [
9
]
10
build-backend = "setuptools.build_meta"
11
12
+[tool.ruff]
13
+# Allow lines to be as long as 80.
14
+line-length = 80
15
+
16
[tool.ruff.lint]
17
select = [
18
# pycodestyle
@@ -29,8 +33,6 @@ ignore = [
33
"F405", "F403",
34
# lambda expression assignment
35
"E731",
32
- # line too long, handled by black formatting
33
- "E501",
36
# .strip() with multi-character strings
37
"B005",
38
# Loop control variable not used within loop body
setup.py
+2
-2
@@ -142,8 +142,8 @@ def get_pytorch_rocm_arch() -> Set[str]:
142
# If we don't have PYTORCH_ROCM_ARCH specified pull the list from rocm_agent_enumerator
143
if env_arch_list is None:
144
command = "rocm_agent_enumerator"
145
- env_arch_list = subprocess.check_output([command]).decode('utf-8')\
146
- .strip().replace("\n", ";")
145
+ env_arch_list = (subprocess.check_output(
146
+ [command]).decode('utf-8').strip().replace("\n", ";"))
147
arch_source_str = "rocm_agent_enumerator"
148
else:
149
arch_source_str = "PYTORCH_ROCM_ARCH env variable"
tests/async_engine/test_chat_template.py
+4
-2
@@ -73,7 +73,7 @@ def test_load_chat_template():
73
assert template_content is not None
74
# Hard coded value for template_chatml.jinja
75
assert template_content == """{% for message in messages %}{{'<|im_start|>' + message['role'] + '\\n' + message['content']}}{% if (loop.last and add_generation_prompt) or not loop.last %}{{ '<|im_end|>' + '\\n'}}{% endif %}{% endfor %}
76
-{% if add_generation_prompt and messages[-1]['role'] != 'assistant' %}{{ '<|im_start|>assistant\\n' }}{% endif %}"""
76
+{% if add_generation_prompt and messages[-1]['role'] != 'assistant' %}{{ '<|im_start|>assistant\\n' }}{% endif %}""" # noqa: E501
77
78
79
def test_no_load_chat_template():
@@ -117,4 +117,6 @@ async def test_get_gen_prompt(model, template, add_generation_prompt,
117
add_generation_prompt=mock_request.add_generation_prompt)
118
119
# Test assertion
120
- assert result == expected_output, f"The generated prompt does not match the expected output for model {model} and template {template}"
120
+ assert result == expected_output, (
121
+ f"The generated prompt does not match the expected output for "
122
+ f"model {model} and template {template}")
tests/core/test_block_manager.py
+2
-1
@@ -4,7 +4,8 @@ from typing import List
4
5
from vllm import SamplingParams
6
from vllm.block import PhysicalTokenBlock
7
-from vllm.core.block_manager import BlockAllocator, BlockSpaceManager, AllocStatus
7
+from vllm.core.block_manager import (BlockAllocator, BlockSpaceManager,
8
+ AllocStatus)
9
from vllm.utils import Device
10
from vllm.sequence import Sequence, SequenceGroup, SequenceStatus, Logprob
11
tests/entrypoints/test_guided_processors.py
+2
-2
@@ -46,8 +46,8 @@ TEST_SCHEMA = {
46
"required": ["name", "age", "skills", "work history"]
47
}
48
49
-TEST_REGEX = r"((25[0-5]|(2[0-4]|1\d|[1-9]|)\d)\.){3}" + \
50
- r"(25[0-5]|(2[0-4]|1\d|[1-9]|)\d)"
49
+TEST_REGEX = (r"((25[0-5]|(2[0-4]|1\d|[1-9]|)\d)\.){3}"
50
+ r"(25[0-5]|(2[0-4]|1\d|[1-9]|)\d)")
51
52
53
def test_guided_logits_processors():
tests/entrypoints/test_openai_server.py
+23
-13
@@ -5,9 +5,12 @@ import time
5
import sys
6
import pytest
7
import requests
8
-import ray # using Ray for overall ease of process management, parallel requests, and debugging.
8
+# using Ray for overall ease of process management, parallel requests,
9
+# and debugging.
10
+import ray
11
import openai # use the official client for correctness check
10
-from huggingface_hub import snapshot_download # downloading lora to test lora requests
12
+# downloading lora to test lora requests
13
+from huggingface_hub import snapshot_download
14
15
# imports for guided decoding tests
16
import json
@@ -17,8 +20,11 @@ import re
20
from vllm.transformers_utils.tokenizer import get_tokenizer
21
22
MAX_SERVER_START_WAIT_S = 600 # wait for server to start for 60 seconds
20
-MODEL_NAME = "HuggingFaceH4/zephyr-7b-beta" # any model with a chat template should work here
21
-LORA_NAME = "typeof/zephyr-7b-beta-lora" # technically this needs Mistral-7B-v0.1 as base, but we're not testing generation quality here
23
+# any model with a chat template should work here
24
+MODEL_NAME = "HuggingFaceH4/zephyr-7b-beta"
25
+# technically this needs Mistral-7B-v0.1 as base, but we're not testing
26
+# generation quality here
27
+LORA_NAME = "typeof/zephyr-7b-beta-lora"
28
29
TEST_SCHEMA = {
30
"type": "object",
@@ -59,8 +65,8 @@ TEST_SCHEMA = {
65
"required": ["name", "age", "skills", "work history"]
66
}
67
62
-TEST_REGEX = r"((25[0-5]|(2[0-4]|1\d|[1-9]|)\d)\.){3}" + \
63
- r"(25[0-5]|(2[0-4]|1\d|[1-9]|)\d)"
68
+TEST_REGEX = (r"((25[0-5]|(2[0-4]|1\d|[1-9]|)\d)\.){3}"
69
+ r"(25[0-5]|(2[0-4]|1\d|[1-9]|)\d)")
70
71
TEST_CHOICE = [
72
"Python", "Java", "JavaScript", "C++", "C#", "PHP", "TypeScript", "Ruby",
@@ -120,8 +126,9 @@ def server(zephyr_lora_files):
126
server_runner = ServerRunner.remote([
127
"--model",
128
MODEL_NAME,
129
+ # use half precision for speed and memory savings in CI environment
130
"--dtype",
124
- "bfloat16", # use half precision for speed and memory savings in CI environment
131
+ "bfloat16",
132
"--max-model-len",
133
"8192",
134
"--enforce-eager",
@@ -392,7 +399,8 @@ async def test_batch_completions(server, client: openai.AsyncOpenAI,
399
max_tokens=5,
400
temperature=0.0,
401
extra_body=dict(
395
- # NOTE: this has to be true for n > 1 in vLLM, but not necessary for official client.
402
+ # NOTE: this has to be true for n > 1 in vLLM, but not necessary
403
+ # for official client.
404
use_beam_search=True),
405
)
406
assert len(batch.choices) == 4
@@ -469,8 +477,8 @@ async def test_logits_bias(server, client: openai.AsyncOpenAI):
477
async def test_guided_json_completion(server, client: openai.AsyncOpenAI):
478
completion = await client.completions.create(
479
model=MODEL_NAME,
472
- prompt=
473
- f"Give an example JSON for an employee profile that fits this schema: {TEST_SCHEMA}",
480
+ prompt=f"Give an example JSON for an employee profile "
481
+ f"that fits this schema: {TEST_SCHEMA}",
482
n=3,
483
temperature=1.0,
484
max_tokens=500,
@@ -489,9 +497,11 @@ async def test_guided_json_chat(server, client: openai.AsyncOpenAI):
497
"role": "system",
498
"content": "you are a helpful assistant"
499
}, {
492
- "role": "user",
493
- "content": "Give an example JSON for an employee profile that " + \
494
- f"fits this schema: {TEST_SCHEMA}"
500
+ "role":
501
+ "user",
502
+ "content":
503
+ f"Give an example JSON for an employee profile that "
504
+ f"fits this schema: {TEST_SCHEMA}"
505
}]
506
chat_completion = await client.chat.completions.create(
507
model=MODEL_NAME,
tests/kernels/test_moe.py
+2
-1
@@ -57,7 +57,8 @@ def test_fused_moe(
57
[torch.float32, torch.float16, torch.bfloat16])
58
@torch.inference_mode()
59
def test_mixtral_moe(dtype: torch.dtype):
60
- "Make sure our Mixtral MoE implementation agrees with the one from huggingface."
60
+ """Make sure our Mixtral MoE implementation agrees with the one from
61
+ huggingface."""
62
63
# Instantiate our and huggingface's MoE blocks
64
config = MixtralConfig()
tests/kernels/test_prefix_prefill.py
+2
-1
@@ -114,7 +114,8 @@ def test_contexted_kv_attention(
114
v_cache = v_cache.view(-1, block_size, num_kv_heads,
115
head_size).permute(0, 2, 3, 1).contiguous()
116
117
- # Warm up the Triton kernel by calling it once before actually measuring generation time
117
+ # Warm up the Triton kernel by calling it once before actually measuring
118
+ # generation time
119
context_attention_fwd(query, k, v, output, k_cache, v_cache, block_table,
120
b_start_loc, b_seq_len, b_ctx_len, max_input_len)
121
torch.cuda.synchronize()
tests/lora/test_layer_variation.py
+3
-3
@@ -11,9 +11,9 @@ from .conftest import cleanup
11
12
MODEL_PATH = "Felladrin/Llama-68M-Chat-v1"
13
PROMPTS = [
14
- "[system] Given a target sentence construct the underlying meaning representation\nof the input sentence as a single function with attributes and attribute\nvalues. This function should describe the target string accurately and the\nfunction must be one of the following ['inform', 'request', 'give_opinion',\n'confirm', 'verify_attribute', 'suggest', 'request_explanation',\n'recommend', 'request_attribute'].\n\nThe attributes must be one of the following:\n['name', 'exp_release_date', 'release_year', 'developer', 'esrb', 'rating',\n'genres', 'player_perspective', 'has_multiplayer', 'platforms',\n'available_on_steam', 'has_linux_release', 'has_mac_release', 'specifier'] [/system] [user] Here is the target sentence:\nSpellForce 3 is a pretty bad game. The developer Grimlore Games is clearly a bunch of no-talent hacks, and 2017 was a terrible year for games anyway. [/user] [assistant]",
15
- "[system] Given a target sentence construct the underlying meaning representation\nof the input sentence as a single function with attributes and attribute\nvalues. This function should describe the target string accurately and the\nfunction must be one of the following ['inform', 'request', 'give_opinion',\n'confirm', 'verify_attribute', 'suggest', 'request_explanation',\n'recommend', 'request_attribute'].\n\nThe attributes must be one of the following:\n['name', 'exp_release_date', 'release_year', 'developer', 'esrb', 'rating',\n'genres', 'player_perspective', 'has_multiplayer', 'platforms',\n'available_on_steam', 'has_linux_release', 'has_mac_release', 'specifier'] [/system] [user] Here is the target sentence:\nI wanted to like Grimlore Games' 2017 entry, but in SpellForce 3 they just didn't get anything right. [/user] [assistant]",
16
- "[system] Given a target sentence construct the underlying meaning representation\nof the input sentence as a single function with attributes and attribute\nvalues. This function should describe the target string accurately and the\nfunction must be one of the following ['inform', 'request', 'give_opinion',\n'confirm', 'verify_attribute', 'suggest', 'request_explanation',\n'recommend', 'request_attribute'].\n\nThe attributes must be one of the following:\n['name', 'exp_release_date', 'release_year', 'developer', 'esrb', 'rating',\n'genres', 'player_perspective', 'has_multiplayer', 'platforms',\n'available_on_steam', 'has_linux_release', 'has_mac_release', 'specifier'] [/system] [user] Here is the target sentence:\nBioShock is a good role-playing, action-adventure, shooter that released for PlayStation, Xbox, and PC in 2007. It is available on Steam, and it has a Mac release but not a Linux release. [/user] [assistant]",
14
+ "[system] Given a target sentence construct the underlying meaning representation\nof the input sentence as a single function with attributes and attribute\nvalues. This function should describe the target string accurately and the\nfunction must be one of the following ['inform', 'request', 'give_opinion',\n'confirm', 'verify_attribute', 'suggest', 'request_explanation',\n'recommend', 'request_attribute'].\n\nThe attributes must be one of the following:\n['name', 'exp_release_date', 'release_year', 'developer', 'esrb', 'rating',\n'genres', 'player_perspective', 'has_multiplayer', 'platforms',\n'available_on_steam', 'has_linux_release', 'has_mac_release', 'specifier'] [/system] [user] Here is the target sentence:\nSpellForce 3 is a pretty bad game. The developer Grimlore Games is clearly a bunch of no-talent hacks, and 2017 was a terrible year for games anyway. [/user] [assistant]", # noqa: E501
15
+ "[system] Given a target sentence construct the underlying meaning representation\nof the input sentence as a single function with attributes and attribute\nvalues. This function should describe the target string accurately and the\nfunction must be one of the following ['inform', 'request', 'give_opinion',\n'confirm', 'verify_attribute', 'suggest', 'request_explanation',\n'recommend', 'request_attribute'].\n\nThe attributes must be one of the following:\n['name', 'exp_release_date', 'release_year', 'developer', 'esrb', 'rating',\n'genres', 'player_perspective', 'has_multiplayer', 'platforms',\n'available_on_steam', 'has_linux_release', 'has_mac_release', 'specifier'] [/system] [user] Here is the target sentence:\nI wanted to like Grimlore Games' 2017 entry, but in SpellForce 3 they just didn't get anything right. [/user] [assistant]", # noqa: E501
16
+ "[system] Given a target sentence construct the underlying meaning representation\nof the input sentence as a single function with attributes and attribute\nvalues. This function should describe the target string accurately and the\nfunction must be one of the following ['inform', 'request', 'give_opinion',\n'confirm', 'verify_attribute', 'suggest', 'request_explanation',\n'recommend', 'request_attribute'].\n\nThe attributes must be one of the following:\n['name', 'exp_release_date', 'release_year', 'developer', 'esrb', 'rating',\n'genres', 'player_perspective', 'has_multiplayer', 'platforms',\n'available_on_steam', 'has_linux_release', 'has_mac_release', 'specifier'] [/system] [user] Here is the target sentence:\nBioShock is a good role-playing, action-adventure, shooter that released for PlayStation, Xbox, and PC in 2007. It is available on Steam, and it has a Mac release but not a Linux release. [/user] [assistant]", # noqa: E501
17
]
18
19
tests/lora/test_layers.py
+9
-6
@@ -17,14 +17,16 @@ from vllm.lora.layers import (
17
LoRAMapping,
18
BaseLayerWithLoRA,
19
)
20
-from vllm.lora.models import LoRALayerWeights, convert_mapping, PackedLoRALayerWeights
20
+from vllm.lora.models import (LoRALayerWeights, convert_mapping,
21
+ PackedLoRALayerWeights)
22
from vllm.config import LoRAConfig
23
from vllm.model_executor.layers.sampler import Sampler
24
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
25
MergedColumnParallelLinear,
26
RowParallelLinear,
27
QKVParallelLinear)
27
-from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding, ParallelLMHead
28
+from vllm.model_executor.layers.vocab_parallel_embedding import (
29
+ VocabParallelEmbedding, ParallelLMHead)
30
from vllm.model_executor.utils import set_random_seed
31
32
from .utils import DummyLoRAManager
@@ -258,7 +260,8 @@ def test_embeddings(dist_init, num_loras, device) -> None:
260
261
262
@torch.inference_mode()
261
-# @pytest.mark.skip(reason="Fails when loras are in any slot other than the first.")
263
+# @pytest.mark.skip(
264
+# reason="Fails when loras are in any slot other than the first.")
265
@pytest.mark.parametrize("num_loras", [1, 2, 4, 8])
266
@pytest.mark.parametrize("device", CUDA_DEVICES)
267
def test_embeddings_with_new_embeddings(dist_init, num_loras, device) -> None:
@@ -674,9 +677,9 @@ def test_column_parallel_packed(dist_init, num_loras, repeats, device) -> None:
677
result = linear(input_)[0]
678
subloras = sublora_dict[lora_id]
679
for i, sublora in enumerate(subloras):
677
- result[:, sublora.lora_b.shape[1] * i:sublora.lora_b.shape[1] * (
678
- i + 1
679
- )] += input_ @ sublora.lora_a @ sublora.lora_b * sublora.scaling
680
+ result[:, sublora.lora_b.shape[1] * i:sublora.lora_b.shape[1] *
681
+ (i + 1)] += (input_ @ sublora.lora_a @ sublora.lora_b *
682
+ sublora.scaling)
683
expected_results.append(result)
684
expected_result = torch.cat(expected_results)
685
tests/lora/test_llama.py
+25
-22
@@ -10,12 +10,12 @@ MODEL_PATH = "meta-llama/Llama-2-7b-hf"
10
11
def do_sample(llm, lora_path: str, lora_id: int):
12
prompts = [
13
- "[user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_74 (icao VARCHAR, airport VARCHAR)\n\n question: Name the ICAO for lilongwe international airport [/user] [assistant]",
14
- "[user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_11 (nationality VARCHAR, elector VARCHAR)\n\n question: When Anchero Pantaleone was the elector what is under nationality? [/user] [assistant]",
15
- "[user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_95 (one_mora VARCHAR, gloss VARCHAR, accented_mora VARCHAR)\n\n question: What is the one mora for a low tone mora with a gloss of /˩okiru/ [òkìɽɯ́]? [/user] [assistant]",
16
- "[user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE candidate (people_id VARCHAR, unsure_rate INTEGER); CREATE TABLE people (sex VARCHAR, people_id VARCHAR)\n\n question: which gender got the highest average uncertain ratio. [/user] [assistant]",
17
- "[user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_60 (pick INTEGER, former_wnba_team VARCHAR)\n\n question: What pick was a player that previously played for the Minnesota Lynx? [/user] [assistant]",
18
- "[user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_28138035_4 (womens_doubles VARCHAR, mens_singles VARCHAR)\n\n question: Name the women's doubles for werner schlager [/user] [assistant]"
13
+ "[user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_74 (icao VARCHAR, airport VARCHAR)\n\n question: Name the ICAO for lilongwe international airport [/user] [assistant]", # noqa: E501
14
+ "[user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_11 (nationality VARCHAR, elector VARCHAR)\n\n question: When Anchero Pantaleone was the elector what is under nationality? [/user] [assistant]", # noqa: E501
15
+ "[user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_95 (one_mora VARCHAR, gloss VARCHAR, accented_mora VARCHAR)\n\n question: What is the one mora for a low tone mora with a gloss of /˩okiru/ [òkìɽɯ́]? [/user] [assistant]", # noqa: E501
16
+ "[user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE candidate (people_id VARCHAR, unsure_rate INTEGER); CREATE TABLE people (sex VARCHAR, people_id VARCHAR)\n\n question: which gender got the highest average uncertain ratio. [/user] [assistant]", # noqa: E501
17
+ "[user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_60 (pick INTEGER, former_wnba_team VARCHAR)\n\n question: What pick was a player that previously played for the Minnesota Lynx? [/user] [assistant]", # noqa: E501
18
+ "[user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_28138035_4 (womens_doubles VARCHAR, mens_singles VARCHAR)\n\n question: Name the women's doubles for werner schlager [/user] [assistant]" # noqa: E501
19
]
20
sampling_params = vllm.SamplingParams(temperature=0,
21
max_tokens=256,
@@ -48,20 +48,20 @@ def test_llama_lora(sql_lora_files, tp_size):
48
tensor_parallel_size=tp_size)
49
50
expected_no_lora_output = [
51
- "\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_75 (icao VARCHAR, airport VARCHAR)\n\n question: Name the ICAO for lilongwe international airport [/user] [assistant]\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_76 (icao VARCHAR, airport VARCHAR)\n\n question: Name the ICAO for lilongwe international airport [/user] [assistant]\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_77 (icao VARCHAR, airport VARCHAR)\n\n question: Name the ICAO for lilongwe international airport [/user] [assistant]\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_78 (icao VARCHAR, airport VARCHAR)\n\n question: Name the ICAO for lilongwe international airport [/user]",
52
- " Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_11 (nationality VARCHAR, elector VARCHAR)\n\n question: When Anchero Pantaleone was the elector what is under nationality? ",
53
- "\n\n answer: 1\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_96 (one_mora VARCHAR, gloss VARCHAR, accented_mora VARCHAR)\n\n question: What is the one mora for a high tone mora with a gloss of /˧kot/ [kòt]? [/user] [assistant]\n\n answer: 2\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_97 (one_mora VARCHAR, gloss VARCHAR, accented_mora VARCHAR)\n\n question: What is the one mora for a high tone mora with a gloss of /˧kot/ [kòt]? [/user] [assistant]\n\n answer: 2\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_98 (one_mora VARCHAR, gloss VARCHAR, accented_mora VARCHAR)\n\n question: What is the one m",
54
- " Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE candidate (people_id VARCHAR, unsure_rate INTEGER); CREATE TABLE people (sex VARCHAR, people_id VARCHAR)\n\n question: which gender got the highest average uncertain ratio. ",
55
- " Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_60 (pick INTEGER, former_wnba_team VARCHAR)\n\n question: What pick was a player that previously played for the Minnesota Lynx? ",
56
- "\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_28138035_4 (womens_doubles VARCHAR, mens_singles VARCHAR)\n\n question: Name the women's doubles for werner schlager [/user] [assistant]\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_28138035_4 (womens_doubles VARCHAR, mens_singles VARCHAR)\n\n question: Name the women's doubles for werner schlager [/user] [assistant]\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_28138035_4 (womens_doubles VARCHAR, mens_singles VARCHAR)\n\n question: Name the women's doubles for werner schlager [/user] [assistant]\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE",
51
+ "\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_75 (icao VARCHAR, airport VARCHAR)\n\n question: Name the ICAO for lilongwe international airport [/user] [assistant]\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_76 (icao VARCHAR, airport VARCHAR)\n\n question: Name the ICAO for lilongwe international airport [/user] [assistant]\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_77 (icao VARCHAR, airport VARCHAR)\n\n question: Name the ICAO for lilongwe international airport [/user] [assistant]\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_78 (icao VARCHAR, airport VARCHAR)\n\n question: Name the ICAO for lilongwe international airport [/user]", # noqa: E501
52
+ " Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_11 (nationality VARCHAR, elector VARCHAR)\n\n question: When Anchero Pantaleone was the elector what is under nationality? ", # noqa: E501
53
+ "\n\n answer: 1\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_96 (one_mora VARCHAR, gloss VARCHAR, accented_mora VARCHAR)\n\n question: What is the one mora for a high tone mora with a gloss of /˧kot/ [kòt]? [/user] [assistant]\n\n answer: 2\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_97 (one_mora VARCHAR, gloss VARCHAR, accented_mora VARCHAR)\n\n question: What is the one mora for a high tone mora with a gloss of /˧kot/ [kòt]? [/user] [assistant]\n\n answer: 2\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_98 (one_mora VARCHAR, gloss VARCHAR, accented_mora VARCHAR)\n\n question: What is the one m", # noqa: E501
54
+ " Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE candidate (people_id VARCHAR, unsure_rate INTEGER); CREATE TABLE people (sex VARCHAR, people_id VARCHAR)\n\n question: which gender got the highest average uncertain ratio. ", # noqa: E501
55
+ " Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_60 (pick INTEGER, former_wnba_team VARCHAR)\n\n question: What pick was a player that previously played for the Minnesota Lynx? ", # noqa: E501
56
+ "\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_28138035_4 (womens_doubles VARCHAR, mens_singles VARCHAR)\n\n question: Name the women's doubles for werner schlager [/user] [assistant]\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_28138035_4 (womens_doubles VARCHAR, mens_singles VARCHAR)\n\n question: Name the women's doubles for werner schlager [/user] [assistant]\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_28138035_4 (womens_doubles VARCHAR, mens_singles VARCHAR)\n\n question: Name the women's doubles for werner schlager [/user] [assistant]\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE", # noqa: E501
57
]
58
expected_lora_output = [
59
- " SELECT icao FROM table_name_74 WHERE airport = 'lilongwe international airport' ",
60
- " SELECT nationality FROM table_name_11 WHERE elector = 'anchero pantaleone' ",
61
- " SELECT one_mora FROM table_name_95 WHERE gloss = 'low tone mora with a gloss of /˩okiru/' [òkìɽɯ́] AND accented_mora = 'low tone mora with a gloss of /˩okiru/' [òkìɽɯ́] ",
62
- " SELECT sex FROM people WHERE people_id IN (SELECT people_id FROM candidate GROUP BY sex ORDER BY COUNT(people_id) DESC LIMIT 1) ",
63
- " SELECT pick FROM table_name_60 WHERE former_wnba_team = 'Minnesota Lynx' ",
64
- " SELECT womens_doubles FROM table_28138035_4 WHERE mens_singles = 'Werner Schlager' "
59
+ " SELECT icao FROM table_name_74 WHERE airport = 'lilongwe international airport' ", # noqa: E501
60
+ " SELECT nationality FROM table_name_11 WHERE elector = 'anchero pantaleone' ", # noqa: E501
61
+ " SELECT one_mora FROM table_name_95 WHERE gloss = 'low tone mora with a gloss of /˩okiru/' [òkìɽɯ́] AND accented_mora = 'low tone mora with a gloss of /˩okiru/' [òkìɽɯ́] ", # noqa: E501
62
+ " SELECT sex FROM people WHERE people_id IN (SELECT people_id FROM candidate GROUP BY sex ORDER BY COUNT(people_id) DESC LIMIT 1) ", # noqa: E501
63
+ " SELECT pick FROM table_name_60 WHERE former_wnba_team = 'Minnesota Lynx' ", # noqa: E501
64
+ " SELECT womens_doubles FROM table_28138035_4 WHERE mens_singles = 'Werner Schlager' " # noqa: E501
65
]
66
67
print("lora adapter created")
@@ -121,7 +121,8 @@ def test_llama_tensor_parallel_equality(sql_lora_files):
121
122
123
def test_llama_lora_warmup(sql_lora_files):
124
- """Test that the LLM initialization works with a warmup LORA path and is more conservative"""
124
+ """Test that the LLM initialization works with a warmup LORA path and
125
+ is more conservative"""
126
127
@ray.remote(num_gpus=1)
128
def get_num_gpu_blocks_lora():
@@ -132,13 +133,15 @@ def test_llama_lora_warmup(sql_lora_files):
133
@ray.remote(num_gpus=1)
134
def get_num_gpu_blocks_no_lora():
135
llm = vllm.LLM(MODEL_PATH, max_num_seqs=16)
135
- num_gpu_blocks_no_lora_warmup = llm.llm_engine.cache_config.num_gpu_blocks
136
+ num_gpu_blocks_no_lora_warmup = (
137
+ llm.llm_engine.cache_config.num_gpu_blocks)
138
return num_gpu_blocks_no_lora_warmup
139
140
num_gpu_blocks_lora_warmup = ray.get(get_num_gpu_blocks_lora.remote())
141
num_gpu_blocks_no_lora_warmup = ray.get(
142
get_num_gpu_blocks_no_lora.remote())
143
assert num_gpu_blocks_lora_warmup < num_gpu_blocks_no_lora_warmup, (
142
- "The warmup with lora should be more"
143
- " conservative than without lora, therefore the number of memory blocks for the KV cache should be "
144
+ "The warmup with lora should be more "
145
+ "conservative than without lora, therefore the number of "
146
+ "memory blocks for the KV cache should be "
147
"less when using lora than when not using lora")
tests/lora/test_mixtral.py
+6
-6
@@ -9,9 +9,9 @@ MODEL_PATH = "mistralai/Mixtral-8x7B-Instruct-v0.1"
9
10
def do_sample(llm, lora_path: str, lora_id: int):
11
prompts = [
12
- "[system] Given a target sentence construct the underlying meaning representation\nof the input sentence as a single function with attributes and attribute\nvalues. This function should describe the target string accurately and the\nfunction must be one of the following ['inform', 'request', 'give_opinion',\n'confirm', 'verify_attribute', 'suggest', 'request_explanation',\n'recommend', 'request_attribute'].\n\nThe attributes must be one of the following:\n['name', 'exp_release_date', 'release_year', 'developer', 'esrb', 'rating',\n'genres', 'player_perspective', 'has_multiplayer', 'platforms',\n'available_on_steam', 'has_linux_release', 'has_mac_release', 'specifier'] [/system] [user] Here is the target sentence:\nSpellForce 3 is a pretty bad game. The developer Grimlore Games is clearly a bunch of no-talent hacks, and 2017 was a terrible year for games anyway. [/user] [assistant]",
13
- "[system] Given a target sentence construct the underlying meaning representation\nof the input sentence as a single function with attributes and attribute\nvalues. This function should describe the target string accurately and the\nfunction must be one of the following ['inform', 'request', 'give_opinion',\n'confirm', 'verify_attribute', 'suggest', 'request_explanation',\n'recommend', 'request_attribute'].\n\nThe attributes must be one of the following:\n['name', 'exp_release_date', 'release_year', 'developer', 'esrb', 'rating',\n'genres', 'player_perspective', 'has_multiplayer', 'platforms',\n'available_on_steam', 'has_linux_release', 'has_mac_release', 'specifier'] [/system] [user] Here is the target sentence:\nI wanted to like Grimlore Games' 2017 entry, but in SpellForce 3 they just didn't get anything right. [/user] [assistant]",
14
- "[system] Given a target sentence construct the underlying meaning representation\nof the input sentence as a single function with attributes and attribute\nvalues. This function should describe the target string accurately and the\nfunction must be one of the following ['inform', 'request', 'give_opinion',\n'confirm', 'verify_attribute', 'suggest', 'request_explanation',\n'recommend', 'request_attribute'].\n\nThe attributes must be one of the following:\n['name', 'exp_release_date', 'release_year', 'developer', 'esrb', 'rating',\n'genres', 'player_perspective', 'has_multiplayer', 'platforms',\n'available_on_steam', 'has_linux_release', 'has_mac_release', 'specifier'] [/system] [user] Here is the target sentence:\nBioShock is a good role-playing, action-adventure, shooter that released for PlayStation, Xbox, and PC in 2007. It is available on Steam, and it has a Mac release but not a Linux release. [/user] [assistant]",
12
+ "[system] Given a target sentence construct the underlying meaning representation\nof the input sentence as a single function with attributes and attribute\nvalues. This function should describe the target string accurately and the\nfunction must be one of the following ['inform', 'request', 'give_opinion',\n'confirm', 'verify_attribute', 'suggest', 'request_explanation',\n'recommend', 'request_attribute'].\n\nThe attributes must be one of the following:\n['name', 'exp_release_date', 'release_year', 'developer', 'esrb', 'rating',\n'genres', 'player_perspective', 'has_multiplayer', 'platforms',\n'available_on_steam', 'has_linux_release', 'has_mac_release', 'specifier'] [/system] [user] Here is the target sentence:\nSpellForce 3 is a pretty bad game. The developer Grimlore Games is clearly a bunch of no-talent hacks, and 2017 was a terrible year for games anyway. [/user] [assistant]", # noqa: E501
13
+ "[system] Given a target sentence construct the underlying meaning representation\nof the input sentence as a single function with attributes and attribute\nvalues. This function should describe the target string accurately and the\nfunction must be one of the following ['inform', 'request', 'give_opinion',\n'confirm', 'verify_attribute', 'suggest', 'request_explanation',\n'recommend', 'request_attribute'].\n\nThe attributes must be one of the following:\n['name', 'exp_release_date', 'release_year', 'developer', 'esrb', 'rating',\n'genres', 'player_perspective', 'has_multiplayer', 'platforms',\n'available_on_steam', 'has_linux_release', 'has_mac_release', 'specifier'] [/system] [user] Here is the target sentence:\nI wanted to like Grimlore Games' 2017 entry, but in SpellForce 3 they just didn't get anything right. [/user] [assistant]", # noqa: E501
14
+ "[system] Given a target sentence construct the underlying meaning representation\nof the input sentence as a single function with attributes and attribute\nvalues. This function should describe the target string accurately and the\nfunction must be one of the following ['inform', 'request', 'give_opinion',\n'confirm', 'verify_attribute', 'suggest', 'request_explanation',\n'recommend', 'request_attribute'].\n\nThe attributes must be one of the following:\n['name', 'exp_release_date', 'release_year', 'developer', 'esrb', 'rating',\n'genres', 'player_perspective', 'has_multiplayer', 'platforms',\n'available_on_steam', 'has_linux_release', 'has_mac_release', 'specifier'] [/system] [user] Here is the target sentence:\nBioShock is a good role-playing, action-adventure, shooter that released for PlayStation, Xbox, and PC in 2007. It is available on Steam, and it has a Mac release but not a Linux release. [/user] [assistant]", # noqa: E501
15
]
16
sampling_params = vllm.SamplingParams(temperature=0, max_tokens=256)
17
outputs = llm.generate(
@@ -42,9 +42,9 @@ def test_mixtral_lora(mixtral_lora_files, tp_size):
42
worker_use_ray=True)
43
44
expected_lora_output = [
45
- "give_opinion(name[SpellForce 3], release_year[2017], developer[Grimlore Games], rating[poor])",
46
- "give_opinion(name[SpellForce 3], release_year[2017], developer[Grimlore Games], rating[poor])",
47
- "inform(name[BioShock], release_year[2007], rating[good], genres[action-adventure, role-playing, shooter], platforms[PlayStation, Xbox, PC], available_on_steam[yes], has_linux_release[no], has_mac_release[yes])",
45
+ "give_opinion(name[SpellForce 3], release_year[2017], developer[Grimlore Games], rating[poor])", # noqa: E501
46
+ "give_opinion(name[SpellForce 3], release_year[2017], developer[Grimlore Games], rating[poor])", # noqa: E501
47
+ "inform(name[BioShock], release_year[2007], rating[good], genres[action-adventure, role-playing, shooter], platforms[PlayStation, Xbox, PC], available_on_steam[yes], has_linux_release[no], has_mac_release[yes])", # noqa: E501
48
]
49
50
assert do_sample(llm, mixtral_lora_files,
tests/metrics/test_metrics.py
+8
-6
@@ -21,7 +21,8 @@ def test_metric_counter_prompt_tokens(
21
gpu_memory_utilization=0.4)
22
tokenizer = vllm_model.model.get_tokenizer()
23
prompt_token_counts = [len(tokenizer.encode(p)) for p in example_prompts]
24
- # This test needs at least 2 prompts in a batch of different lengths to verify their token count is correct despite padding.
24
+ # This test needs at least 2 prompts in a batch of different lengths to
25
+ # verify their token count is correct despite padding.
26
assert len(example_prompts) > 1, "at least 2 prompts are required"
27
assert prompt_token_counts[0] != prompt_token_counts[1], (
28
"prompts of different lengths are required")
@@ -33,8 +34,8 @@ def test_metric_counter_prompt_tokens(
34
**stat_logger.labels)._value.get()
35
36
assert vllm_prompt_token_count == metric_count, (
36
- f"prompt token count: {vllm_prompt_token_count!r}\nmetric: {metric_count!r}"
37
- )
37
+ f"prompt token count: {vllm_prompt_token_count!r}\n"
38
+ f"metric: {metric_count!r}")
39
40
41
@pytest.mark.parametrize("model", MODELS)
@@ -60,9 +61,10 @@ def test_metric_counter_generation_tokens(
61
for i in range(len(example_prompts)):
62
vllm_output_ids, vllm_output_str = vllm_outputs[i]
63
prompt_ids = tokenizer.encode(example_prompts[i])
63
- # vllm_output_ids contains both prompt tokens and generation tokens. We're interested only in the count of the generation tokens.
64
+ # vllm_output_ids contains both prompt tokens and generation tokens.
65
+ # We're interested only in the count of the generation tokens.
66
vllm_generation_count += len(vllm_output_ids) - len(prompt_ids)
67
68
assert vllm_generation_count == metric_count, (
67
- f"generation token count: {vllm_generation_count!r}\nmetric: {metric_count!r}"
68
- )
69
+ f"generation token count: {vllm_generation_count!r}\n"
70
+ f"metric: {metric_count!r}")
tests/models/test_marlin.py
+8
-7
@@ -1,7 +1,7 @@
1
"""Compare the outputs of a GPTQ model to a Marlin model.
2
3
-Note: GPTQ and Marlin do not have bitwise correctness.
4
-As a result, in this test, we just confirm that the top selected tokens of the
3
+Note: GPTQ and Marlin do not have bitwise correctness.
4
+As a result, in this test, we just confirm that the top selected tokens of the
5
Marlin/GPTQ models are in the top 3 selections of each other.
6
7
Note: Marlin internally uses locks to synchronize the threads. This can
@@ -14,7 +14,8 @@ Run `pytest tests/models/test_marlin.py --forked`.
14
import pytest
15
import torch
16
from dataclasses import dataclass
17
-from vllm.model_executor.layers.quantization import _QUANTIZATION_CONFIG_REGISTRY
17
+from vllm.model_executor.layers.quantization import (
18
+ _QUANTIZATION_CONFIG_REGISTRY)
19
20
capability = torch.cuda.get_device_capability()
21
capability = capability[0] * 10 + capability[1]
@@ -87,11 +88,11 @@ def test_models(
88
if marlin_output_id != gptq_output_id:
89
# Each predicted token must be in top 5 of the other's
90
assert gptq_output_id in marlin_logprobs[idx], (
90
- f"Test{prompt_idx}:\nGPTQ:\t{gptq_output_str!r}\nMarlin:\t{marlin_output_str!r}"
91
- )
91
+ f"Test{prompt_idx}:\nGPTQ:\t{gptq_output_str!r}\n"
92
+ f"Marlin:\t{marlin_output_str!r}")
93
assert marlin_output_id in gptq_logprobs[idx], (
93
- f"Test{prompt_idx}:\nGPTQ:\t{gptq_output_str!r}\nMarlin:\t{marlin_output_str!r}"
94
- )
94
+ f"Test{prompt_idx}:\nGPTQ:\t{gptq_output_str!r}\n"
95
+ f"Marlin:\t{marlin_output_str!r}")
96
97
# Break out since sequences will now diverge.
98
break
tests/prefix_caching/test_prefix_caching.py
+10
-5
@@ -20,20 +20,23 @@ def test_block_allocator(
20
num_blocks,
21
enable_caching=True)
22
23
- # Allocate two PysicalTokenBlocks with the same hash and check that they are the same PhysicalTokenBlock
23
+ # Allocate two PysicalTokenBlocks with the same hash and check
24
+ # that they are the same PhysicalTokenBlock
25
first_block = block_allocator.allocate(block_hash, 0)
26
second_block = block_allocator.allocate(block_hash, 0)
27
assert (first_block == second_block)
28
assert (second_block.ref_count == 2)
29
29
- # Free the first_block and confirm that the ref_count is correctly decremented on the second block
30
+ # Free the first_block and confirm that the ref_count is correctly
31
+ # decremented on the second block
32
block_allocator.free(first_block)
33
assert (second_block.ref_count == 1)
34
35
# Free the second block
36
block_allocator.free(second_block)
37
36
- # Reallocate the first block and confirm that, even after the block had its ref_count go to 0, we still get the same block back
38
+ # Reallocate the first block and confirm that, even after the block
39
+ # had its ref_count go to 0, we still get the same block back
40
first_block = block_allocator.allocate(block_hash, 0)
41
assert (first_block == second_block)
42
assert (first_block.block_hash == block_hash)
@@ -56,7 +59,8 @@ def test_eviction(num_blocks: int, ):
59
for block in blocks:
60
block_allocator.free(block)
61
59
- # Allocate a new block and confirm that it's the first block freed. I.E The Least Recently Used block
62
+ # Allocate a new block and confirm that it's the first block freed.
63
+ # I.E The Least Recently Used block
64
new_block_hash = block_size
65
new_block = block_allocator.allocate(new_block_hash, 0)
66
assert (new_block == blocks[0])
@@ -68,7 +72,8 @@ def test_eviction(num_blocks: int, ):
72
assert (realloc_block == blocks[realloc_block_hash])
73
assert (realloc_block.block_hash == realloc_block_hash)
74
71
- # Allocate a new block and confirm that it's not the realloc_block, since the realloc_block shouldn't be in the free list
75
+ # Allocate a new block and confirm that it's not the realloc_block,
76
+ # since the realloc_block shouldn't be in the free list
77
new_block_hash = block_size + 1
78
new_block = block_allocator.allocate(new_block_hash, 0)
79
assert (realloc_block != new_block)
tests/samplers/test_logprobs.py
+2
-2
@@ -70,8 +70,8 @@ def test_get_prompt_logprobs(
70
hf_logprob[i][-1][token_id].item(),
71
atol=1e-2,
72
rtol=1e-2)
73
- assert isinstance(sample_logprob.decoded_token, str), \
74
- ("The token should be decoded by the time it is returned "
73
+ assert isinstance(sample_logprob.decoded_token, str), (
74
+ "The token should be decoded by the time it is returned "
75
" to the user.")
76
77
tests/samplers/test_sampler.py
+10
-7
@@ -255,9 +255,10 @@ def test_sampler_mixed(seed: int, device: str):
255
if metadata.sampling_params.use_beam_search:
256
continue
257
258
- if metadata.sampling_params.seed is not None \
259
- and expected_tokens[i] is None:
260
- # Record seeded random result to compare with results of second invocation
258
+ if (metadata.sampling_params.seed is not None
259
+ and expected_tokens[i] is None):
260
+ # Record seeded random result to compare with results of
261
+ # second invocation
262
expected_tokens[i] = [
263
nth_output.output_token
264
for nth_output in sequence_output.samples
@@ -265,11 +266,13 @@ def test_sampler_mixed(seed: int, device: str):
266
continue
267
268
for n, nth_output in enumerate(sequence_output.samples):
268
- if metadata.sampling_params.temperature == 0 or metadata.sampling_params.seed is not None:
269
+ if (metadata.sampling_params.temperature == 0
270
+ or metadata.sampling_params.seed is not None):
271
# Ensure exact matches for greedy or random with seed
272
assert nth_output.output_token == expected_tokens[i][n]
273
else:
272
- # For non-seeded random check that one of the high-logit tokens were chosen
274
+ # For non-seeded random check that one of the high-logit
275
+ # tokens were chosen
276
assert nth_output.output_token in expected_tokens[i]
277
278
# Test batch
@@ -284,8 +287,8 @@ def test_sampler_mixed(seed: int, device: str):
287
input_tensor.data = input_tensor.index_select(0, target_index)
288
fake_logits.data = fake_logits.index_select(0, target_index)
289
287
- # This time, results of seeded random samples will be compared with the corresponding
288
- # sample in the pre-shuffled batch
290
+ # This time, results of seeded random samples will be compared with
291
+ # the corresponding sample in the pre-shuffled batch
292
test_sampling(model_runner)
293
294
del model_runner
tests/spec_decode/test_metrics.py
+4
-2
@@ -150,8 +150,10 @@ def test_initial_metrics_has_correct_values(has_data: bool):
150
assert metrics.emitted_tokens == num_emitted_tokens
151
152
if has_data:
153
- assert metrics.draft_acceptance_rate == num_accepted_tokens / num_draft_tokens
154
- assert metrics.system_efficiency == num_emitted_tokens / num_possible_tokens
153
+ assert (metrics.draft_acceptance_rate == num_accepted_tokens /
154
+ num_draft_tokens)
155
+ assert (metrics.system_efficiency == num_emitted_tokens /
156
+ num_possible_tokens)
157
else:
158
assert math.isnan(metrics.draft_acceptance_rate)
159
assert math.isnan(metrics.system_efficiency)
tests/spec_decode/test_multi_step_worker.py
+2
-1
@@ -3,7 +3,8 @@ import random
3
import pytest
4
from unittest.mock import MagicMock
5
6
-from vllm.spec_decode.multi_step_worker import MultiStepWorker, DraftModelTop1Proposer
6
+from vllm.spec_decode.multi_step_worker import (MultiStepWorker,
7
+ DraftModelTop1Proposer)
8
from vllm.worker.worker import Worker
9
from vllm.model_executor.utils import set_random_seed
10
from vllm.sequence import SamplerOutput
tests/spec_decode/test_spec_decode_worker.py
+12
-6
@@ -4,12 +4,15 @@ import pytest
4
from unittest.mock import MagicMock
5
6
from vllm.spec_decode.multi_step_worker import MultiStepWorker
7
-from vllm.spec_decode.spec_decode_worker import SpecDecodeWorker, split_num_cache_blocks_evenly
7
+from vllm.spec_decode.spec_decode_worker import (SpecDecodeWorker,
8
+ split_num_cache_blocks_evenly)
9
from vllm.spec_decode.interfaces import SpeculativeProposals
10
from vllm.model_executor.utils import set_random_seed
11
from vllm.model_executor.layers.rejection_sampler import RejectionSampler
11
-from .utils import mock_worker, create_batch, ExecuteModelData, create_sampler_output_list
12
-from vllm.spec_decode.metrics import SpecDecodeWorkerMetrics, AsyncMetricsCollector
12
+from .utils import (mock_worker, create_batch, ExecuteModelData,
13
+ create_sampler_output_list)
14
+from vllm.spec_decode.metrics import (SpecDecodeWorkerMetrics,
15
+ AsyncMetricsCollector)
16
17
18
@pytest.mark.parametrize('k', [1, 2, 6])
@@ -391,13 +394,15 @@ def test_collects_metrics(k: int, batch_size: int, returns_metrics: bool):
394
395
mock_rejsample_metrics = MagicMock(
396
spec=SpecDecodeWorkerMetrics) if returns_metrics else None
394
- metrics_collector.maybe_collect_rejsample_metrics.return_value = mock_rejsample_metrics
397
+ metrics_collector.maybe_collect_rejsample_metrics.return_value = (
398
+ mock_rejsample_metrics)
399
400
output = worker.execute_model(**execute_model_data.to_dict(),
401
num_spec_tokens=k)
402
assert output[0].spec_decode_worker_metrics == mock_rejsample_metrics
403
400
- call_args_list = metrics_collector.maybe_collect_rejsample_metrics.call_args_list
404
+ call_args_list = (
405
+ metrics_collector.maybe_collect_rejsample_metrics.call_args_list)
406
assert len(call_args_list) == 1
407
args, kwargs = call_args_list[0]
408
assert args[0] == k or kwargs.get('k', -1) == k
@@ -547,7 +552,8 @@ def test_profile_num_available_blocks(available_gpu_blocks: int,
552
553
target_worker.profile_num_available_blocks.return_value = (
554
available_gpu_blocks, available_cpu_blocks)
550
- target_worker.get_cache_block_size_bytes.return_value = target_cache_block_size_bytes
555
+ target_worker.get_cache_block_size_bytes.return_value = (
556
+ target_cache_block_size_bytes)
557
draft_worker.get_cache_block_size_bytes.return_value = draft_kv_size_bytes
558
559
worker = SpecDecodeWorker(draft_worker, target_worker, rejection_sampler,
vllm/config.py
+8
-6
@@ -45,7 +45,7 @@ class ModelConfig:
45
a tag name, or a commit id. If unspecified, will use the default
46
version.
47
code_revision: The specific revision to use for the model code on
48
- Hugging Face Hub. It can be a branch name, a tag name, or a
48
+ Hugging Face Hub. It can be a branch name, a tag name, or a
49
commit id. If unspecified, will use the default version.
50
tokenizer_revision: The specific tokenizer version to use. It can be a
51
branch name, a tag name, or a commit id. If unspecified, will use
@@ -189,8 +189,8 @@ class ModelConfig:
189
if is_hip(
190
) and self.quantization in rocm_not_supported_quantization:
191
raise ValueError(
192
- f"{self.quantization} quantization is currently not supported "
193
- f"in ROCm.")
192
+ f"{self.quantization} quantization is currently not "
193
+ f"supported in ROCm.")
194
if self.quantization != "marlin":
195
logger.warning(
196
f"{self.quantization} quantization is not fully "
@@ -321,7 +321,8 @@ class CacheConfig:
321
self.num_cpu_blocks = None
322
323
def metrics_info(self):
324
- # convert cache_config to dict(key: str, value: str) for prometheus metrics info
324
+ # convert cache_config to dict(key: str, value: str) for prometheus
325
+ # metrics info
326
return {key: str(value) for key, value in self.__dict__.items()}
327
328
def _verify_args(self) -> None:
@@ -399,8 +400,9 @@ class ParallelConfig:
400
) -> None:
401
self.pipeline_parallel_size = pipeline_parallel_size
402
if is_neuron():
402
- # For Neuron device support, here we assign TP=1 to avoid sharding within vLLM directly.
403
- # Transformer-neuronx would take neuron_tp_degree attribute, and distribute the workload
403
+ # For Neuron device support, here we assign TP=1 to avoid sharding
404
+ # within vLLM directly. Transformer-neuronx would take
405
+ # neuron_tp_degree attribute, and distribute the workload
406
# to multiple NeuronCores.
407
self.tensor_parallel_size = 1
408
self.neuron_tp_degree = tensor_parallel_size
vllm/core/block_manager.py
+10
-5
@@ -95,13 +95,15 @@ class BlockAllocator:
95
del self.cached_blocks[block.block_hash]
96
97
def get_num_free_blocks(self) -> int:
98
- return self.num_blocks - self.current_num_blocks + self.evictor.num_blocks
98
+ return (self.num_blocks - self.current_num_blocks +
99
+ self.evictor.num_blocks)
100
101
def contains_block(self, block_hash: int) -> bool:
102
return block_hash in self.cached_blocks or block_hash in self.evictor
103
104
def update_hash(self, block_hash: int, block: PhysicalTokenBlock):
104
- # If caching is enabled, update the hash of block and the cached_blocks dictionary.
105
+ # If caching is enabled, update the hash of block and the
106
+ # cached_blocks dictionary.
107
if self.enable_caching:
108
assert not self.contains_block(block_hash)
109
old_hash = block.block_hash
@@ -218,10 +220,12 @@ class BlockSpaceManager:
220
seq: Sequence,
221
last_block: PhysicalTokenBlock,
222
) -> PhysicalTokenBlock:
221
- # Compute a new hash for the block so that it can be shared by other Sequences
223
+ # Compute a new hash for the block so that it can be shared by
224
+ # other Sequences
225
new_hash = seq.hash_of_block(len(seq.logical_token_blocks) - 1)
226
224
- # if new_hash is already in the cached table, then free last_block and return the cached version
227
+ # if new_hash is already in the cached table, then free last_block
228
+ # and return the cached version
229
if self.gpu_allocator.contains_block(new_hash):
230
self.gpu_allocator.free(last_block)
231
return self.gpu_allocator.allocate(new_hash)
@@ -289,7 +293,8 @@ class BlockSpaceManager:
293
assert last_block.device == Device.GPU
294
if last_block.ref_count == 1:
295
# Not shared with other sequences. Appendable.
292
- # If the last block is now complete, promote it to a full block so that it can be shared
296
+ # If the last block is now complete, promote it to a full block so
297
+ # that it can be shared
298
new_block = self._maybe_promote_last_block(seq, last_block)
299
block_table[-1] = new_block
300
return None
vllm/core/evictor.py
+3
-3
@@ -39,9 +39,9 @@ class Evictor(ABC):
39
@abstractmethod
40
def remove(self, block_hash: int) -> PhysicalTokenBlock:
41
"""Simply removes the block with the hash value block_hash from the
42
- evictor. Caller is responsible for making sure that block_hash is contained
43
- in the evictor before calling remove. Should be used to "bring back" blocks
44
- that have been freed but not evicted yet.
42
+ evictor. Caller is responsible for making sure that block_hash is
43
+ contained in the evictor before calling remove. Should be used to
44
+ "bring back" blocks that have been freed but not evicted yet.
45
"""
46
pass
47
vllm/core/scheduler.py
+4
-4
@@ -214,8 +214,8 @@ class Scheduler:
214
lora_int_id = 0
215
if self.lora_enabled:
216
lora_int_id = seq_group.lora_int_id
217
- if lora_int_id > 0 and lora_int_id not in curr_loras and len(
218
- curr_loras) >= self.lora_config.max_loras:
217
+ if (lora_int_id > 0 and lora_int_id not in curr_loras
218
+ and len(curr_loras) >= self.lora_config.max_loras):
219
# We don't have a space for another LoRA, so
220
# we ignore this request for now.
221
leftover_waiting_sequences.appendleft(seq_group)
@@ -309,8 +309,8 @@ class Scheduler:
309
lora_int_id = 0
310
if self.lora_enabled:
311
lora_int_id = seq_group.lora_int_id
312
- if lora_int_id > 0 and lora_int_id not in curr_loras and len(
313
- curr_loras) >= self.lora_config.max_loras:
312
+ if (lora_int_id > 0 and lora_int_id not in curr_loras
313
+ and len(curr_loras) >= self.lora_config.max_loras):
314
# We don't have a space for another LoRA, so
315
# we ignore this request for now.
316
leftover_swapped.appendleft(seq_group)
vllm/engine/llm_engine.py
+15
-12
@@ -100,7 +100,8 @@ class LLMEngine:
100
f"download_dir={model_config.download_dir!r}, "
101
f"load_format={model_config.load_format}, "
102
f"tensor_parallel_size={parallel_config.tensor_parallel_size}, "
103
- f"disable_custom_all_reduce={parallel_config.disable_custom_all_reduce}, "
103
+ f"disable_custom_all_reduce="
104
+ f"{parallel_config.disable_custom_all_reduce}, "
105
f"quantization={model_config.quantization}, "
106
f"enforce_eager={model_config.enforce_eager}, "
107
f"kv_cache_dtype={cache_config.cache_dtype}, "
@@ -929,7 +930,8 @@ class LLMEngine:
930
# Latency Timings.
931
time_last_iters = []
932
for seq_group in scheduler_outputs.scheduled_seq_groups:
932
- # Time since last token. (n.b. updates seq_group.metrics.last_token_time)
933
+ # Time since last token.
934
+ # (n.b. updates seq_group.metrics.last_token_time)
935
time_last_iters.append(seq_group.get_last_latency(now))
936
# Time since arrival for all finished requests.
937
if seq_group.is_finished():
@@ -961,16 +963,17 @@ class LLMEngine:
963
for token_id, sample_logprob in logprobs.items():
964
if (sample_logprob.decoded_token is None and token_id != -1):
965
all_input_ids_with_logprob = all_input_ids[:-1] + [token_id]
964
- _, new_text, prefix_offset, read_offset = detokenize_incrementally(
965
- self.get_tokenizer_for_seq(seq),
966
- all_input_ids=all_input_ids_with_logprob,
967
- prev_tokens=seq.tokens,
968
- prefix_offset=seq.prefix_offset,
969
- read_offset=seq.read_offset,
970
- skip_special_tokens=prms.skip_special_tokens,
971
- spaces_between_special_tokens=prms.
972
- spaces_between_special_tokens,
973
- )
966
+ (_, new_text, prefix_offset,
967
+ read_offset) = detokenize_incrementally(
968
+ self.get_tokenizer_for_seq(seq),
969
+ all_input_ids=all_input_ids_with_logprob,
970
+ prev_tokens=seq.tokens,
971
+ prefix_offset=seq.prefix_offset,
972
+ read_offset=seq.read_offset,
973
+ skip_special_tokens=prms.skip_special_tokens,
974
+ spaces_between_special_tokens=prms.
975
+ spaces_between_special_tokens,
976
+ )
977
sample_logprob.decoded_token = new_text
978
979
def _decode_sequence(self, seq: Sequence, prms: SamplingParams) -> None:
vllm/engine/metrics.py
+13
-9
@@ -1,5 +1,6 @@
1
from vllm.logger import init_logger
2
-from prometheus_client import Counter, Gauge, Histogram, Info, REGISTRY, disable_created_metrics
2
+from prometheus_client import (Counter, Gauge, Histogram, Info, REGISTRY,
3
+ disable_created_metrics)
4
5
import time
6
import numpy as np
@@ -177,10 +178,12 @@ class StatLogger:
178
def _log_prometheus_interval(self, prompt_throughput: float,
179
generation_throughput: float) -> None:
180
# Logs metrics to prometheus that are computed every logging_interval.
180
- # Support legacy gauge metrics that make throughput calculations on the vLLM side.
181
- # Moving forward, we should use counters like counter_prompt_tokens, counter_generation_tokens
182
- # Which log raw data and calculate summaries using rate() on the grafana/prometheus side.
183
- # See https://github.com/vllm-project/vllm/pull/2316#discussion_r1464204666
181
+ # Support legacy gauge metrics that make throughput calculations on
182
+ # the vLLM side. Moving forward, we should use counters like
183
+ # counter_prompt_tokens, counter_generation_tokens
184
+ # Which log raw data and calculate summaries using rate() on the
185
+ # grafana/prometheus side. See
186
+ # https://github.com/vllm-project/vllm/pull/2316#discussion_r1464204666
187
self.metrics.gauge_avg_prompt_throughput.labels(
188
**self.labels).set(prompt_throughput)
189
self.metrics.gauge_avg_generation_throughput.labels(
@@ -188,7 +191,7 @@ class StatLogger:
191
192
def log(self, stats: Stats) -> None:
193
"""Called by LLMEngine.
191
- Logs to prometheus and tracked stats every iteration.
194
+ Logs to prometheus and tracked stats every iteration.
195
Logs to Stdout every self.local_interval seconds."""
196
197
# Log to prometheus.
@@ -200,8 +203,8 @@ class StatLogger:
203
204
# Log locally every local_interval seconds.
205
if self._local_interval_elapsed(stats.now):
203
-
204
- # Compute summary metrics for tracked stats (and log them to promethus if applicable).
206
+ # Compute summary metrics for tracked stats (and log them
207
+ # to promethus if applicable).
208
prompt_throughput = self._get_throughput(self.num_prompt_tokens,
209
now=stats.now)
210
generation_throughput = self._get_throughput(
@@ -213,7 +216,8 @@ class StatLogger:
216
# Log to stdout.
217
logger.info(
218
f"Avg prompt throughput: {prompt_throughput:.1f} tokens/s, "
216
- f"Avg generation throughput: {generation_throughput:.1f} tokens/s, "
219
+ f"Avg generation throughput: "
220
+ f"{generation_throughput:.1f} tokens/s, "
221
f"Running: {stats.num_running} reqs, "
222
f"Swapped: {stats.num_swapped} reqs, "
223
f"Pending: {stats.num_waiting} reqs, "
vllm/entrypoints/api_server.py
+5
-3
@@ -1,7 +1,9 @@
1
"""
2
-NOTE: This API server is used only for demonstrating usage of AsyncEngine and simple performance benchmarks.
3
-It is not intended for production use. For production use, we recommend using our OpenAI compatible server.
4
-We are also not going to accept PRs modifying this file, please change `vllm/entrypoints/openai/api_server.py` instead.
2
+NOTE: This API server is used only for demonstrating usage of AsyncEngine
3
+and simple performance benchmarks. It is not intended for production use.
4
+For production use, we recommend using our OpenAI compatible server.
5
+We are also not going to accept PRs modifying this file, please
6
+change `vllm/entrypoints/openai/api_server.py` instead.
7
"""
8
9
import argparse
vllm/entrypoints/openai/api_server.py
+16
-17
@@ -18,7 +18,9 @@ from fastapi.responses import JSONResponse, StreamingResponse, Response
18
import vllm
19
from vllm.engine.arg_utils import AsyncEngineArgs
20
from vllm.engine.async_llm_engine import AsyncLLMEngine
21
-from vllm.entrypoints.openai.protocol import CompletionRequest, ChatCompletionRequest, ErrorResponse
21
+from vllm.entrypoints.openai.protocol import (CompletionRequest,
22
+ ChatCompletionRequest,
23
+ ErrorResponse)
24
from vllm.logger import init_logger
25
from vllm.entrypoints.openai.serving_chat import OpenAIServingChat
26
from vllm.entrypoints.openai.serving_completion import OpenAIServingCompletion
@@ -84,13 +86,11 @@ def parse_args():
86
type=json.loads,
87
default=["*"],
88
help="allowed headers")
87
- parser.add_argument(
88
- "--api-key",
89
- type=str,
90
- default=None,
91
- help=
92
- "If provided, the server will require this key to be presented in the header."
93
- )
89
+ parser.add_argument("--api-key",
90
+ type=str,
91
+ default=None,
92
+ help="If provided, the server will require this key "
93
+ "to be presented in the header.")
94
parser.add_argument("--served-model-name",
95
type=str,
96
default=None,
@@ -103,9 +103,8 @@ def parse_args():
103
default=None,
104
nargs='+',
105
action=LoRAParserAction,
106
- help=
107
- "LoRA module configurations in the format name=path. Multiple modules can be specified."
108
- )
106
+ help="LoRA module configurations in the format name=path. "
107
+ "Multiple modules can be specified.")
108
parser.add_argument("--chat-template",
109
type=str,
110
default=None,
@@ -138,9 +137,10 @@ def parse_args():
137
help="Additional ASGI middleware to apply to the app. "
138
"We accept multiple --middleware arguments. "
139
"The value should be an import path. "
141
- "If a function is provided, vLLM will add it to the server using @app.middleware('http'). "
142
- "If a class is provided, vLLM will add it to the server using app.add_middleware(). "
143
- )
140
+ "If a function is provided, vLLM will add it to the server "
141
+ "using @app.middleware('http'). "
142
+ "If a class is provided, vLLM will add it to the server "
143
+ "using app.add_middleware(). ")
144
145
parser = AsyncEngineArgs.add_cli_args(parser)
146
return parser.parse_args()
@@ -235,9 +235,8 @@ if __name__ == "__main__":
235
elif inspect.iscoroutinefunction(imported):
236
app.middleware("http")(imported)
237
else:
238
- raise ValueError(
239
- f"Invalid middleware {middleware}. Must be a function or a class."
240
- )
238
+ raise ValueError(f"Invalid middleware {middleware}. "
239
+ f"Must be a function or a class.")
240
241
logger.info(f"vLLM API server version {vllm.__version__}")
242
logger.info(f"args: {args}")
vllm/entrypoints/openai/serving_chat.py
+15
-10
@@ -12,7 +12,8 @@ from vllm.entrypoints.openai.protocol import (
12
UsageInfo)
13
from vllm.outputs import RequestOutput
14
from vllm.entrypoints.openai.serving_engine import OpenAIServing, LoRA
15
-from vllm.model_executor.guided_decoding import get_guided_decoding_logits_processor
15
+from vllm.model_executor.guided_decoding import (
16
+ get_guided_decoding_logits_processor)
17
18
logger = init_logger(__name__)
19
@@ -37,8 +38,9 @@ class OpenAIServingChat(OpenAIServing):
38
ChatCompletionResponse]:
39
"""Completion API similar to OpenAI's API.
40
40
- See https://platform.openai.com/docs/api-reference/chat/create
41
- for the API specification. This API mimics the OpenAI ChatCompletion API.
41
+ See https://platform.openai.com/docs/api-reference/chat/create
42
+ for the API specification. This API mimics the OpenAI
43
+ ChatCompletion API.
44
45
NOTE: Currently we do not support the following feature:
46
- function_call (Users should implement this by themselves)
@@ -116,7 +118,8 @@ class OpenAIServingChat(OpenAIServing):
118
# the result_generator, it needs to be sent as the FIRST
119
# response (by the try...catch).
120
if first_iteration:
119
- # Send first response for each request.n (index) with the role
121
+ # Send first response for each request.n (index) with
122
+ # the role
123
role = self.get_chat_request_role(request)
124
for i in range(request.n):
125
choice_data = ChatCompletionResponseStreamChoice(
@@ -133,7 +136,8 @@ class OpenAIServingChat(OpenAIServing):
136
data = chunk.model_dump_json(exclude_unset=True)
137
yield f"data: {data}\n\n"
138
136
- # Send response to echo the input portion of the last message
139
+ # Send response to echo the input portion of the
140
+ # last message
141
if request.echo:
142
last_msg_content = ""
143
if request.messages and isinstance(
@@ -145,11 +149,12 @@ class OpenAIServingChat(OpenAIServing):
149
150
if last_msg_content:
151
for i in range(request.n):
148
- choice_data = ChatCompletionResponseStreamChoice(
149
- index=i,
150
- delta=DeltaMessage(
151
- content=last_msg_content),
152
- finish_reason=None)
152
+ choice_data = (
153
+ ChatCompletionResponseStreamChoice(
154
+ index=i,
155
+ delta=DeltaMessage(
156
+ content=last_msg_content),
157
+ finish_reason=None))
158
chunk = ChatCompletionStreamResponse(
159
id=request_id,
160
object=chunk_object_type,
vllm/entrypoints/openai/serving_completion.py
+17
-11
@@ -1,7 +1,8 @@
1
import asyncio
2
import time
3
from fastapi import Request
4
-from typing import AsyncGenerator, AsyncIterator, Callable, List, Optional, Dict, Tuple
4
+from typing import (AsyncGenerator, AsyncIterator, Callable, List, Optional,
5
+ Dict, Tuple)
6
from vllm.logger import init_logger
7
from vllm.utils import random_uuid
8
from vllm.engine.async_llm_engine import AsyncLLMEngine
@@ -16,7 +17,8 @@ from vllm.entrypoints.openai.protocol import (
17
)
18
from vllm.outputs import RequestOutput
19
from vllm.entrypoints.openai.serving_engine import OpenAIServing, LoRA
19
-from vllm.model_executor.guided_decoding import get_guided_decoding_logits_processor
20
+from vllm.model_executor.guided_decoding import (
21
+ get_guided_decoding_logits_processor)
22
23
logger = init_logger(__name__)
24
@@ -44,9 +46,8 @@ def parse_prompt_format(prompt) -> Tuple[bool, list]:
46
prompt_is_tokens = True
47
prompts = prompt # case 4: array of token arrays
48
else:
47
- raise ValueError(
48
- "prompt must be a string, array of strings, array of tokens, or array of token arrays"
49
- )
49
+ raise ValueError("prompt must be a string, array of strings, "
50
+ "array of tokens, or array of token arrays")
51
return prompt_is_tokens, prompts
52
53
@@ -156,7 +157,8 @@ class OpenAIServingCompletion(OpenAIServing):
157
int, RequestOutput]] = merge_async_iterators(*generators)
158
159
# Similar to the OpenAI API, when n != best_of, we do not stream the
159
- # results. In addition, we do not stream the results when use beam search.
160
+ # results. In addition, we do not stream the results when use
161
+ # beam search.
162
stream = (request.stream
163
and (request.best_of is None or request.n == request.best_of)
164
and not request.use_beam_search)
@@ -223,7 +225,8 @@ class OpenAIServingCompletion(OpenAIServing):
225
226
for output in res.outputs:
227
i = output.index + prompt_idx * request.n
226
- # TODO(simon): optimize the performance by avoiding full text O(n^2) sending.
228
+ # TODO(simon): optimize the performance by avoiding full
229
+ # text O(n^2) sending.
230
231
if request.echo and request.max_tokens == 0:
232
# only return the prompt
@@ -231,11 +234,12 @@ class OpenAIServingCompletion(OpenAIServing):
234
delta_token_ids = res.prompt_token_ids
235
top_logprobs = res.prompt_logprobs
236
has_echoed[i] = True
234
- elif request.echo and request.max_tokens > 0 and not has_echoed[
235
- i]:
237
+ elif (request.echo and request.max_tokens > 0
238
+ and not has_echoed[i]):
239
# echo the prompt and first token
240
delta_text = res.prompt + output.text
238
- delta_token_ids = res.prompt_token_ids + output.token_ids
241
+ delta_token_ids = (res.prompt_token_ids +
242
+ output.token_ids)
243
top_logprobs = res.prompt_logprobs + (output.logprobs
244
or [])
245
has_echoed[i] = True
@@ -248,7 +252,9 @@ class OpenAIServingCompletion(OpenAIServing):
252
i]:] if output.logprobs else None
253
254
if request.logprobs is not None:
251
- assert top_logprobs is not None, "top_logprobs must be provided when logprobs is requested"
255
+ assert top_logprobs is not None, (
256
+ "top_logprobs must be provided when logprobs "
257
+ "is requested")
258
logprobs = self._create_logprobs(
259
token_ids=delta_token_ids,
260
top_logprobs=top_logprobs,
vllm/entrypoints/openai/serving_engine.py
+8
-5
@@ -50,10 +50,12 @@ class OpenAIServing:
50
except RuntimeError:
51
event_loop = None
52
53
- if event_loop is not None and event_loop.is_running(
54
- ): # If the current is instanced by Ray Serve, there is already a running event loop
53
+ if event_loop is not None and event_loop.is_running():
54
+ # If the current is instanced by Ray Serve,
55
+ # there is already a running event loop
56
event_loop.create_task(self._post_init())
56
- else: # When using single vLLM without engine_use_ray
57
+ else:
58
+ # When using single vLLM without engine_use_ray
59
asyncio.run(self._post_init())
60
61
async def _post_init(self):
@@ -178,8 +180,9 @@ class OpenAIServing:
180
181
if token_num + request.max_tokens > self.max_model_len:
182
raise ValueError(
181
- f"This model's maximum context length is {self.max_model_len} tokens. "
182
- f"However, you requested {request.max_tokens + token_num} tokens "
183
+ f"This model's maximum context length is "
184
+ f"{self.max_model_len} tokens. However, you requested "
185
+ f"{request.max_tokens + token_num} tokens "
186
f"({token_num} in the messages, "
187
f"{request.max_tokens} in the completion). "
188
f"Please reduce the length of the messages or completion.", )
vllm/lora/layers.py
+8
-6
@@ -20,10 +20,12 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
20
RowParallelLinear,
21
QKVParallelLinear,
22
MergedColumnParallelLinear)
23
-from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding, ParallelLMHead
23
+from vllm.model_executor.layers.vocab_parallel_embedding import (
24
+ VocabParallelEmbedding, ParallelLMHead)
25
from vllm.model_executor.parallel_utils.parallel_state import (
26
get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
26
-from vllm.model_executor.parallel_utils.utils import split_tensor_along_last_dim
27
+from vllm.model_executor.parallel_utils.utils import (
28
+ split_tensor_along_last_dim)
29
30
if TYPE_CHECKING:
31
pass
@@ -84,7 +86,8 @@ def _apply_lora_packed_nslice(
86
lora_b_stacked: 3 element tuple of (num_loras, output_dim, lora_rank)
87
indices: (batch_size)
88
output: (batch_size, q_slice_size + 2*kv_slice_size)
87
- output_slices: n-1 element tuple of (slice_size...), where n is number of slices
89
+ output_slices: n-1 element tuple of (slice_size...),
90
+ where n is number of slices
91
"""
92
org_output = output
93
x = x.view(-1, x.shape[-1])
@@ -819,9 +822,8 @@ class SamplerWithLoRA(BaseLayerWithLoRA):
822
) -> None:
823
# Keep this in sync with csrc/punica/bgmv/bgmv_config.h
824
if 32000 < self.base_layer.vocab_size > 33024:
822
- raise ValueError(
823
- "When using LoRA, vocab size must be 32000 >= vocab_size <= 33024"
824
- )
825
+ raise ValueError("When using LoRA, vocab size must be "
826
+ "32000 >= vocab_size <= 33024")
827
self.lora_a_stacked = torch.zeros(
828
(
829
max_loras,
vllm/lora/models.py
+2
-1
@@ -13,7 +13,8 @@ from torch import nn
13
from vllm.config import LoRAConfig
14
from vllm.utils import LRUCache, in_wsl
15
16
-from vllm.lora.layers import BaseLayerWithLoRA, LoRAMapping, from_layer, from_layer_sampler
16
+from vllm.lora.layers import (BaseLayerWithLoRA, LoRAMapping, from_layer,
17
+ from_layer_sampler)
18
from vllm.lora.lora import LoRALayerWeights, PackedLoRALayerWeights
19
from vllm.lora.utils import parse_fine_tuned_lora_name, replace_submodule
20
vllm/lora/worker_manager.py
+3
-4
@@ -154,10 +154,9 @@ class WorkerLoRAManager(AbstractWorkerLoRAManager):
154
f"LoRA rank {lora.rank} is greater than max_lora_rank "
155
f"{self.lora_config.max_lora_rank}.")
156
if lora.extra_vocab_size > self.lora_config.lora_extra_vocab_size:
157
- raise ValueError(
158
- f"LoRA added vocab size {lora.extra_vocab_size} is greater than "
159
- f"lora_extra_vocab_size {self.lora_config.lora_extra_vocab_size}."
160
- )
157
+ raise ValueError(f"LoRA added vocab size {lora.extra_vocab_size} "
158
+ f"is greater than lora_extra_vocab_size "
159
+ f"{self.lora_config.lora_extra_vocab_size}.")
160
return lora
161
162
def add_dummy_lora(self, lora_request: LoRARequest, rank: int) -> bool:
vllm/model_executor/guided_decoding.py
+4
-2
@@ -8,8 +8,10 @@ from re import escape as regex_escape
8
from typing import Union, Tuple
9
from pydantic import BaseModel
10
11
-from vllm.entrypoints.openai.protocol import CompletionRequest, ChatCompletionRequest
12
-from vllm.model_executor.guided_logits_processors import JSONLogitsProcessor, RegexLogitsProcessor
11
+from vllm.entrypoints.openai.protocol import (CompletionRequest,
12
+ ChatCompletionRequest)
13
+from vllm.model_executor.guided_logits_processors import (JSONLogitsProcessor,
14
+ RegexLogitsProcessor)
15
16
17
class GuidedDecodingMode(Enum):
vllm/model_executor/guided_logits_processors.py
+9
-6
@@ -107,12 +107,15 @@ class JSONLogitsProcessor(RegexLogitsProcessor):
107
Parameters
108
----------
109
schema
110
- A JSON schema that encodes the structure we want the model to generate
110
+ A JSON schema that encodes the structure we want the model to
111
+ generate
112
tokenizer
113
The model's tokenizer
114
whitespace_pattern
114
- Pattern to use for JSON syntactic whitespace (doesn't impact string literals)
115
- Example: allow only a single space or newline with `whitespace_pattern=r"[\n ]?"`
115
+ Pattern to use for JSON syntactic whitespace (doesn't impact
116
+ string literals)
117
+ Example: allow only a single space or newline with
118
+ `whitespace_pattern=r"[\n ]?"`
119
"""
120
if isinstance(schema, type(BaseModel)):
121
schema_str = json.dumps(schema.model_json_schema())
@@ -122,8 +125,8 @@ class JSONLogitsProcessor(RegexLogitsProcessor):
125
schema_str = schema
126
else:
127
raise ValueError(
125
- f"Cannot parse schema {schema}. The schema must be either " +
126
- "a Pydantic object, a dictionary or a string that contains the JSON "
127
- + "Schema specification")
128
+ f"Cannot parse schema {schema}. The schema must be either "
129
+ f"a Pydantic object, a dictionary or a string that contains "
130
+ f"the JSON Schema specification")
131
regex_string = build_regex_from_schema(schema_str, whitespace_pattern)
132
super().__init__(regex_string, tokenizer)
vllm/model_executor/layers/attention/attention.py
+2
-2
@@ -35,12 +35,12 @@ class Attention(nn.Module):
35
) -> None:
36
super().__init__()
37
if _use_flash_attn():
38
- from vllm.model_executor.layers.attention.backends.flash_attn import FlashAttentionBackend
38
+ from vllm.model_executor.layers.attention.backends.flash_attn import FlashAttentionBackend # noqa: E501
39
self.backend = FlashAttentionBackend(num_heads, head_size, scale,
40
num_kv_heads, alibi_slopes,
41
sliding_window)
42
else:
43
- from vllm.model_executor.layers.attention.backends.xformers import XFormersBackend
43
+ from vllm.model_executor.layers.attention.backends.xformers import XFormersBackend # noqa: E501
44
self.backend = XFormersBackend(num_heads, head_size, scale,
45
num_kv_heads, alibi_slopes,
46
sliding_window)
vllm/model_executor/layers/fused_moe/fused_moe.py
+70
-37
@@ -30,9 +30,10 @@ def fused_moe_kernel(
30
K,
31
EM,
32
num_valid_tokens,
33
- # The stride variables represent how much to increase the ptr by when moving by 1
34
- # element in a particular dimension. E.g. `stride_am` is how much to increase `a_ptr`
35
- # by to get the element one row down (A has M rows).
33
+ # The stride variables represent how much to increase the ptr by when
34
+ # moving by 1 element in a particular dimension. E.g. `stride_am` is
35
+ # how much to increase `a_ptr` by to get the element one row down
36
+ # (A has M rows).
37
stride_am,
38
stride_ak,
39
stride_be,
@@ -50,17 +51,30 @@ def fused_moe_kernel(
51
compute_type: tl.constexpr,
52
):
53
"""
53
- Implements the fused computation for a Mixture of Experts (MOE) using token and expert matrices.
54
+ Implements the fused computation for a Mixture of Experts (MOE) using
55
+ token and expert matrices.
56
57
Key Parameters:
56
- - A: The input tensor representing tokens with shape (*, K), where '*' can be any shape representing batches and K is the feature dimension of each token.
57
- - B: The stacked MOE weight tensor with shape (E, N, K), where E is the number of experts, K is the input feature dimension, and N is the output feature dimension.
58
- - C: The output cache tensor with shape (M, topk, N), where M is the total number of tokens post padding, topk is the number of times each token is repeated,
59
- and N is the output feature dimension.
60
- - sorted_token_ids: A tensor containing the sorted indices of tokens, repeated topk times and arranged by the expert index they are assigned to.
61
- - expert_ids: A tensor containing the indices of the expert for each block. It determines which expert matrix from B should be used for each block in A.
62
- This kernel performs the multiplication of a token by its corresponding expert matrix as determined by `expert_ids`. The sorting of `sorted_token_ids`
63
- by expert index and padding ensures divisibility by BLOCK_SIZE_M, which is necessary to maintain consistency in block matrix multiplication across different blocks processed by the same expert.
58
+ - A: The input tensor representing tokens with shape (*, K), where '*' can
59
+ be any shape representing batches and K is the feature dimension of
60
+ each token.
61
+ - B: The stacked MOE weight tensor with shape (E, N, K), where E is
62
+ the number of experts, K is the input feature dimension, and N is
63
+ the output feature dimension.
64
+ - C: The output cache tensor with shape (M, topk, N), where M is the
65
+ total number of tokens post padding, topk is the number of times
66
+ each token is repeated, and N is the output feature dimension.
67
+ - sorted_token_ids: A tensor containing the sorted indices of tokens,
68
+ repeated topk times and arranged by the expert index they are
69
+ assigned to.
70
+ - expert_ids: A tensor containing the indices of the expert for each
71
+ block. It determines which expert matrix from B should be used for
72
+ each block in A.
73
+ This kernel performs the multiplication of a token by its corresponding
74
+ expert matrix as determined by `expert_ids`. The sorting of
75
+ `sorted_token_ids` by expert index and padding ensures divisibility by
76
+ BLOCK_SIZE_M, which is necessary to maintain consistency in block matrix
77
+ multiplication across different blocks processed by the same expert.
78
"""
79
# -----------------------------------------------------------
80
# Map program ids `pid` to the block of C it should compute.
@@ -105,7 +119,8 @@ def fused_moe_kernel(
119
accumulator = tl.zeros((BLOCK_SIZE_M, BLOCK_SIZE_N), dtype=tl.float32)
120
121
for k in range(0, tl.cdiv(K, BLOCK_SIZE_K)):
108
- # Load the next block of A and B, generate a mask by checking the K dimension.
122
+ # Load the next block of A and B, generate a mask by checking the
123
+ # K dimension.
124
a = tl.load(a_ptrs,
125
mask=token_mask[:, None] &
126
(offs_k[None, :] < K - k * BLOCK_SIZE_K),
@@ -139,30 +154,41 @@ def moe_align_block_size(
154
topk_ids: torch.Tensor, block_size: int,
155
num_experts: int) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
156
"""
142
- Aligns the token distribution across experts to be compatible with block size for matrix multiplication.
157
+ Aligns the token distribution across experts to be compatible with block
158
+ size for matrix multiplication.
159
160
Parameters:
145
- - topk_ids: A tensor of shape [total_tokens, top_k] representing the top-k expert indices for each token.
161
+ - topk_ids: A tensor of shape [total_tokens, top_k] representing the
162
+ top-k expert indices for each token.
163
- block_size: The block size used in block matrix multiplication.
164
- num_experts: The total number of experts.
165
166
Returns:
150
- - sorted_token_ids: A tensor containing the sorted token indices according to their allocated expert.
167
+ - sorted_token_ids: A tensor containing the sorted token indices according
168
+ to their allocated expert.
169
- expert_ids: A tensor indicating the assigned expert index for each block.
152
- - num_tokens_post_padded: The total number of tokens after padding, ensuring divisibility by block_size.
170
+ - num_tokens_post_padded: The total number of tokens after padding,
171
+ ensuring divisibility by block_size.
172
154
- This function pads the number of tokens that each expert needs to process so that it is divisible by block_size.
155
- Padding ensures that during block matrix multiplication, the dimensions align correctly.
173
+ This function pads the number of tokens that each expert needs to process
174
+ so that it is divisible by block_size.
175
+ Padding ensures that during block matrix multiplication, the dimensions
176
+ align correctly.
177
178
Example:
158
- Given topk_ids = [[2, 3, 4], [1, 2, 4], [1, 3, 4], [1, 2, 3]], block_size = 4, and num_experts = 4:
159
- - We initially have 12 tokens (after repeating 'top_k' times) and 4 experts, with each expert needing to process 3 tokens.
179
+ Given topk_ids = [[2, 3, 4], [1, 2, 4], [1, 3, 4], [1, 2, 3]],
180
+ block_size = 4, and num_experts = 4:
181
+ - We initially have 12 tokens (after repeating 'top_k' times) and 4 experts,
182
+ with each expert needing to process 3 tokens.
183
- As block_size is 4, we pad 1 token for each expert.
184
- First, flatten topk_ids to [2, 3, 4, 1, 2, 4, 1, 3, 4, 1, 2, 3].
185
- Then append padding tokens [12, 12, 12, 12] for each block.
163
- - After sorting by expert index, we obtain token_ids [3, 6, 9, 12, 0, 4, 10, 12, 1, 7, 11, 12, 2, 5, 8, 12].
164
- Tokens 12 are non-existent (padding) and are ignored in the subsequent matrix multiplication.
165
- - The padding ensures that the total number of tokens is now divisible by block_size for proper block matrix operations.
186
+ - After sorting by expert index, we obtain token_ids
187
+ [3, 6, 9, 12, 0, 4, 10, 12, 1, 7, 11, 12, 2, 5, 8, 12].
188
+ Tokens 12 are non-existent (padding) and are ignored in
189
+ the subsequent matrix multiplication.
190
+ - The padding ensures that the total number of tokens is now divisible
191
+ by block_size for proper block matrix operations.
192
"""
193
sorted_ids = torch.empty(
194
(topk_ids.numel() + num_experts * (block_size - 1), ),
@@ -224,13 +250,14 @@ def get_moe_configs(E: int, N: int) -> Optional[Dict[int, Any]]:
250
"""
251
Return optimized configurations for the fused MoE kernel.
252
227
- The return value will be a dictionary that maps an irregular grid of batch sizes
228
- to configurations of the fused_moe kernel. To evaluate the kernel on a given batch
229
- size bs, the closest batch size in the grid should be picked and the associated
230
- configuration chosen to invoke the kernel.
253
+ The return value will be a dictionary that maps an irregular grid of
254
+ batch sizes to configurations of the fused_moe kernel. To evaluate the
255
+ kernel on a given batch size bs, the closest batch size in the grid should
256
+ be picked and the associated configuration chosen to invoke the kernel.
257
"""
258
233
- # First look up if an optimized configuration is available in the configs directory
259
+ # First look up if an optimized configuration is available in the configs
260
+ # directory
261
device_name = torch.cuda.get_device_name().replace(" ", "_")
262
263
config_file_path = os.path.join(
@@ -243,7 +270,8 @@ def get_moe_configs(E: int, N: int) -> Optional[Dict[int, Any]]:
270
# If a configuration has been found, return it
271
return {int(key): val for key, val in json.load(f).items()}
272
246
- # If no optimized configuration is available, we will use the default configuration
273
+ # If no optimized configuration is available, we will use the default
274
+ # configuration
275
return None
276
277
@@ -258,18 +286,22 @@ def fused_moe(
286
override_config: Optional[Dict[str, Any]] = None,
287
) -> torch.Tensor:
288
"""
261
- This function computes a Mixture of Experts (MoE) layer using two sets of weights, w1 and w2, and top-k gating mechanism.
262
-
289
+ This function computes a Mixture of Experts (MoE) layer using two sets of
290
+ weights, w1 and w2, and top-k gating mechanism.
291
+
292
Parameters:
293
- hidden_states (torch.Tensor): The input tensor to the MoE layer.
294
- w1 (torch.Tensor): The first set of expert weights.
295
- w2 (torch.Tensor): The second set of expert weights.
267
- - gating_output (torch.Tensor): The output of the gating operation (before softmax).
296
+ - gating_output (torch.Tensor): The output of the gating operation
297
+ (before softmax).
298
- topk (int): The number of top-k experts to select.
299
- renormalize (bool): If True, renormalize the top-k weights to sum to 1.
270
- - inplace (bool): If True, perform the operation in-place. Defaults to False.
271
- - override_config (Optional[Dict[str, Any]]): Optional override for the kernel configuration.
272
-
300
+ - inplace (bool): If True, perform the operation in-place.
301
+ Defaults to False.
302
+ - override_config (Optional[Dict[str, Any]]): Optional override
303
+ for the kernel configuration.
304
+
305
Returns:
306
- torch.Tensor: The output tensor after applying the MoE layer.
307
"""
@@ -325,7 +357,8 @@ def fused_moe(
357
configs = get_moe_configs(E, w2.shape[2])
358
359
if configs:
328
- # If an optimal configuration map has been found, look up the optimal config
360
+ # If an optimal configuration map has been found, look up the
361
+ # optimal config
362
config = configs[min(configs.keys(), key=lambda x: abs(x - M))]
363
else:
364
# Else use the default config
vllm/model_executor/layers/linear.py
+8
-4
@@ -285,7 +285,8 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
285
shard_size = shard_size // param.pack_factor
286
shard_offset = shard_offset // param.pack_factor
287
288
- # If marlin, we need to adjust the offset and size to account for the tiling.
288
+ # If marlin, we need to adjust the offset and size to
289
+ # account for the tiling.
290
shard_size, shard_offset = adjust_marlin_shard(
291
param, shard_size, shard_offset)
292
@@ -307,7 +308,8 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
308
shard_size = shard_size // param.pack_factor
309
shard_offset = shard_offset // param.pack_factor
310
310
- # If marlin, we need to adjust the offset and size to account for the tiling.
311
+ # If marlin, we need to adjust the offset and size to
312
+ # account for the tiling.
313
shard_size, shard_offset = adjust_marlin_shard(
314
param, shard_size, shard_offset)
315
@@ -413,7 +415,8 @@ class QKVParallelLinear(ColumnParallelLinear):
415
shard_size = shard_size // param.pack_factor
416
shard_offset = shard_offset // param.pack_factor
417
416
- # If marlin, we need to adjust the offset and size to account for the tiling.
418
+ # If marlin, we need to adjust the offset and size to
419
+ # account for the tiling.
420
shard_size, shard_offset = adjust_marlin_shard(
421
param, shard_size, shard_offset)
422
@@ -442,7 +445,8 @@ class QKVParallelLinear(ColumnParallelLinear):
445
shard_size = shard_size // param.pack_factor
446
shard_offset = shard_offset // param.pack_factor
447
445
- # If marlin, we need to adjust the offset and size to account for the tiling.
448
+ # If marlin, we need to adjust the offset and size to
449
+ # account for the tiling.
450
shard_size, shard_offset = adjust_marlin_shard(
451
param, shard_size, shard_offset)
452
vllm/model_executor/layers/quantization/__init__.py
+2
-1
@@ -1,6 +1,7 @@
1
from typing import Type
2
3
-from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
3
+from vllm.model_executor.layers.quantization.base_config import (
4
+ QuantizationConfig)
5
from vllm.model_executor.layers.quantization.awq import AWQConfig
6
from vllm.model_executor.layers.quantization.gptq import GPTQConfig
7
from vllm.model_executor.layers.quantization.squeezellm import SqueezeLLMConfig
vllm/model_executor/layers/quantization/awq.py
+4
-2
@@ -6,7 +6,8 @@ from torch.nn.parameter import Parameter
6
from vllm._C import ops
7
from vllm.model_executor.layers.linear import (LinearMethodBase,
8
set_weight_attrs)
9
-from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
9
+from vllm.model_executor.layers.quantization.base_config import (
10
+ QuantizationConfig)
11
12
13
class AWQConfig(QuantizationConfig):
@@ -50,7 +51,8 @@ class AWQConfig(QuantizationConfig):
51
def get_config_filenames() -> List[str]:
52
return [
53
"quant_config.json", # E.g., casperhansen/vicuna-7b-v1.5-awq
53
- "quantize_config.json", # E.g., abhinavkulkarni/mosaicml-mpt-7b-instruct-w4-g128-awq
54
+ # E.g., abhinavkulkarni/mosaicml-mpt-7b-instruct-w4-g128-awq
55
+ "quantize_config.json",
56
]
57
58
@classmethod
vllm/model_executor/layers/quantization/gptq.py
+6
-4
@@ -31,8 +31,8 @@ class GPTQConfig(QuantizationConfig):
31
self.pack_factor = Fraction(32, self.weight_bits)
32
if self.weight_bits not in [2, 3, 4, 8]:
33
raise ValueError(
34
- "Currently, only 2/3/4/8-bit weight quantization is supported for "
35
- f"GPTQ, but got {self.weight_bits} bits.")
34
+ "Currently, only 2/3/4/8-bit weight quantization is "
35
+ f"supported for GPTQ, but got {self.weight_bits} bits.")
36
37
def __repr__(self) -> str:
38
return (f"GPTQConfig(weight_bits={self.weight_bits}, "
@@ -101,7 +101,8 @@ class GPTQLinearMethod(LinearMethodBase):
101
"The input size is not aligned with the quantized "
102
"weight shape. This can be caused by too large "
103
"tensor parallel size.")
104
- if output_size_per_partition % self.quant_config.pack_factor.numerator != 0:
104
+ if (output_size_per_partition % self.quant_config.pack_factor.numerator
105
+ != 0):
106
raise ValueError(
107
"The output size is not aligned with the quantized "
108
"weight shape. This can be caused by too large "
@@ -114,7 +115,8 @@ class GPTQLinearMethod(LinearMethodBase):
115
exllama_state = ExllamaState.UNINITIALIZED
116
scale_and_zero_size = input_size // group_size
117
scale_and_zero_input_dim = None
117
- if input_size != input_size_per_partition and self.quant_config.group_size != -1:
118
+ if (input_size != input_size_per_partition
119
+ and self.quant_config.group_size != -1):
120
# For act-order models, we cannot use Exllama for row parallel layer
121
if self.quant_config.desc_act:
122
exllama_state = ExllamaState.UNUSED
vllm/model_executor/layers/quantization/marlin.py
+24
-15
@@ -5,7 +5,8 @@ from torch.nn.parameter import Parameter
5
6
from vllm._C import ops
7
from vllm.model_executor.layers.linear import LinearMethodBase, set_weight_attrs
8
-from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
8
+from vllm.model_executor.layers.quantization.base_config import (
9
+ QuantizationConfig)
10
11
12
class MarlinConfig(QuantizationConfig):
@@ -22,8 +23,9 @@ class MarlinConfig(QuantizationConfig):
23
self.group_size = group_size
24
if self.group_size != 128 and self.group_size != -1:
25
raise ValueError(
25
- "Currently, only group size 128 and -1 (channelwise) is supported for "
26
- f"Marlin, but got group_size of {self.group_size}")
26
+ "Currently, only group size 128 and -1 (channelwise) "
27
+ "is supported for Marlin, but got group_size of "
28
+ f"{self.group_size}")
29
30
# 4 Bits packed into 32 bit datatype.
31
self.pack_factor = 32 // 4
@@ -37,7 +39,8 @@ class MarlinConfig(QuantizationConfig):
39
# Min in_features dim
40
self.min_k_threads = 128
41
40
- # Max parallel problems to solve at once (improves large batch performance)
42
+ # Max parallel problems to solve at once (improves large
43
+ # batch performance)
44
self.max_parallel = 16
45
46
# Permutation length used by the marlin kernels.
@@ -102,22 +105,26 @@ class MarlinLinearMethod(LinearMethodBase):
105
# Validate output_size_per_partition
106
if output_size_per_partition % self.quant_config.min_n_threads != 0:
107
raise ValueError(
105
- f"Weight output_size_per_partition = {output_size_per_partition} is not divisible by min_n_threads = {self.quant_config.min_n_threads}."
106
- )
108
+ f"Weight output_size_per_partition = "
109
+ f"{output_size_per_partition} is not divisible by "
110
+ f"min_n_threads = {self.quant_config.min_n_threads}.")
111
if output_size_per_partition % self.quant_config.pack_factor != 0:
112
raise ValueError(
109
- f"Weight output_size_per_partition = {output_size_per_partition} is not divisible by pack_factor = {self.quant_config.pack_factor}."
110
- )
113
+ f"Weight output_size_per_partition = "
114
+ f"{output_size_per_partition} is not divisible by "
115
+ f"pack_factor = {self.quant_config.pack_factor}.")
116
117
# Validate input_size_per_partition
118
if input_size_per_partition % self.quant_config.min_k_threads != 0:
119
raise ValueError(
115
- f"Weight input_size_per_partition = {input_size_per_partition} is not divisible by min_k_threads = {self.quant_config.min_k_threads}."
116
- )
117
- if self.quant_config.group_size != -1 and input_size_per_partition % self.quant_config.group_size != 0:
118
- raise ValueError(
119
- f"Weight input_size_per_partition = f{input_size_per_partition} is not divisible by group_size = {self.quant_config.group_size}."
120
- )
120
+ f"Weight input_size_per_partition = "
121
+ f"{input_size_per_partition} is not divisible by "
122
+ f"min_k_threads = {self.quant_config.min_k_threads}.")
123
+ if (self.quant_config.group_size != -1 and
124
+ input_size_per_partition % self.quant_config.group_size != 0):
125
+ raise ValueError(f"Weight input_size_per_partition = "
126
+ f"{input_size_per_partition} is not divisible by "
127
+ f"group_size = {self.quant_config.group_size}.")
128
129
# Check that we have at least 4 tiles horizontally in the shard
130
num_tiles_per_perm = self.quant_config.perm_len // (
@@ -149,7 +156,9 @@ class MarlinLinearMethod(LinearMethodBase):
156
)
157
158
# Determine if channelwise or not
152
- input_groups = 1 if self.quant_config.group_size == -1 else input_size_per_partition // self.quant_config.group_size
159
+ input_groups = (1 if self.quant_config.group_size == -1 else
160
+ input_size_per_partition //
161
+ self.quant_config.group_size)
162
163
scales = Parameter(
164
torch.empty(
vllm/model_executor/layers/quantization/squeezellm.py
+2
-1
@@ -6,7 +6,8 @@ from torch.nn.parameter import Parameter
6
from vllm._C import ops
7
from vllm.model_executor.layers.linear import (LinearMethodBase,
8
set_weight_attrs)
9
-from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
9
+from vllm.model_executor.layers.quantization.base_config import (
10
+ QuantizationConfig)
11
from vllm.utils import is_hip
12
13
vllm/model_executor/layers/sampler.py
+2
-1
@@ -6,7 +6,8 @@ import torch.nn as nn
6
7
from vllm.model_executor.parallel_utils.communication_op import (
8
tensor_model_parallel_gather)
9
-from vllm.model_executor.sampling_metadata import SamplingMetadata, SamplingTensors
9
+from vllm.model_executor.sampling_metadata import (SamplingMetadata,
10
+ SamplingTensors)
11
from vllm.sampling_params import SamplingParams, SamplingType
12
from vllm.sequence import (Logprob, PromptLogprobs, SampleLogprobs,
13
SamplerOutput, SequenceData, SequenceGroupOutput,
vllm/model_executor/models/baichuan.py
+2
-1
@@ -333,7 +333,8 @@ class BaiChuanBaseForCausalLM(nn.Module):
333
if "rotary_emb.inv_freq" in name:
334
continue
335
if name == "lm_head.weight":
336
- # Unlike Baichuan, Baichuan2 normalizes the head weights. Refer to:
336
+ # Unlike Baichuan, Baichuan2 normalizes the head weights.
337
+ # Refer to:
338
# https://huggingface.co/baichuan-inc/Baichuan2-7B-Chat/blob/84603cde5ebffb6084e476cfaeceaf0b8b91fe54/modeling_baichuan.py#L508
339
# Distinguish between Baichuan and Baichuan2 by checking the
340
# vocab size. This is suggested by
vllm/model_executor/models/deepseek.py
+5
-3
@@ -119,7 +119,8 @@ class DeepseekMoE(nn.Module):
119
linear_method=None)
120
121
if config.n_shared_experts is not None:
122
- intermediate_size = config.moe_intermediate_size * config.n_shared_experts
122
+ intermediate_size = (config.moe_intermediate_size *
123
+ config.n_shared_experts)
124
self.shared_experts = DeepseekMLP(
125
hidden_size=config.hidden_size,
126
intermediate_size=intermediate_size,
@@ -273,8 +274,9 @@ class DeepseekDecoderLayer(nn.Module):
274
max_position_embeddings=max_position_embeddings,
275
linear_method=linear_method,
276
)
276
- if (config.n_routed_experts is not None and \
277
- layer_idx >= config.first_k_dense_replace and layer_idx % config.moe_layer_freq == 0):
277
+ if (config.n_routed_experts is not None
278
+ and layer_idx >= config.first_k_dense_replace
279
+ and layer_idx % config.moe_layer_freq == 0):
280
self.mlp = DeepseekMoE(config=config, linear_method=linear_method)
281
else:
282
self.mlp = DeepseekMLP(
vllm/model_executor/models/gpt_j.py
+2
-1
@@ -143,7 +143,8 @@ class GPTJBlock(nn.Module):
143
linear_method: Optional[LinearMethodBase] = None,
144
):
145
super().__init__()
146
- inner_dim = 4 * config.n_embd if config.n_inner is None else config.n_inner
146
+ inner_dim = (4 * config.n_embd
147
+ if config.n_inner is None else config.n_inner)
148
self.ln_1 = nn.LayerNorm(config.n_embd, eps=config.layer_norm_epsilon)
149
self.attn = GPTJAttention(config, linear_method)
150
self.mlp = GPTJMLP(inner_dim, config, linear_method)
vllm/model_executor/models/internlm2.py
+2
-1
@@ -305,7 +305,8 @@ class InternLM2ForCausalLM(nn.Module):
305
param = params_dict[name]
306
if "wqkv" in name:
307
config = self.config
308
- kv_groups = config.num_attention_heads // config.num_key_value_heads
308
+ kv_groups = (config.num_attention_heads //
309
+ config.num_key_value_heads)
310
head_dim = config.hidden_size // config.num_attention_heads
311
loaded_weight = loaded_weight.view(-1, 2 + kv_groups,
312
head_dim,
vllm/model_executor/models/olmo.py
+12
-7
@@ -52,7 +52,8 @@ from vllm.model_executor.layers.linear import (
52
)
53
from vllm.model_executor.layers.rotary_embedding import get_rope
54
from vllm.model_executor.layers.sampler import Sampler
55
-from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding
55
+from vllm.model_executor.layers.vocab_parallel_embedding import (
56
+ VocabParallelEmbedding)
57
from vllm.model_executor.parallel_utils.parallel_state import (
58
get_tensor_model_parallel_world_size, )
59
from vllm.model_executor.sampling_metadata import SamplingMetadata
@@ -81,7 +82,8 @@ class SwiGLU(nn.Module):
82
83
class OlmoAttention(nn.Module):
84
"""
84
- This is the attention block where the output is computed as ``Attention(LN(x))`` in ``MLP(LN(x + Attention(LN(x))))``
85
+ This is the attention block where the output is computed as
86
+ ``Attention(LN(x))`` in ``MLP(LN(x + Attention(LN(x))))``
87
(plus another skip connection).
88
"""
89
@@ -94,11 +96,12 @@ class OlmoAttention(nn.Module):
96
self.config = config
97
self.hidden_size = config.d_model
98
assert config.d_model % config.n_heads == 0
97
- tensor_model_parallel_world_size = get_tensor_model_parallel_world_size(
98
- )
99
+ tensor_model_parallel_world_size = (
100
+ get_tensor_model_parallel_world_size())
101
self.total_num_heads = self.config.n_heads
102
assert self.total_num_heads % tensor_model_parallel_world_size == 0
101
- self.num_heads = self.total_num_heads // tensor_model_parallel_world_size
103
+ self.num_heads = (self.total_num_heads //
104
+ tensor_model_parallel_world_size)
105
self.head_dim = self.hidden_size // self.total_num_heads
106
107
# Layer norms.
@@ -158,7 +161,8 @@ class OlmoAttention(nn.Module):
161
162
class OlmoMLP(nn.Module):
163
"""
161
- This is the MLP block where the output is computed as ``MLP(LN(x))`` in ``MLP(LN(x + Attention(LN(x))))``
164
+ This is the MLP block where the output is computed as
165
+ ``MLP(LN(x))`` in ``MLP(LN(x + Attention(LN(x))))``
166
(plus another skip connection).
167
"""
168
@@ -217,7 +221,8 @@ class OlmoMLP(nn.Module):
221
222
class OlmoBlock(nn.Module):
223
"""
220
- This is a typical transformer block where the output is computed as ``MLP(LN(x + Attention(LN(x))))``
224
+ This is a typical transformer block where the output is
225
+ computed as ``MLP(LN(x + Attention(LN(x))))``
226
(plus another skip connection).
227
"""
228
vllm/model_executor/models/qwen2.py
+2
-1
@@ -170,7 +170,8 @@ class Qwen2DecoderLayer(nn.Module):
170
self.hidden_size = config.hidden_size
171
# Requires transformers > 4.32.0
172
rope_theta = getattr(config, "rope_theta", 1000000)
173
- use_sliding_window = config.use_sliding_window and layer_idx < config.max_window_layers
173
+ use_sliding_window = (config.use_sliding_window
174
+ and layer_idx < config.max_window_layers)
175
self.self_attn = Qwen2Attention(
176
hidden_size=self.hidden_size,
177
num_heads=config.num_attention_heads,
vllm/model_executor/models/stablelm.py
+7
-6
@@ -1,5 +1,6 @@
1
# coding=utf-8
2
-# Copyright 2023 Stability AI, EleutherAI, and The HuggingFace Inc. team. All rights reserved.
2
+# Copyright 2023 Stability AI, EleutherAI, and The HuggingFace Inc. team.
3
+# All rights reserved.
4
#
5
# Licensed under the Apache License, Version 2.0 (the "License");
6
# you may not use this file except in compliance with the License.
@@ -16,7 +17,8 @@
17
# This code is based off the following work:
18
# https://huggingface.co/stabilityai/stablelm-3b-4e1t/blob/main/modeling_stablelm_epoch.py
19
# https://huggingface.co/stabilityai/stablelm-3b-4e1t/blob/main/config.json
19
-"""Inference-only StabeLM (https://github.com/Stability-AI/StableLM) model compatible with HuggingFace weights."""
20
+"""Inference-only StabeLM (https://github.com/Stability-AI/StableLM)
21
+model compatible with HuggingFace weights."""
22
from typing import List, Optional, Tuple
23
24
import torch
@@ -102,9 +104,9 @@ class StablelmAttention(nn.Module):
104
self.kv_size = self.num_key_value_heads * self.head_dim
105
self.qkv_bias = getattr(config, "use_qkv_bias", False)
106
if (self.head_dim * self.num_heads * tp_size) != self.hidden_size:
105
- raise ValueError(
106
- f"hidden_size must be divisible by num_heads (got `hidden_size`: {self.hidden_size}"
107
- f" and `num_heads`: {self.num_heads}).")
107
+ raise ValueError(f"hidden_size must be divisible by num_heads "
108
+ f"(got `hidden_size`: {self.hidden_size}"
109
+ f" and `num_heads`: {self.num_heads}).")
110
111
self.qkv_proj = QKVParallelLinear(self.hidden_size,
112
self.head_dim,
@@ -192,7 +194,6 @@ class StableLMEpochModel(nn.Module):
194
config: PretrainedConfig,
195
linear_method: Optional[LinearMethodBase] = None) -> None:
196
super().__init__()
195
- # self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size, config.pad_token_id)
197
self.embed_tokens = VocabParallelEmbedding(
198
config.vocab_size,
199
config.hidden_size,
vllm/model_executor/models/starcoder2.py
+2
-1
@@ -35,7 +35,8 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
35
from vllm.model_executor.layers.sampler import Sampler
36
from vllm.model_executor.layers.vocab_parallel_embedding import (
37
VocabParallelEmbedding, ParallelLMHead, DEFAULT_VOCAB_PADDING_SIZE)
38
-from vllm.model_executor.parallel_utils.parallel_state import get_tensor_model_parallel_world_size
38
+from vllm.model_executor.parallel_utils.parallel_state import (
39
+ get_tensor_model_parallel_world_size)
40
from vllm.model_executor.weight_utils import (default_weight_loader,
41
hf_model_weights_iterator)
42
from vllm.sequence import SamplerOutput
vllm/model_executor/neuron_model_loader.py
+2
-1
@@ -34,7 +34,8 @@ def _get_model_architecture(config: PretrainedConfig) -> Type[nn.Module]:
34
35
def get_model(model_config: ModelConfig, device_config: DeviceConfig,
36
**kwargs) -> nn.Module:
37
- from transformers_neuronx.config import NeuronConfig, ContinuousBatchingConfig
37
+ from transformers_neuronx.config import (NeuronConfig,
38
+ ContinuousBatchingConfig)
39
40
parallel_config = kwargs.get("parallel_config")
41
scheduler_config = kwargs.get("scheduler_config")
vllm/model_executor/parallel_utils/communication_op.py
+3
-2
@@ -11,7 +11,8 @@ from vllm.model_executor.parallel_utils.parallel_state import (
11
get_tensor_model_parallel_group,
12
is_cupy_nccl_enabled_for_all_reduce,
13
)
14
-from vllm.model_executor.parallel_utils.custom_all_reduce import custom_all_reduce
14
+from vllm.model_executor.parallel_utils.custom_all_reduce import (
15
+ custom_all_reduce)
16
17
18
def tensor_model_parallel_all_reduce(input_: torch.Tensor) -> torch.Tensor:
@@ -24,7 +25,7 @@ def tensor_model_parallel_all_reduce(input_: torch.Tensor) -> torch.Tensor:
25
and GPU topology.
26
27
TLDR: always assume this function modifies its input, but use the return
27
- value as the output.
28
+ value as the output.
29
"""
30
# Bypass the function if we are using only 1 GPU.
31
if get_tensor_model_parallel_world_size() == 1:
vllm/model_executor/sampling_metadata.py
+2
-1
@@ -114,7 +114,8 @@ class SamplingTensors:
114
do_penalties = True
115
if (i < sampling_metadata.num_prompts
116
and sampling_params.prompt_logprobs is not None):
117
- # For tokens in the prompt that we only need to get their logprobs
117
+ # For tokens in the prompt that we only need to get
118
+ # their logprobs
119
prompt_len = sampling_metadata.prompt_lens[i]
120
temperatures += [temperature] * (prompt_len - 1)
121
top_ps += [top_p] * (prompt_len - 1)
vllm/sampling_params.py
+2
-2
@@ -74,8 +74,8 @@ class SamplingParams:
74
stop_token_ids: List of tokens that stop the generation when they are
75
generated. The returned output will contain the stop tokens unless
76
the stop tokens are special tokens.
77
- include_stop_str_in_output: Whether to include the stop strings in output
78
- text. Defaults to False.
77
+ include_stop_str_in_output: Whether to include the stop strings in
78
+ output text. Defaults to False.
79
ignore_eos: Whether to ignore the EOS token and continue generating
80
tokens after the EOS token is generated.
81
max_tokens: Maximum number of tokens to generate per output sequence.
vllm/sequence.py
+2
-1
@@ -351,7 +351,8 @@ class SequenceGroup:
351
self.metrics.first_token_time = time
352
353
def maybe_set_first_scheduled_time(self, time: float) -> None:
354
- """Sets the first scheduled time and time in queue for Request level timings."""
354
+ """Sets the first scheduled time and time in queue for Request
355
+ level timings."""
356
if self.metrics.first_scheduled_time is None:
357
self.metrics.first_scheduled_time = time
358
self.metrics.time_in_queue = time - self.metrics.arrival_time
vllm/spec_decode/batch_expansion.py
+18
-11
@@ -5,8 +5,12 @@ import torch
5
6
from vllm.sequence import (SamplerOutput, SequenceGroupMetadata, SequenceData)
7
from vllm.worker.worker import Worker
8
-from vllm.spec_decode.util import nvtx_range, sampler_output_to_torch, get_all_seq_ids, split_batch_by_proposal_len
9
-from vllm.spec_decode.interfaces import SpeculativeScorer, SpeculativeProposals, SpeculativeScores
8
+from vllm.spec_decode.util import (nvtx_range, sampler_output_to_torch,
9
+ get_all_seq_ids,
10
+ split_batch_by_proposal_len)
11
+from vllm.spec_decode.interfaces import (SpeculativeScorer,
12
+ SpeculativeProposals,
13
+ SpeculativeScores)
14
15
SeqId = int
16
TargetSeqId = int
@@ -68,11 +72,12 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
72
proposal_lens_list = proposals.proposal_lens.tolist()
73
proposal_token_ids_list = proposals.proposal_token_ids.tolist()
74
71
- spec_indices, non_spec_indices, target_seq_group_metadata_list, num_scoring_tokens = self._expand_batch(
72
- seq_group_metadata_list=seq_group_metadata_list,
73
- proposal_token_ids_list=proposal_token_ids_list,
74
- proposal_lens_list=proposal_lens_list,
75
- )
75
+ (spec_indices, non_spec_indices, target_seq_group_metadata_list,
76
+ num_scoring_tokens) = self._expand_batch(
77
+ seq_group_metadata_list=seq_group_metadata_list,
78
+ proposal_token_ids_list=proposal_token_ids_list,
79
+ proposal_lens_list=proposal_lens_list,
80
+ )
81
82
target_sampler_output = self._scorer_worker.execute_model(
83
seq_group_metadata_list=target_seq_group_metadata_list,
@@ -125,7 +130,8 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
130
num_scoring_tokens = len(target_seq_group_metadata_list)
131
target_seq_group_metadata_list.extend(non_spec_seqs)
132
128
- return spec_indices, non_spec_indices, target_seq_group_metadata_list, num_scoring_tokens
133
+ return (spec_indices, non_spec_indices, target_seq_group_metadata_list,
134
+ num_scoring_tokens)
135
136
def _contract_batch(self, original_bs: int,
137
target_sampler_output: List[SamplerOutput],
@@ -306,10 +312,11 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
312
# Convert non-speculative output tokens to tensors.
313
sampler_output.sampled_token_probs = non_spec_probs
314
sampler_output.sampled_token_ids = non_spec_sampled_tokens
309
- non_spec_target_token_ids, non_spec_target_probs = sampler_output_to_torch(
310
- [sampler_output])
315
+ non_spec_target_token_ids, non_spec_target_probs = (
316
+ sampler_output_to_torch([sampler_output]))
317
312
- return target_token_ids, target_probs, non_spec_target_token_ids, non_spec_target_probs
318
+ return (target_token_ids, target_probs, non_spec_target_token_ids,
319
+ non_spec_target_probs)
320
321
def _create_target_seq_id_iterator(
322
self, seq_ids: List[SeqId]) -> Iterator[TargetSeqId]:
vllm/spec_decode/multi_step_worker.py
+9
-5
@@ -5,7 +5,8 @@ import torch
5
6
from vllm.sequence import SamplerOutput, SequenceGroupMetadata
7
from vllm.worker.worker import Worker
8
-from vllm.spec_decode.interfaces import SpeculativeProposals, SpeculativeProposer
8
+from vllm.spec_decode.interfaces import (SpeculativeProposals,
9
+ SpeculativeProposer)
10
from vllm.spec_decode.util import sampler_output_to_torch
11
12
@@ -247,8 +248,9 @@ class DraftModelTop1Proposer(SpeculativeProposer):
248
"""
249
250
# Split speculative- and non-speculative- sequences.
250
- proposal_lens, nonzero_proposal_len_seqs, nonzero_proposal_len_indices = self._split_by_max_model_len(
251
- seq_group_metadata_list, max_proposal_len)
251
+ (proposal_lens, nonzero_proposal_len_seqs,
252
+ nonzero_proposal_len_indices) = self._split_by_max_model_len(
253
+ seq_group_metadata_list, max_proposal_len)
254
255
if nonzero_proposal_len_seqs:
256
# Speculate tokens using the draft worker for the speculative
@@ -306,7 +308,8 @@ class DraftModelTop1Proposer(SpeculativeProposer):
308
else:
309
proposal_lens.append(0)
310
309
- return proposal_lens, nonzero_proposal_len_seqs, nonzero_proposal_len_indices
311
+ return (proposal_lens, nonzero_proposal_len_seqs,
312
+ nonzero_proposal_len_indices)
313
314
def _merge_outputs(
315
self,
@@ -356,7 +359,8 @@ class DraftModelTop1Proposer(SpeculativeProposer):
359
device=self._device)
360
entire_proposal_probs[nonzero_proposal_len_indices] = proposal_probs
361
359
- proposal_tokens, proposal_probs = entire_proposal_tokens, entire_proposal_probs
362
+ proposal_tokens, proposal_probs = (entire_proposal_tokens,
363
+ entire_proposal_probs)
364
365
proposal_lens = torch.zeros(batch_size,
366
dtype=torch.long,
vllm/spec_decode/spec_decode_worker.py
+11
-8
@@ -10,7 +10,8 @@ from vllm.worker.worker import Worker
10
from vllm.spec_decode.multi_step_worker import MultiStepWorker
11
from vllm.model_executor.layers.rejection_sampler import RejectionSampler
12
from vllm.config import CacheConfig
13
-from vllm.spec_decode.util import nvtx_range, get_all_seq_ids, split_batch_by_proposal_len
13
+from vllm.spec_decode.util import (nvtx_range, get_all_seq_ids,
14
+ split_batch_by_proposal_len)
15
from vllm.spec_decode.interfaces import SpeculativeProposals, SpeculativeScores
16
from vllm.spec_decode.batch_expansion import BatchExpansionTop1Scorer
17
from vllm.spec_decode.interfaces import SpeculativeScorer
@@ -25,7 +26,7 @@ class SpecDecodeWorker:
26
LLM, after which some verification routine determines which (if any) of the
27
speculative tokens are accepted by the larger LLM.
28
28
- See https://github.com/vllm-project/vllm/pull/2188 and
29
+ See https://github.com/vllm-project/vllm/pull/2188 and
30
https://github.com/vllm-project/vllm/pull/3103 for more info.
31
32
The current implementation has the following limitations:
@@ -109,10 +110,12 @@ class SpecDecodeWorker:
110
block_size, gpu_memory_utilization, cpu_swap_space,
111
cache_dtype))
112
112
- scorer_cache_block_size_bytes = self.scorer_worker.get_cache_block_size_bytes(
113
- block_size, cache_dtype)
114
- proposer_cache_block_size_bytes = self.proposer_worker.get_cache_block_size_bytes(
115
- block_size, cache_dtype)
113
+ scorer_cache_block_size_bytes = (
114
+ self.scorer_worker.get_cache_block_size_bytes(
115
+ block_size, cache_dtype))
116
+ proposer_cache_block_size_bytes = (
117
+ self.proposer_worker.get_cache_block_size_bytes(
118
+ block_size, cache_dtype))
119
120
new_num_gpu_blocks = split_num_cache_blocks_evenly(
121
scorer_cache_block_size_bytes, proposer_cache_block_size_bytes,
@@ -320,8 +323,8 @@ class SpecDecodeWorker:
323
sampler_output_list.append(
324
SamplerOutput(outputs=step_output_token_ids))
325
323
- maybe_rejsample_metrics = self._metrics.maybe_collect_rejsample_metrics(
324
- k)
326
+ maybe_rejsample_metrics = (
327
+ self._metrics.maybe_collect_rejsample_metrics(k))
328
if maybe_rejsample_metrics is not None:
329
sampler_output_list[
330
0].spec_decode_worker_metrics = maybe_rejsample_metrics
vllm/transformers_utils/configs/mpt.py
+17
-72
@@ -62,62 +62,6 @@ class MPTConfig(PretrainedConfig):
62
fc_type: str = 'torch',
63
verbose: Optional[int] = None,
64
**kwargs: Any):
65
- """The MPT configuration class.
66
- Args:
67
- d_model (int): The size of the embedding dimension of the model.
68
- n_heads (int): The number of attention heads.
69
- n_layers (int): The number of layers in the model.
70
- expansion_ratio (int): The ratio of the up/down scale in the ffn.
71
- max_seq_len (int): The maximum sequence length of the model.
72
- vocab_size (int): The size of the vocabulary.
73
- resid_pdrop (float): The dropout probability applied to the attention output before combining with residual.
74
- emb_pdrop (float): The dropout probability for the embedding layer.
75
- learned_pos_emb (bool): Whether to use learned positional embeddings
76
- attn_config (Dict): A dictionary used to configure the model's attention module:
77
- attn_type (str): type of attention to use. Options: multihead_attention, multiquery_attention, grouped_query_attention
78
- attn_pdrop (float): The dropout probability for the attention layers.
79
- attn_impl (str): The attention implementation to use. One of 'torch', 'flash', or 'triton'.
80
- qk_ln (bool): Whether to apply layer normalization to the queries and keys in the attention layer.
81
- clip_qkv (Optional[float]): If not None, clip the queries, keys, and values in the attention layer to
82
- this value.
83
- softmax_scale (Optional[float]): If not None, scale the softmax in the attention layer by this value. If None,
84
- use the default scale of ``1/sqrt(d_keys)``.
85
- prefix_lm (Optional[bool]): Whether the model should operate as a Prefix LM. This requires passing an
86
- extra `prefix_mask` argument which indicates which tokens belong to the prefix. Tokens in the prefix
87
- can attend to one another bi-directionally. Tokens outside the prefix use causal attention.
88
- attn_uses_sequence_id (Optional[bool]): Whether to restrict attention to tokens that have the same sequence_id.
89
- When the model is in `train` mode, this requires passing an extra `sequence_id` argument which indicates
90
- which sub-sequence each token belongs to.
91
- Defaults to ``False`` meaning any provided `sequence_id` will be ignored.
92
- alibi (bool): Whether to use the alibi bias instead of position embeddings.
93
- alibi_bias_max (int): The maximum value of the alibi bias.
94
- kv_n_heads (Optional[int]): For grouped_query_attention only, allow user to specify number of kv heads.
95
- ffn_config (Dict): A dictionary used to configure the model's ffn module:
96
- ffn_type (str): type of ffn to use. Options: mptmlp, te_ln_mlp
97
- init_device (str): The device to use for parameter initialization.
98
- logit_scale (Optional[Union[float, str]]): If not None, scale the logits by this value.
99
- no_bias (bool): Whether to use bias in all layers.
100
- verbose (int): The verbosity level. 0 is silent.
101
- embedding_fraction (float): The fraction to scale the gradients of the embedding layer by.
102
- norm_type (str): choose type of norm to use
103
- use_cache (bool): Whether or not the model should return the last key/values attentions
104
- init_config (Dict): A dictionary used to configure the model initialization:
105
- init_config.name: The parameter initialization scheme to use. Options: 'default_', 'baseline_',
106
- 'kaiming_uniform_', 'kaiming_normal_', 'neox_init_', 'small_init_', 'xavier_uniform_', or
107
- 'xavier_normal_'. These mimic the parameter initialization methods in PyTorch.
108
- init_div_is_residual (Union[int, float, str, bool]): Value to divide initial weights by if ``module._is_residual`` is True.
109
- emb_init_std (Optional[float]): The standard deviation of the normal distribution used to initialize the embedding layer.
110
- emb_init_uniform_lim (Optional[Union[Tuple[float, float], float]]): The lower and upper limits of the uniform distribution
111
- used to initialize the embedding layer. Mutually exclusive with ``emb_init_std``.
112
- init_std (float): The standard deviation of the normal distribution used to initialize the model,
113
- if using the baseline_ parameter initialization scheme.
114
- init_gain (float): The gain to use for parameter initialization with kaiming or xavier initialization schemes.
115
- fan_mode (str): The fan mode to use for parameter initialization with kaiming initialization schemes.
116
- init_nonlinearity (str): The nonlinearity to use for parameter initialization with kaiming initialization schemes.
117
- ---
118
- See llmfoundry.models.utils.param_init_fns.py for info on other param init config options
119
- fc_type (str): choose fc layer implementation. Options: torch and te. te layers support fp8 when using H100 GPUs.
120
- """
65
self.d_model = d_model
66
self.n_heads = n_heads
67
self.n_layers = n_layers
@@ -139,8 +83,8 @@ class MPTConfig(PretrainedConfig):
83
self.fc_type = fc_type
84
if verbose is not None:
85
warnings.warn(DeprecationWarning(
142
- 'verbose argument for MPTConfig is now ignored and will be removed. Use python_log_level instead.'
143
- ),
86
+ 'verbose argument for MPTConfig is now ignored and '
87
+ 'will be removed. Use python_log_level instead.'),
88
stacklevel=2)
89
if 'name' in kwargs:
90
del kwargs['name']
@@ -149,7 +93,8 @@ class MPTConfig(PretrainedConfig):
93
if self.attn_config.get('alibi', False):
94
self.learned_pos_emb = False
95
warnings.warn(
152
- f'alibi is turned on, setting `learned_pos_emb` to {self.learned_pos_emb}`',
96
+ f'alibi is turned on, setting `learned_pos_emb` '
97
+ f'to {self.learned_pos_emb}`',
98
stacklevel=2)
99
super().__init__(**kwargs)
100
self._validate_config()
@@ -176,8 +121,8 @@ class MPTConfig(PretrainedConfig):
121
[self.attn_config['attn_pdrop'], self.resid_pdrop, self.emb_pdrop]
122
)):
123
raise ValueError(
179
- "self.attn_config['attn_pdrop'], resid_pdrop, emb_pdrop are probabilities and must be between 0 and 1" # pylint: disable=line-too-long
180
- )
124
+ "self.attn_config['attn_pdrop'], resid_pdrop, emb_pdrop are "
125
+ "probabilities and must be between 0 and 1")
126
if self.attn_config['attn_impl'] not in ['torch', 'flash', 'triton']:
127
raise ValueError(
128
f"Unknown attn_impl={self.attn_config['attn_impl']}")
@@ -193,17 +138,17 @@ class MPTConfig(PretrainedConfig):
138
if self.attn_config['attn_uses_sequence_id'] and self.attn_config[
139
'attn_impl'] not in ['torch', 'triton']:
140
raise NotImplementedError(
196
- 'attn_uses_sequence_id only implemented with torch and triton attention.' # pylint: disable=line-too-long
197
- )
141
+ 'attn_uses_sequence_id only implemented with torch '
142
+ 'and triton attention.')
143
if self.embedding_fraction > 1 or self.embedding_fraction <= 0:
144
raise ValueError(
200
- 'model.embedding_fraction must be between 0 (exclusive) and 1 (inclusive)!' # pylint: disable=line-too-long
201
- )
145
+ 'model.embedding_fraction must be between 0 (exclusive) '
146
+ 'and 1 (inclusive)!')
147
if isinstance(self.logit_scale,
148
str) and self.logit_scale != 'inv_sqrt_d_model':
149
raise ValueError(
205
- f"self.logit_scale={self.logit_scale!r} is not recognized as an option; use numeric value or 'inv_sqrt_d_model'." # pylint: disable=line-too-long
206
- )
150
+ f"self.logit_scale={self.logit_scale!r} is not recognized as "
151
+ "an option; use numeric value or 'inv_sqrt_d_model'.")
152
if self.init_config.get('name', None) is None:
153
raise ValueError(
154
f"self.init_config={self.init_config!r} 'name' needs to be set."
@@ -219,11 +164,11 @@ class MPTConfig(PretrainedConfig):
164
del te
165
except Exception as exc:
166
raise ImportError(
222
- # pylint: disable=line-too-long
223
- 'TransformerEngine import fail. `fc_type: te` requires TransformerEngine be installed. '
224
- +
225
- 'The required version of transformer_engine also requires FlashAttention v1.0.6 is installed:\n'
226
- + 'pip install flash-attn==1.0.6 --no-build-isolation \n' +
167
+ 'TransformerEngine import fail. `fc_type: te` requires '
168
+ 'TransformerEngine be installed. '
169
+ 'The required version of transformer_engine also requires '
170
+ 'FlashAttention v1.0.6 is installed:\n'
171
+ 'pip install flash-attn==1.0.6 --no-build-isolation \n'
172
'pip install git+https://github.com/NVIDIA/TransformerEngine.git@144e4888b2cdd60bd52e706d5b7a79cb9c1a7156'
173
) from exc
174
if self.ffn_config['ffn_type'] == 'mptmlp':
vllm/transformers_utils/configs/starcoder2.py
-72
@@ -2,78 +2,6 @@ from transformers import PretrainedConfig
2
3
4
class Starcoder2Config(PretrainedConfig):
5
- r"""
6
- This is the configuration class to store the configuration of a [`Starcoder2Model`]. It is used to instantiate a
7
- Starcoder2 model according to the specified arguments, defining the model architecture. Instantiating a configuration
8
- with the defaults will yield a similar configuration to that of the [bigcode/starcoder2-7b_16k](https://huggingface.co/bigcode/starcoder2-7b_16k) model.
9
-
10
-
11
- Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
12
- documentation from [`PretrainedConfig`] for more information.
13
-
14
-
15
- Args:
16
- vocab_size (`int`, *optional*, defaults to 49152):
17
- Vocabulary size of the Starcoder2 model. Defines the number of different tokens that can be represented by the
18
- `inputs_ids` passed when calling [`Starcoder2Model`]
19
- hidden_size (`int`, *optional*, defaults to 3072):
20
- Dimension of the hidden representations.
21
- intermediate_size (`int`, *optional*, defaults to 12288):
22
- Dimension of the MLP representations.
23
- num_hidden_layers (`int`, *optional*, defaults to 30):
24
- Number of hidden layers in the Transformer encoder.
25
- num_attention_heads (`int`, *optional*, defaults to 24):
26
- Number of attention heads for each attention layer in the Transformer encoder.
27
- num_key_value_heads (`int`, *optional*, defaults to 2):
28
- This is the number of key_value heads that should be used to implement Grouped Query Attention. If
29
- `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if
30
- `num_key_value_heads=1 the model will use Multi Query Attention (MQA) otherwise GQA is used. When
31
- converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed
32
- by meanpooling all the original heads within that group. For more details checkout [this
33
- paper](https://arxiv.org/pdf/2305.13245.pdf). If it is not specified, will default to `8`.
34
- hidden_act (`str` or `function`, *optional*, defaults to `"gelu_pytorch_tanh"`):
35
- The non-linear activation function (function or string) in the decoder.
36
- max_position_embeddings (`int`, *optional*, defaults to 4096):
37
- The maximum sequence length that this model might ever be used with. Starcoder2's sliding window attention
38
- allows sequence of up to 4096*32 tokens.
39
- initializer_range (`float`, *optional*, defaults to 0.02):
40
- The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
41
- norm_epsilon (`float`, *optional*, defaults to 1e-05):
42
- Epsilon value for the layer norm
43
- use_cache (`bool`, *optional*, defaults to `True`):
44
- Whether or not the model should return the last key/values attentions (not used by all models). Only
45
- relevant if `config.is_decoder=True`.
46
- bos_token_id (`int`, *optional*, defaults to 50256):
47
- The id of the "beginning-of-sequence" token.
48
- eos_token_id (`int`, *optional*, defaults to 50256):
49
- The id of the "end-of-sequence" token.
50
- rope_theta (`float`, *optional*, defaults to 10000.0):
51
- The base period of the RoPE embeddings.
52
- sliding_window (`int`, *optional*):
53
- Sliding window attention window size. If not specified, will default to `None` (no sliding window).
54
- attention_dropout (`float`, *optional*, defaults to 0.0):
55
- The dropout ratio for the attention probabilities.
56
- residual_dropout (`float`, *optional*, defaults to 0.0):
57
- Residual connection dropout value.
58
- embedding_dropout (`float`, *optional*, defaults to 0.0):
59
- Embedding dropout.
60
- use_bias (`bool`, *optional*, defaults to `True`):
61
- Whether to use bias term on linear layers of the model.
62
-
63
-
64
- ```python
65
- >>> from transformers import Starcoder2Model, Starcoder2Config
66
-
67
- >>> # Initializing a Starcoder2 7B style configuration
68
- >>> configuration = Starcoder2Config()
69
-
70
- >>> # Initializing a model from the Starcoder2 7B style configuration
71
- >>> model = Starcoder2Model(configuration)
72
-
73
- >>> # Accessing the model configuration
74
- >>> configuration = model.config
75
- ```"""
76
-
5
model_type = "starcoder2"
6
keys_to_ignore_at_inference = ["past_key_values"]
7
vllm/transformers_utils/tokenizers/baichuan.py
+42
-50
@@ -1,4 +1,3 @@
1
-# yapf: disable
1
# Adapted from
2
# https://huggingface.co/baichuan-inc/Baichuan2-13B-Chat/blob/8f6e343d545c503b91429582231d1d354dac2740/tokenization_baichuan.py
3
# This includes a fix suggested in
@@ -13,7 +12,6 @@ import sentencepiece as spm
12
from transformers.tokenization_utils import AddedToken, PreTrainedTokenizer
13
from transformers.utils import logging
14
16
-
15
logger = logging.get_logger(__name__)
16
17
VOCAB_FILES_NAMES = {"vocab_file": "tokenizer.model"}
@@ -52,27 +50,16 @@ class BaichuanTokenizer(PreTrainedTokenizer):
50
clean_up_tokenization_spaces=False,
51
**kwargs,
52
):
55
- self.sp_model_kwargs = {} if sp_model_kwargs is None else sp_model_kwargs
56
- bos_token = (
57
- AddedToken(bos_token, lstrip=False, rstrip=False)
58
- if isinstance(bos_token, str)
59
- else bos_token
60
- )
61
- eos_token = (
62
- AddedToken(eos_token, lstrip=False, rstrip=False)
63
- if isinstance(eos_token, str)
64
- else eos_token
65
- )
66
- unk_token = (
67
- AddedToken(unk_token, lstrip=False, rstrip=False)
68
- if isinstance(unk_token, str)
69
- else unk_token
70
- )
71
- pad_token = (
72
- AddedToken(pad_token, lstrip=False, rstrip=False)
73
- if isinstance(pad_token, str)
74
- else pad_token
75
- )
53
+ self.sp_model_kwargs = ({} if sp_model_kwargs is None else
54
+ sp_model_kwargs)
55
+ bos_token = (AddedToken(bos_token, lstrip=False, rstrip=False)
56
+ if isinstance(bos_token, str) else bos_token)
57
+ eos_token = (AddedToken(eos_token, lstrip=False, rstrip=False)
58
+ if isinstance(eos_token, str) else eos_token)
59
+ unk_token = (AddedToken(unk_token, lstrip=False, rstrip=False)
60
+ if isinstance(unk_token, str) else unk_token)
61
+ pad_token = (AddedToken(pad_token, lstrip=False, rstrip=False)
62
+ if isinstance(pad_token, str) else pad_token)
63
self.vocab_file = vocab_file
64
self.add_bos_token = add_bos_token
65
self.add_eos_token = add_eos_token
@@ -107,7 +94,10 @@ class BaichuanTokenizer(PreTrainedTokenizer):
94
95
def get_vocab(self):
96
"""Returns vocab as a dict"""
110
- vocab = {self.convert_ids_to_tokens(i): i for i in range(self.vocab_size)}
97
+ vocab = {
98
+ self.convert_ids_to_tokens(i): i
99
+ for i in range(self.vocab_size)
100
+ }
101
vocab.update(self.added_tokens_encoder)
102
return vocab
103
@@ -130,7 +120,8 @@ class BaichuanTokenizer(PreTrainedTokenizer):
120
out_string = ""
121
prev_is_special = False
122
for i, token in enumerate(tokens):
133
- # make sure that special tokens are not decoded using sentencepiece model
123
+ # make sure that special tokens are not decoded using
124
+ # sentencepiece model
125
if token in self.all_special_tokens:
126
if not prev_is_special and i != 0:
127
out_string += " "
@@ -143,9 +134,9 @@ class BaichuanTokenizer(PreTrainedTokenizer):
134
out_string += self.sp_model.decode(current_sub_tokens)
135
return out_string
136
146
- def save_vocabulary(
147
- self, save_directory, filename_prefix: Optional[str] = None
148
- ) -> Tuple[str]:
137
+ def save_vocabulary(self,
138
+ save_directory,
139
+ filename_prefix: Optional[str] = None) -> Tuple[str]:
140
"""
141
Save the vocabulary and special tokens file to a directory.
142
@@ -157,24 +148,24 @@ class BaichuanTokenizer(PreTrainedTokenizer):
148
`Tuple(str)`: Paths to the files saved.
149
"""
150
if not os.path.isdir(save_directory):
160
- logger.error(f"Vocabulary path ({save_directory}) should be a directory")
151
+ logger.error(f"Vocabulary path ({save_directory}) "
152
+ "should be a directory")
153
return
154
out_vocab_file = os.path.join(
155
save_directory,
164
- (filename_prefix + "-" if filename_prefix else "")
165
- + VOCAB_FILES_NAMES["vocab_file"],
156
+ (filename_prefix + "-" if filename_prefix else "") +
157
+ VOCAB_FILES_NAMES["vocab_file"],
158
)
159
160
if os.path.abspath(self.vocab_file) != os.path.abspath(
169
- out_vocab_file
170
- ) and os.path.isfile(self.vocab_file):
161
+ out_vocab_file) and os.path.isfile(self.vocab_file):
162
copyfile(self.vocab_file, out_vocab_file)
163
elif not os.path.isfile(self.vocab_file):
164
with open(out_vocab_file, "wb") as fi:
165
content_spiece_model = self.sp_model.serialized_model_proto()
166
fi.write(content_spiece_model)
167
177
- return (out_vocab_file,)
168
+ return (out_vocab_file, )
169
170
def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None):
171
bos_token_id = [self.bos_token_id] if self.add_bos_token else []
@@ -194,7 +185,8 @@ class BaichuanTokenizer(PreTrainedTokenizer):
185
already_has_special_tokens: bool = False,
186
) -> List[int]:
187
"""
197
- Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding
188
+ Retrieve sequence ids from a token list that has no special tokens
189
+ added. This method is called when adding
190
special tokens using the tokenizer `prepare_for_model` method.
191
192
Args:
@@ -202,11 +194,14 @@ class BaichuanTokenizer(PreTrainedTokenizer):
194
List of IDs.
195
token_ids_1 (`List[int]`, *optional*):
196
Optional second list of IDs for sequence pairs.
205
- already_has_special_tokens (`bool`, *optional*, defaults to `False`):
206
- Whether or not the token list is already formatted with special tokens for the model.
197
+ already_has_special_tokens (`bool`, *optional*, defaults to
198
+ `False`):
199
+ Whether or not the token list is already formatted with
200
+ special tokens for the model.
201
202
Returns:
209
- `List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
203
+ `List[int]`: A list of integers in the range [0, 1]:
204
+ 1 for a special token, 0 for a sequence token.
205
"""
206
if already_has_special_tokens:
207
return super().get_special_tokens_mask(
@@ -220,20 +215,16 @@ class BaichuanTokenizer(PreTrainedTokenizer):
215
216
if token_ids_1 is None:
217
return bos_token_id + ([0] * len(token_ids_0)) + eos_token_id
223
- return (
224
- bos_token_id
225
- + ([0] * len(token_ids_0))
226
- + eos_token_id
227
- + bos_token_id
228
- + ([0] * len(token_ids_1))
229
- + eos_token_id
230
- )
218
+ return (bos_token_id + ([0] * len(token_ids_0)) + eos_token_id +
219
+ bos_token_id + ([0] * len(token_ids_1)) + eos_token_id)
220
221
def create_token_type_ids_from_sequences(
233
- self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
234
- ) -> List[int]:
222
+ self,
223
+ token_ids_0: List[int],
224
+ token_ids_1: Optional[List[int]] = None) -> List[int]:
225
"""
236
- Creates a mask from the two sequences passed to be used in a sequence-pair classification task. An ALBERT
226
+ Creates a mask from the two sequences passed to be used in a
227
+ sequence-pair classification task. An ALBERT
228
sequence pair mask has the following format:
229
230
```
@@ -250,7 +241,8 @@ class BaichuanTokenizer(PreTrainedTokenizer):
241
Optional second list of IDs for sequence pairs.
242
243
Returns:
253
- `List[int]`: List of [token type IDs](../glossary#token-type-ids) according to the given sequence(s).
244
+ `List[int]`: List of [token type IDs](../glossary#token-type-ids)
245
+ according to the given sequence(s).
246
"""
247
bos_token_id = [self.bos_token_id] if self.add_bos_token else []
248
eos_token_id = [self.eos_token_id] if self.add_eos_token else []
vllm/utils.py
+6
-6
@@ -133,9 +133,10 @@ def get_max_shared_memory_bytes(gpu: int = 0) -> int:
133
# the Neuron-X backend does not have the `cuda_utils` module.
134
from vllm._C import cuda_utils
135
136
- max_shared_mem = cuda_utils.get_max_shared_memory_per_block_device_attribute(
137
- gpu)
138
- # value 0 will cause MAX_SEQ_LEN become negative and test_attention.py will fail
136
+ max_shared_mem = (
137
+ cuda_utils.get_max_shared_memory_per_block_device_attribute(gpu))
138
+ # value 0 will cause MAX_SEQ_LEN become negative and test_attention.py
139
+ # will fail
140
assert max_shared_mem > 0, "max_shared_mem can not be zero"
141
return int(max_shared_mem)
142
@@ -209,9 +210,8 @@ def get_nvcc_cuda_version() -> Optional[Version]:
210
if not cuda_home:
211
cuda_home = '/usr/local/cuda'
212
if os.path.isfile(cuda_home + '/bin/nvcc'):
212
- logger.info(
213
- f'CUDA_HOME is not found in the environment. Using {cuda_home} as CUDA_HOME.'
214
- )
213
+ logger.info(f'CUDA_HOME is not found in the environment. '
214
+ f'Using {cuda_home} as CUDA_HOME.')
215
else:
216
logger.warning(
217
f'Not found nvcc in {cuda_home}. Skip cuda version check!')
vllm/worker/model_runner.py
+5
-6
@@ -93,14 +93,13 @@ class ModelRunner:
93
scheduler_config=self.scheduler_config)
94
95
self.model_memory_usage = m.consumed_memory
96
- logger.info(
97
- f"Loading model weights took {self.model_memory_usage / float(2**30):.4f} GB"
98
- )
96
+ logger.info(f"Loading model weights took "
97
+ f"{self.model_memory_usage / float(2**30):.4f} GB")
98
99
if self.lora_config:
101
- assert hasattr(
102
- self.model, "supported_lora_modules"
103
- ) and self.model.supported_lora_modules, "Model does not support LoRA"
100
+ assert hasattr(self.model, "supported_lora_modules"
101
+ ) and self.model.supported_lora_modules, (
102
+ "Model does not support LoRA")
103
assert hasattr(
104
self.model,
105
"embedding_modules"), "Model does not have embedding_modules"
vllm/worker/neuron_worker.py
+4
-2
@@ -79,7 +79,8 @@ class Worker:
79
cpu_swap_space: int = 0,
80
cache_dtype: str = "float16",
81
) -> Tuple[int, int]:
82
- """Simply returns max_num_seqs as num_gpu_blocks, 0 as num_cpu_blocks."""
82
+ """Simply returns max_num_seqs as num_gpu_blocks, 0 as
83
+ num_cpu_blocks."""
84
num_gpu_blocks = self.scheduler_config.max_num_seqs
85
num_cpu_blocks = 0
86
return num_gpu_blocks, num_cpu_blocks
@@ -177,7 +178,8 @@ def _init_distributed_environment(
178
"distributed_init_method must be set if torch.distributed "
179
"is not already initialized")
180
else:
180
- distributed_backend = distributed_backend if distributed_backend else "nccl"
181
+ distributed_backend = (distributed_backend
182
+ if distributed_backend else "nccl")
183
torch.distributed.init_process_group(
184
backend=distributed_backend,
185
world_size=parallel_config.world_size,