@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 2f8844ba08
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

Re-enable the 80 char line width limit (#3305)

Zhuohan Li committed Mar 10, 2024 at 19:49 UTC 2f8844ba08d77af8a64784317055b03a475f6051
67 files changed +557 -528
pyproject.toml
+4 -2
@@ -9,6 +9,10 @@ requires = [
9 ]
10 build-backend = "setuptools.build_meta"
11
12 +[tool.ruff]
13 +# Allow lines to be as long as 80.
14 +line-length = 80
15 +
16 [tool.ruff.lint]
17 select = [
18 # pycodestyle
@@ -29,8 +33,6 @@ ignore = [
33 "F405", "F403",
34 # lambda expression assignment
35 "E731",
32 - # line too long, handled by black formatting
33 - "E501",
36 # .strip() with multi-character strings
37 "B005",
38 # Loop control variable not used within loop body
setup.py
+2 -2
@@ -142,8 +142,8 @@ def get_pytorch_rocm_arch() -> Set[str]:
142 # If we don't have PYTORCH_ROCM_ARCH specified pull the list from rocm_agent_enumerator
143 if env_arch_list is None:
144 command = "rocm_agent_enumerator"
145 - env_arch_list = subprocess.check_output([command]).decode('utf-8')\
146 - .strip().replace("\n", ";")
145 + env_arch_list = (subprocess.check_output(
146 + [command]).decode('utf-8').strip().replace("\n", ";"))
147 arch_source_str = "rocm_agent_enumerator"
148 else:
149 arch_source_str = "PYTORCH_ROCM_ARCH env variable"
tests/async_engine/test_chat_template.py
+4 -2
@@ -73,7 +73,7 @@ def test_load_chat_template():
73 assert template_content is not None
74 # Hard coded value for template_chatml.jinja
75 assert template_content == """{% for message in messages %}{{'<|im_start|>' + message['role'] + '\\n' + message['content']}}{% if (loop.last and add_generation_prompt) or not loop.last %}{{ '<|im_end|>' + '\\n'}}{% endif %}{% endfor %}
76 -{% if add_generation_prompt and messages[-1]['role'] != 'assistant' %}{{ '<|im_start|>assistant\\n' }}{% endif %}"""
76 +{% if add_generation_prompt and messages[-1]['role'] != 'assistant' %}{{ '<|im_start|>assistant\\n' }}{% endif %}""" # noqa: E501
77
78
79 def test_no_load_chat_template():
@@ -117,4 +117,6 @@ async def test_get_gen_prompt(model, template, add_generation_prompt,
117 add_generation_prompt=mock_request.add_generation_prompt)
118
119 # Test assertion
120 - assert result == expected_output, f"The generated prompt does not match the expected output for model {model} and template {template}"
120 + assert result == expected_output, (
121 + f"The generated prompt does not match the expected output for "
122 + f"model {model} and template {template}")
tests/core/test_block_manager.py
+2 -1
@@ -4,7 +4,8 @@ from typing import List
4
5 from vllm import SamplingParams
6 from vllm.block import PhysicalTokenBlock
7 -from vllm.core.block_manager import BlockAllocator, BlockSpaceManager, AllocStatus
7 +from vllm.core.block_manager import (BlockAllocator, BlockSpaceManager,
8 + AllocStatus)
9 from vllm.utils import Device
10 from vllm.sequence import Sequence, SequenceGroup, SequenceStatus, Logprob
11
tests/entrypoints/test_guided_processors.py
+2 -2
@@ -46,8 +46,8 @@ TEST_SCHEMA = {
46 "required": ["name", "age", "skills", "work history"]
47 }
48
49 -TEST_REGEX = r"((25[0-5]|(2[0-4]|1\d|[1-9]|)\d)\.){3}" + \
50 - r"(25[0-5]|(2[0-4]|1\d|[1-9]|)\d)"
49 +TEST_REGEX = (r"((25[0-5]|(2[0-4]|1\d|[1-9]|)\d)\.){3}"
50 + r"(25[0-5]|(2[0-4]|1\d|[1-9]|)\d)")
51
52
53 def test_guided_logits_processors():
tests/entrypoints/test_openai_server.py
+23 -13
@@ -5,9 +5,12 @@ import time
5 import sys
6 import pytest
7 import requests
8 -import ray # using Ray for overall ease of process management, parallel requests, and debugging.
8 +# using Ray for overall ease of process management, parallel requests,
9 +# and debugging.
10 +import ray
11 import openai # use the official client for correctness check
10 -from huggingface_hub import snapshot_download # downloading lora to test lora requests
12 +# downloading lora to test lora requests
13 +from huggingface_hub import snapshot_download
14
15 # imports for guided decoding tests
16 import json
@@ -17,8 +20,11 @@ import re
20 from vllm.transformers_utils.tokenizer import get_tokenizer
21
22 MAX_SERVER_START_WAIT_S = 600 # wait for server to start for 60 seconds
20 -MODEL_NAME = "HuggingFaceH4/zephyr-7b-beta" # any model with a chat template should work here
21 -LORA_NAME = "typeof/zephyr-7b-beta-lora" # technically this needs Mistral-7B-v0.1 as base, but we're not testing generation quality here
23 +# any model with a chat template should work here
24 +MODEL_NAME = "HuggingFaceH4/zephyr-7b-beta"
25 +# technically this needs Mistral-7B-v0.1 as base, but we're not testing
26 +# generation quality here
27 +LORA_NAME = "typeof/zephyr-7b-beta-lora"
28
29 TEST_SCHEMA = {
30 "type": "object",
@@ -59,8 +65,8 @@ TEST_SCHEMA = {
65 "required": ["name", "age", "skills", "work history"]
66 }
67
62 -TEST_REGEX = r"((25[0-5]|(2[0-4]|1\d|[1-9]|)\d)\.){3}" + \
63 - r"(25[0-5]|(2[0-4]|1\d|[1-9]|)\d)"
68 +TEST_REGEX = (r"((25[0-5]|(2[0-4]|1\d|[1-9]|)\d)\.){3}"
69 + r"(25[0-5]|(2[0-4]|1\d|[1-9]|)\d)")
70
71 TEST_CHOICE = [
72 "Python", "Java", "JavaScript", "C++", "C#", "PHP", "TypeScript", "Ruby",
@@ -120,8 +126,9 @@ def server(zephyr_lora_files):
126 server_runner = ServerRunner.remote([
127 "--model",
128 MODEL_NAME,
129 + # use half precision for speed and memory savings in CI environment
130 "--dtype",
124 - "bfloat16", # use half precision for speed and memory savings in CI environment
131 + "bfloat16",
132 "--max-model-len",
133 "8192",
134 "--enforce-eager",
@@ -392,7 +399,8 @@ async def test_batch_completions(server, client: openai.AsyncOpenAI,
399 max_tokens=5,
400 temperature=0.0,
401 extra_body=dict(
395 - # NOTE: this has to be true for n > 1 in vLLM, but not necessary for official client.
402 + # NOTE: this has to be true for n > 1 in vLLM, but not necessary
403 + # for official client.
404 use_beam_search=True),
405 )
406 assert len(batch.choices) == 4
@@ -469,8 +477,8 @@ async def test_logits_bias(server, client: openai.AsyncOpenAI):
477 async def test_guided_json_completion(server, client: openai.AsyncOpenAI):
478 completion = await client.completions.create(
479 model=MODEL_NAME,
472 - prompt=
473 - f"Give an example JSON for an employee profile that fits this schema: {TEST_SCHEMA}",
480 + prompt=f"Give an example JSON for an employee profile "
481 + f"that fits this schema: {TEST_SCHEMA}",
482 n=3,
483 temperature=1.0,
484 max_tokens=500,
@@ -489,9 +497,11 @@ async def test_guided_json_chat(server, client: openai.AsyncOpenAI):
497 "role": "system",
498 "content": "you are a helpful assistant"
499 }, {
492 - "role": "user",
493 - "content": "Give an example JSON for an employee profile that " + \
494 - f"fits this schema: {TEST_SCHEMA}"
500 + "role":
501 + "user",
502 + "content":
503 + f"Give an example JSON for an employee profile that "
504 + f"fits this schema: {TEST_SCHEMA}"
505 }]
506 chat_completion = await client.chat.completions.create(
507 model=MODEL_NAME,
tests/kernels/test_moe.py
+2 -1
@@ -57,7 +57,8 @@ def test_fused_moe(
57 [torch.float32, torch.float16, torch.bfloat16])
58 @torch.inference_mode()
59 def test_mixtral_moe(dtype: torch.dtype):
60 - "Make sure our Mixtral MoE implementation agrees with the one from huggingface."
60 + """Make sure our Mixtral MoE implementation agrees with the one from
61 + huggingface."""
62
63 # Instantiate our and huggingface's MoE blocks
64 config = MixtralConfig()
tests/kernels/test_prefix_prefill.py
+2 -1
@@ -114,7 +114,8 @@ def test_contexted_kv_attention(
114 v_cache = v_cache.view(-1, block_size, num_kv_heads,
115 head_size).permute(0, 2, 3, 1).contiguous()
116
117 - # Warm up the Triton kernel by calling it once before actually measuring generation time
117 + # Warm up the Triton kernel by calling it once before actually measuring
118 + # generation time
119 context_attention_fwd(query, k, v, output, k_cache, v_cache, block_table,
120 b_start_loc, b_seq_len, b_ctx_len, max_input_len)
121 torch.cuda.synchronize()
tests/lora/test_layer_variation.py
+3 -3
@@ -11,9 +11,9 @@ from .conftest import cleanup
11
12 MODEL_PATH = "Felladrin/Llama-68M-Chat-v1"
13 PROMPTS = [
14 - "[system] Given a target sentence construct the underlying meaning representation\nof the input sentence as a single function with attributes and attribute\nvalues. This function should describe the target string accurately and the\nfunction must be one of the following ['inform', 'request', 'give_opinion',\n'confirm', 'verify_attribute', 'suggest', 'request_explanation',\n'recommend', 'request_attribute'].\n\nThe attributes must be one of the following:\n['name', 'exp_release_date', 'release_year', 'developer', 'esrb', 'rating',\n'genres', 'player_perspective', 'has_multiplayer', 'platforms',\n'available_on_steam', 'has_linux_release', 'has_mac_release', 'specifier'] [/system] [user] Here is the target sentence:\nSpellForce 3 is a pretty bad game. The developer Grimlore Games is clearly a bunch of no-talent hacks, and 2017 was a terrible year for games anyway. [/user] [assistant]",
15 - "[system] Given a target sentence construct the underlying meaning representation\nof the input sentence as a single function with attributes and attribute\nvalues. This function should describe the target string accurately and the\nfunction must be one of the following ['inform', 'request', 'give_opinion',\n'confirm', 'verify_attribute', 'suggest', 'request_explanation',\n'recommend', 'request_attribute'].\n\nThe attributes must be one of the following:\n['name', 'exp_release_date', 'release_year', 'developer', 'esrb', 'rating',\n'genres', 'player_perspective', 'has_multiplayer', 'platforms',\n'available_on_steam', 'has_linux_release', 'has_mac_release', 'specifier'] [/system] [user] Here is the target sentence:\nI wanted to like Grimlore Games' 2017 entry, but in SpellForce 3 they just didn't get anything right. [/user] [assistant]",
16 - "[system] Given a target sentence construct the underlying meaning representation\nof the input sentence as a single function with attributes and attribute\nvalues. This function should describe the target string accurately and the\nfunction must be one of the following ['inform', 'request', 'give_opinion',\n'confirm', 'verify_attribute', 'suggest', 'request_explanation',\n'recommend', 'request_attribute'].\n\nThe attributes must be one of the following:\n['name', 'exp_release_date', 'release_year', 'developer', 'esrb', 'rating',\n'genres', 'player_perspective', 'has_multiplayer', 'platforms',\n'available_on_steam', 'has_linux_release', 'has_mac_release', 'specifier'] [/system] [user] Here is the target sentence:\nBioShock is a good role-playing, action-adventure, shooter that released for PlayStation, Xbox, and PC in 2007. It is available on Steam, and it has a Mac release but not a Linux release. [/user] [assistant]",
14 + "[system] Given a target sentence construct the underlying meaning representation\nof the input sentence as a single function with attributes and attribute\nvalues. This function should describe the target string accurately and the\nfunction must be one of the following ['inform', 'request', 'give_opinion',\n'confirm', 'verify_attribute', 'suggest', 'request_explanation',\n'recommend', 'request_attribute'].\n\nThe attributes must be one of the following:\n['name', 'exp_release_date', 'release_year', 'developer', 'esrb', 'rating',\n'genres', 'player_perspective', 'has_multiplayer', 'platforms',\n'available_on_steam', 'has_linux_release', 'has_mac_release', 'specifier'] [/system] [user] Here is the target sentence:\nSpellForce 3 is a pretty bad game. The developer Grimlore Games is clearly a bunch of no-talent hacks, and 2017 was a terrible year for games anyway. [/user] [assistant]", # noqa: E501
15 + "[system] Given a target sentence construct the underlying meaning representation\nof the input sentence as a single function with attributes and attribute\nvalues. This function should describe the target string accurately and the\nfunction must be one of the following ['inform', 'request', 'give_opinion',\n'confirm', 'verify_attribute', 'suggest', 'request_explanation',\n'recommend', 'request_attribute'].\n\nThe attributes must be one of the following:\n['name', 'exp_release_date', 'release_year', 'developer', 'esrb', 'rating',\n'genres', 'player_perspective', 'has_multiplayer', 'platforms',\n'available_on_steam', 'has_linux_release', 'has_mac_release', 'specifier'] [/system] [user] Here is the target sentence:\nI wanted to like Grimlore Games' 2017 entry, but in SpellForce 3 they just didn't get anything right. [/user] [assistant]", # noqa: E501
16 + "[system] Given a target sentence construct the underlying meaning representation\nof the input sentence as a single function with attributes and attribute\nvalues. This function should describe the target string accurately and the\nfunction must be one of the following ['inform', 'request', 'give_opinion',\n'confirm', 'verify_attribute', 'suggest', 'request_explanation',\n'recommend', 'request_attribute'].\n\nThe attributes must be one of the following:\n['name', 'exp_release_date', 'release_year', 'developer', 'esrb', 'rating',\n'genres', 'player_perspective', 'has_multiplayer', 'platforms',\n'available_on_steam', 'has_linux_release', 'has_mac_release', 'specifier'] [/system] [user] Here is the target sentence:\nBioShock is a good role-playing, action-adventure, shooter that released for PlayStation, Xbox, and PC in 2007. It is available on Steam, and it has a Mac release but not a Linux release. [/user] [assistant]", # noqa: E501
17 ]
18
19
tests/lora/test_layers.py
+9 -6
@@ -17,14 +17,16 @@ from vllm.lora.layers import (
17 LoRAMapping,
18 BaseLayerWithLoRA,
19 )
20 -from vllm.lora.models import LoRALayerWeights, convert_mapping, PackedLoRALayerWeights
20 +from vllm.lora.models import (LoRALayerWeights, convert_mapping,
21 + PackedLoRALayerWeights)
22 from vllm.config import LoRAConfig
23 from vllm.model_executor.layers.sampler import Sampler
24 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
25 MergedColumnParallelLinear,
26 RowParallelLinear,
27 QKVParallelLinear)
27 -from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding, ParallelLMHead
28 +from vllm.model_executor.layers.vocab_parallel_embedding import (
29 + VocabParallelEmbedding, ParallelLMHead)
30 from vllm.model_executor.utils import set_random_seed
31
32 from .utils import DummyLoRAManager
@@ -258,7 +260,8 @@ def test_embeddings(dist_init, num_loras, device) -> None:
260
261
262 @torch.inference_mode()
261 -# @pytest.mark.skip(reason="Fails when loras are in any slot other than the first.")
263 +# @pytest.mark.skip(
264 +# reason="Fails when loras are in any slot other than the first.")
265 @pytest.mark.parametrize("num_loras", [1, 2, 4, 8])
266 @pytest.mark.parametrize("device", CUDA_DEVICES)
267 def test_embeddings_with_new_embeddings(dist_init, num_loras, device) -> None:
@@ -674,9 +677,9 @@ def test_column_parallel_packed(dist_init, num_loras, repeats, device) -> None:
677 result = linear(input_)[0]
678 subloras = sublora_dict[lora_id]
679 for i, sublora in enumerate(subloras):
677 - result[:, sublora.lora_b.shape[1] * i:sublora.lora_b.shape[1] * (
678 - i + 1
679 - )] += input_ @ sublora.lora_a @ sublora.lora_b * sublora.scaling
680 + result[:, sublora.lora_b.shape[1] * i:sublora.lora_b.shape[1] *
681 + (i + 1)] += (input_ @ sublora.lora_a @ sublora.lora_b *
682 + sublora.scaling)
683 expected_results.append(result)
684 expected_result = torch.cat(expected_results)
685
tests/lora/test_llama.py
+25 -22
@@ -10,12 +10,12 @@ MODEL_PATH = "meta-llama/Llama-2-7b-hf"
10
11 def do_sample(llm, lora_path: str, lora_id: int):
12 prompts = [
13 - "[user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_74 (icao VARCHAR, airport VARCHAR)\n\n question: Name the ICAO for lilongwe international airport [/user] [assistant]",
14 - "[user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_11 (nationality VARCHAR, elector VARCHAR)\n\n question: When Anchero Pantaleone was the elector what is under nationality? [/user] [assistant]",
15 - "[user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_95 (one_mora VARCHAR, gloss VARCHAR, accented_mora VARCHAR)\n\n question: What is the one mora for a low tone mora with a gloss of /˩okiru/ [òkìɽɯ́]? [/user] [assistant]",
16 - "[user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE candidate (people_id VARCHAR, unsure_rate INTEGER); CREATE TABLE people (sex VARCHAR, people_id VARCHAR)\n\n question: which gender got the highest average uncertain ratio. [/user] [assistant]",
17 - "[user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_60 (pick INTEGER, former_wnba_team VARCHAR)\n\n question: What pick was a player that previously played for the Minnesota Lynx? [/user] [assistant]",
18 - "[user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_28138035_4 (womens_doubles VARCHAR, mens_singles VARCHAR)\n\n question: Name the women's doubles for werner schlager [/user] [assistant]"
13 + "[user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_74 (icao VARCHAR, airport VARCHAR)\n\n question: Name the ICAO for lilongwe international airport [/user] [assistant]", # noqa: E501
14 + "[user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_11 (nationality VARCHAR, elector VARCHAR)\n\n question: When Anchero Pantaleone was the elector what is under nationality? [/user] [assistant]", # noqa: E501
15 + "[user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_95 (one_mora VARCHAR, gloss VARCHAR, accented_mora VARCHAR)\n\n question: What is the one mora for a low tone mora with a gloss of /˩okiru/ [òkìɽɯ́]? [/user] [assistant]", # noqa: E501
16 + "[user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE candidate (people_id VARCHAR, unsure_rate INTEGER); CREATE TABLE people (sex VARCHAR, people_id VARCHAR)\n\n question: which gender got the highest average uncertain ratio. [/user] [assistant]", # noqa: E501
17 + "[user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_60 (pick INTEGER, former_wnba_team VARCHAR)\n\n question: What pick was a player that previously played for the Minnesota Lynx? [/user] [assistant]", # noqa: E501
18 + "[user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_28138035_4 (womens_doubles VARCHAR, mens_singles VARCHAR)\n\n question: Name the women's doubles for werner schlager [/user] [assistant]" # noqa: E501
19 ]
20 sampling_params = vllm.SamplingParams(temperature=0,
21 max_tokens=256,
@@ -48,20 +48,20 @@ def test_llama_lora(sql_lora_files, tp_size):
48 tensor_parallel_size=tp_size)
49
50 expected_no_lora_output = [
51 - "\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_75 (icao VARCHAR, airport VARCHAR)\n\n question: Name the ICAO for lilongwe international airport [/user] [assistant]\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_76 (icao VARCHAR, airport VARCHAR)\n\n question: Name the ICAO for lilongwe international airport [/user] [assistant]\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_77 (icao VARCHAR, airport VARCHAR)\n\n question: Name the ICAO for lilongwe international airport [/user] [assistant]\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_78 (icao VARCHAR, airport VARCHAR)\n\n question: Name the ICAO for lilongwe international airport [/user]",
52 - " Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_11 (nationality VARCHAR, elector VARCHAR)\n\n question: When Anchero Pantaleone was the elector what is under nationality? ",
53 - "\n\n answer: 1\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_96 (one_mora VARCHAR, gloss VARCHAR, accented_mora VARCHAR)\n\n question: What is the one mora for a high tone mora with a gloss of /˧kot/ [kòt]? [/user] [assistant]\n\n answer: 2\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_97 (one_mora VARCHAR, gloss VARCHAR, accented_mora VARCHAR)\n\n question: What is the one mora for a high tone mora with a gloss of /˧kot/ [kòt]? [/user] [assistant]\n\n answer: 2\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_98 (one_mora VARCHAR, gloss VARCHAR, accented_mora VARCHAR)\n\n question: What is the one m",
54 - " Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE candidate (people_id VARCHAR, unsure_rate INTEGER); CREATE TABLE people (sex VARCHAR, people_id VARCHAR)\n\n question: which gender got the highest average uncertain ratio. ",
55 - " Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_60 (pick INTEGER, former_wnba_team VARCHAR)\n\n question: What pick was a player that previously played for the Minnesota Lynx? ",
56 - "\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_28138035_4 (womens_doubles VARCHAR, mens_singles VARCHAR)\n\n question: Name the women's doubles for werner schlager [/user] [assistant]\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_28138035_4 (womens_doubles VARCHAR, mens_singles VARCHAR)\n\n question: Name the women's doubles for werner schlager [/user] [assistant]\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_28138035_4 (womens_doubles VARCHAR, mens_singles VARCHAR)\n\n question: Name the women's doubles for werner schlager [/user] [assistant]\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE",
51 + "\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_75 (icao VARCHAR, airport VARCHAR)\n\n question: Name the ICAO for lilongwe international airport [/user] [assistant]\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_76 (icao VARCHAR, airport VARCHAR)\n\n question: Name the ICAO for lilongwe international airport [/user] [assistant]\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_77 (icao VARCHAR, airport VARCHAR)\n\n question: Name the ICAO for lilongwe international airport [/user] [assistant]\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_78 (icao VARCHAR, airport VARCHAR)\n\n question: Name the ICAO for lilongwe international airport [/user]", # noqa: E501
52 + " Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_11 (nationality VARCHAR, elector VARCHAR)\n\n question: When Anchero Pantaleone was the elector what is under nationality? ", # noqa: E501
53 + "\n\n answer: 1\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_96 (one_mora VARCHAR, gloss VARCHAR, accented_mora VARCHAR)\n\n question: What is the one mora for a high tone mora with a gloss of /˧kot/ [kòt]? [/user] [assistant]\n\n answer: 2\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_97 (one_mora VARCHAR, gloss VARCHAR, accented_mora VARCHAR)\n\n question: What is the one mora for a high tone mora with a gloss of /˧kot/ [kòt]? [/user] [assistant]\n\n answer: 2\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_98 (one_mora VARCHAR, gloss VARCHAR, accented_mora VARCHAR)\n\n question: What is the one m", # noqa: E501
54 + " Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE candidate (people_id VARCHAR, unsure_rate INTEGER); CREATE TABLE people (sex VARCHAR, people_id VARCHAR)\n\n question: which gender got the highest average uncertain ratio. ", # noqa: E501
55 + " Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_60 (pick INTEGER, former_wnba_team VARCHAR)\n\n question: What pick was a player that previously played for the Minnesota Lynx? ", # noqa: E501
56 + "\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_28138035_4 (womens_doubles VARCHAR, mens_singles VARCHAR)\n\n question: Name the women's doubles for werner schlager [/user] [assistant]\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_28138035_4 (womens_doubles VARCHAR, mens_singles VARCHAR)\n\n question: Name the women's doubles for werner schlager [/user] [assistant]\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_28138035_4 (womens_doubles VARCHAR, mens_singles VARCHAR)\n\n question: Name the women's doubles for werner schlager [/user] [assistant]\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE", # noqa: E501
57 ]
58 expected_lora_output = [
59 - " SELECT icao FROM table_name_74 WHERE airport = 'lilongwe international airport' ",
60 - " SELECT nationality FROM table_name_11 WHERE elector = 'anchero pantaleone' ",
61 - " SELECT one_mora FROM table_name_95 WHERE gloss = 'low tone mora with a gloss of /˩okiru/' [òkìɽɯ́] AND accented_mora = 'low tone mora with a gloss of /˩okiru/' [òkìɽɯ́] ",
62 - " SELECT sex FROM people WHERE people_id IN (SELECT people_id FROM candidate GROUP BY sex ORDER BY COUNT(people_id) DESC LIMIT 1) ",
63 - " SELECT pick FROM table_name_60 WHERE former_wnba_team = 'Minnesota Lynx' ",
64 - " SELECT womens_doubles FROM table_28138035_4 WHERE mens_singles = 'Werner Schlager' "
59 + " SELECT icao FROM table_name_74 WHERE airport = 'lilongwe international airport' ", # noqa: E501
60 + " SELECT nationality FROM table_name_11 WHERE elector = 'anchero pantaleone' ", # noqa: E501
61 + " SELECT one_mora FROM table_name_95 WHERE gloss = 'low tone mora with a gloss of /˩okiru/' [òkìɽɯ́] AND accented_mora = 'low tone mora with a gloss of /˩okiru/' [òkìɽɯ́] ", # noqa: E501
62 + " SELECT sex FROM people WHERE people_id IN (SELECT people_id FROM candidate GROUP BY sex ORDER BY COUNT(people_id) DESC LIMIT 1) ", # noqa: E501
63 + " SELECT pick FROM table_name_60 WHERE former_wnba_team = 'Minnesota Lynx' ", # noqa: E501
64 + " SELECT womens_doubles FROM table_28138035_4 WHERE mens_singles = 'Werner Schlager' " # noqa: E501
65 ]
66
67 print("lora adapter created")
@@ -121,7 +121,8 @@ def test_llama_tensor_parallel_equality(sql_lora_files):
121
122
123 def test_llama_lora_warmup(sql_lora_files):
124 - """Test that the LLM initialization works with a warmup LORA path and is more conservative"""
124 + """Test that the LLM initialization works with a warmup LORA path and
125 + is more conservative"""
126
127 @ray.remote(num_gpus=1)
128 def get_num_gpu_blocks_lora():
@@ -132,13 +133,15 @@ def test_llama_lora_warmup(sql_lora_files):
133 @ray.remote(num_gpus=1)
134 def get_num_gpu_blocks_no_lora():
135 llm = vllm.LLM(MODEL_PATH, max_num_seqs=16)
135 - num_gpu_blocks_no_lora_warmup = llm.llm_engine.cache_config.num_gpu_blocks
136 + num_gpu_blocks_no_lora_warmup = (
137 + llm.llm_engine.cache_config.num_gpu_blocks)
138 return num_gpu_blocks_no_lora_warmup
139
140 num_gpu_blocks_lora_warmup = ray.get(get_num_gpu_blocks_lora.remote())
141 num_gpu_blocks_no_lora_warmup = ray.get(
142 get_num_gpu_blocks_no_lora.remote())
143 assert num_gpu_blocks_lora_warmup < num_gpu_blocks_no_lora_warmup, (
142 - "The warmup with lora should be more"
143 - " conservative than without lora, therefore the number of memory blocks for the KV cache should be "
144 + "The warmup with lora should be more "
145 + "conservative than without lora, therefore the number of "
146 + "memory blocks for the KV cache should be "
147 "less when using lora than when not using lora")
tests/lora/test_mixtral.py
+6 -6
@@ -9,9 +9,9 @@ MODEL_PATH = "mistralai/Mixtral-8x7B-Instruct-v0.1"
9
10 def do_sample(llm, lora_path: str, lora_id: int):
11 prompts = [
12 - "[system] Given a target sentence construct the underlying meaning representation\nof the input sentence as a single function with attributes and attribute\nvalues. This function should describe the target string accurately and the\nfunction must be one of the following ['inform', 'request', 'give_opinion',\n'confirm', 'verify_attribute', 'suggest', 'request_explanation',\n'recommend', 'request_attribute'].\n\nThe attributes must be one of the following:\n['name', 'exp_release_date', 'release_year', 'developer', 'esrb', 'rating',\n'genres', 'player_perspective', 'has_multiplayer', 'platforms',\n'available_on_steam', 'has_linux_release', 'has_mac_release', 'specifier'] [/system] [user] Here is the target sentence:\nSpellForce 3 is a pretty bad game. The developer Grimlore Games is clearly a bunch of no-talent hacks, and 2017 was a terrible year for games anyway. [/user] [assistant]",
13 - "[system] Given a target sentence construct the underlying meaning representation\nof the input sentence as a single function with attributes and attribute\nvalues. This function should describe the target string accurately and the\nfunction must be one of the following ['inform', 'request', 'give_opinion',\n'confirm', 'verify_attribute', 'suggest', 'request_explanation',\n'recommend', 'request_attribute'].\n\nThe attributes must be one of the following:\n['name', 'exp_release_date', 'release_year', 'developer', 'esrb', 'rating',\n'genres', 'player_perspective', 'has_multiplayer', 'platforms',\n'available_on_steam', 'has_linux_release', 'has_mac_release', 'specifier'] [/system] [user] Here is the target sentence:\nI wanted to like Grimlore Games' 2017 entry, but in SpellForce 3 they just didn't get anything right. [/user] [assistant]",
14 - "[system] Given a target sentence construct the underlying meaning representation\nof the input sentence as a single function with attributes and attribute\nvalues. This function should describe the target string accurately and the\nfunction must be one of the following ['inform', 'request', 'give_opinion',\n'confirm', 'verify_attribute', 'suggest', 'request_explanation',\n'recommend', 'request_attribute'].\n\nThe attributes must be one of the following:\n['name', 'exp_release_date', 'release_year', 'developer', 'esrb', 'rating',\n'genres', 'player_perspective', 'has_multiplayer', 'platforms',\n'available_on_steam', 'has_linux_release', 'has_mac_release', 'specifier'] [/system] [user] Here is the target sentence:\nBioShock is a good role-playing, action-adventure, shooter that released for PlayStation, Xbox, and PC in 2007. It is available on Steam, and it has a Mac release but not a Linux release. [/user] [assistant]",
12 + "[system] Given a target sentence construct the underlying meaning representation\nof the input sentence as a single function with attributes and attribute\nvalues. This function should describe the target string accurately and the\nfunction must be one of the following ['inform', 'request', 'give_opinion',\n'confirm', 'verify_attribute', 'suggest', 'request_explanation',\n'recommend', 'request_attribute'].\n\nThe attributes must be one of the following:\n['name', 'exp_release_date', 'release_year', 'developer', 'esrb', 'rating',\n'genres', 'player_perspective', 'has_multiplayer', 'platforms',\n'available_on_steam', 'has_linux_release', 'has_mac_release', 'specifier'] [/system] [user] Here is the target sentence:\nSpellForce 3 is a pretty bad game. The developer Grimlore Games is clearly a bunch of no-talent hacks, and 2017 was a terrible year for games anyway. [/user] [assistant]", # noqa: E501
13 + "[system] Given a target sentence construct the underlying meaning representation\nof the input sentence as a single function with attributes and attribute\nvalues. This function should describe the target string accurately and the\nfunction must be one of the following ['inform', 'request', 'give_opinion',\n'confirm', 'verify_attribute', 'suggest', 'request_explanation',\n'recommend', 'request_attribute'].\n\nThe attributes must be one of the following:\n['name', 'exp_release_date', 'release_year', 'developer', 'esrb', 'rating',\n'genres', 'player_perspective', 'has_multiplayer', 'platforms',\n'available_on_steam', 'has_linux_release', 'has_mac_release', 'specifier'] [/system] [user] Here is the target sentence:\nI wanted to like Grimlore Games' 2017 entry, but in SpellForce 3 they just didn't get anything right. [/user] [assistant]", # noqa: E501
14 + "[system] Given a target sentence construct the underlying meaning representation\nof the input sentence as a single function with attributes and attribute\nvalues. This function should describe the target string accurately and the\nfunction must be one of the following ['inform', 'request', 'give_opinion',\n'confirm', 'verify_attribute', 'suggest', 'request_explanation',\n'recommend', 'request_attribute'].\n\nThe attributes must be one of the following:\n['name', 'exp_release_date', 'release_year', 'developer', 'esrb', 'rating',\n'genres', 'player_perspective', 'has_multiplayer', 'platforms',\n'available_on_steam', 'has_linux_release', 'has_mac_release', 'specifier'] [/system] [user] Here is the target sentence:\nBioShock is a good role-playing, action-adventure, shooter that released for PlayStation, Xbox, and PC in 2007. It is available on Steam, and it has a Mac release but not a Linux release. [/user] [assistant]", # noqa: E501
15 ]
16 sampling_params = vllm.SamplingParams(temperature=0, max_tokens=256)
17 outputs = llm.generate(
@@ -42,9 +42,9 @@ def test_mixtral_lora(mixtral_lora_files, tp_size):
42 worker_use_ray=True)
43
44 expected_lora_output = [
45 - "give_opinion(name[SpellForce 3], release_year[2017], developer[Grimlore Games], rating[poor])",
46 - "give_opinion(name[SpellForce 3], release_year[2017], developer[Grimlore Games], rating[poor])",
47 - "inform(name[BioShock], release_year[2007], rating[good], genres[action-adventure, role-playing, shooter], platforms[PlayStation, Xbox, PC], available_on_steam[yes], has_linux_release[no], has_mac_release[yes])",
45 + "give_opinion(name[SpellForce 3], release_year[2017], developer[Grimlore Games], rating[poor])", # noqa: E501
46 + "give_opinion(name[SpellForce 3], release_year[2017], developer[Grimlore Games], rating[poor])", # noqa: E501
47 + "inform(name[BioShock], release_year[2007], rating[good], genres[action-adventure, role-playing, shooter], platforms[PlayStation, Xbox, PC], available_on_steam[yes], has_linux_release[no], has_mac_release[yes])", # noqa: E501
48 ]
49
50 assert do_sample(llm, mixtral_lora_files,
tests/metrics/test_metrics.py
+8 -6
@@ -21,7 +21,8 @@ def test_metric_counter_prompt_tokens(
21 gpu_memory_utilization=0.4)
22 tokenizer = vllm_model.model.get_tokenizer()
23 prompt_token_counts = [len(tokenizer.encode(p)) for p in example_prompts]
24 - # This test needs at least 2 prompts in a batch of different lengths to verify their token count is correct despite padding.
24 + # This test needs at least 2 prompts in a batch of different lengths to
25 + # verify their token count is correct despite padding.
26 assert len(example_prompts) > 1, "at least 2 prompts are required"
27 assert prompt_token_counts[0] != prompt_token_counts[1], (
28 "prompts of different lengths are required")
@@ -33,8 +34,8 @@ def test_metric_counter_prompt_tokens(
34 **stat_logger.labels)._value.get()
35
36 assert vllm_prompt_token_count == metric_count, (
36 - f"prompt token count: {vllm_prompt_token_count!r}\nmetric: {metric_count!r}"
37 - )
37 + f"prompt token count: {vllm_prompt_token_count!r}\n"
38 + f"metric: {metric_count!r}")
39
40
41 @pytest.mark.parametrize("model", MODELS)
@@ -60,9 +61,10 @@ def test_metric_counter_generation_tokens(
61 for i in range(len(example_prompts)):
62 vllm_output_ids, vllm_output_str = vllm_outputs[i]
63 prompt_ids = tokenizer.encode(example_prompts[i])
63 - # vllm_output_ids contains both prompt tokens and generation tokens. We're interested only in the count of the generation tokens.
64 + # vllm_output_ids contains both prompt tokens and generation tokens.
65 + # We're interested only in the count of the generation tokens.
66 vllm_generation_count += len(vllm_output_ids) - len(prompt_ids)
67
68 assert vllm_generation_count == metric_count, (
67 - f"generation token count: {vllm_generation_count!r}\nmetric: {metric_count!r}"
68 - )
69 + f"generation token count: {vllm_generation_count!r}\n"
70 + f"metric: {metric_count!r}")
tests/models/test_marlin.py
+8 -7
@@ -1,7 +1,7 @@
1 """Compare the outputs of a GPTQ model to a Marlin model.
2
3 -Note: GPTQ and Marlin do not have bitwise correctness.
4 -As a result, in this test, we just confirm that the top selected tokens of the
3 +Note: GPTQ and Marlin do not have bitwise correctness.
4 +As a result, in this test, we just confirm that the top selected tokens of the
5 Marlin/GPTQ models are in the top 3 selections of each other.
6
7 Note: Marlin internally uses locks to synchronize the threads. This can
@@ -14,7 +14,8 @@ Run `pytest tests/models/test_marlin.py --forked`.
14 import pytest
15 import torch
16 from dataclasses import dataclass
17 -from vllm.model_executor.layers.quantization import _QUANTIZATION_CONFIG_REGISTRY
17 +from vllm.model_executor.layers.quantization import (
18 + _QUANTIZATION_CONFIG_REGISTRY)
19
20 capability = torch.cuda.get_device_capability()
21 capability = capability[0] * 10 + capability[1]
@@ -87,11 +88,11 @@ def test_models(
88 if marlin_output_id != gptq_output_id:
89 # Each predicted token must be in top 5 of the other's
90 assert gptq_output_id in marlin_logprobs[idx], (
90 - f"Test{prompt_idx}:\nGPTQ:\t{gptq_output_str!r}\nMarlin:\t{marlin_output_str!r}"
91 - )
91 + f"Test{prompt_idx}:\nGPTQ:\t{gptq_output_str!r}\n"
92 + f"Marlin:\t{marlin_output_str!r}")
93 assert marlin_output_id in gptq_logprobs[idx], (
93 - f"Test{prompt_idx}:\nGPTQ:\t{gptq_output_str!r}\nMarlin:\t{marlin_output_str!r}"
94 - )
94 + f"Test{prompt_idx}:\nGPTQ:\t{gptq_output_str!r}\n"
95 + f"Marlin:\t{marlin_output_str!r}")
96
97 # Break out since sequences will now diverge.
98 break
tests/prefix_caching/test_prefix_caching.py
+10 -5
@@ -20,20 +20,23 @@ def test_block_allocator(
20 num_blocks,
21 enable_caching=True)
22
23 - # Allocate two PysicalTokenBlocks with the same hash and check that they are the same PhysicalTokenBlock
23 + # Allocate two PysicalTokenBlocks with the same hash and check
24 + # that they are the same PhysicalTokenBlock
25 first_block = block_allocator.allocate(block_hash, 0)
26 second_block = block_allocator.allocate(block_hash, 0)
27 assert (first_block == second_block)
28 assert (second_block.ref_count == 2)
29
29 - # Free the first_block and confirm that the ref_count is correctly decremented on the second block
30 + # Free the first_block and confirm that the ref_count is correctly
31 + # decremented on the second block
32 block_allocator.free(first_block)
33 assert (second_block.ref_count == 1)
34
35 # Free the second block
36 block_allocator.free(second_block)
37
36 - # Reallocate the first block and confirm that, even after the block had its ref_count go to 0, we still get the same block back
38 + # Reallocate the first block and confirm that, even after the block
39 + # had its ref_count go to 0, we still get the same block back
40 first_block = block_allocator.allocate(block_hash, 0)
41 assert (first_block == second_block)
42 assert (first_block.block_hash == block_hash)
@@ -56,7 +59,8 @@ def test_eviction(num_blocks: int, ):
59 for block in blocks:
60 block_allocator.free(block)
61
59 - # Allocate a new block and confirm that it's the first block freed. I.E The Least Recently Used block
62 + # Allocate a new block and confirm that it's the first block freed.
63 + # I.E The Least Recently Used block
64 new_block_hash = block_size
65 new_block = block_allocator.allocate(new_block_hash, 0)
66 assert (new_block == blocks[0])
@@ -68,7 +72,8 @@ def test_eviction(num_blocks: int, ):
72 assert (realloc_block == blocks[realloc_block_hash])
73 assert (realloc_block.block_hash == realloc_block_hash)
74
71 - # Allocate a new block and confirm that it's not the realloc_block, since the realloc_block shouldn't be in the free list
75 + # Allocate a new block and confirm that it's not the realloc_block,
76 + # since the realloc_block shouldn't be in the free list
77 new_block_hash = block_size + 1
78 new_block = block_allocator.allocate(new_block_hash, 0)
79 assert (realloc_block != new_block)
tests/samplers/test_logprobs.py
+2 -2
@@ -70,8 +70,8 @@ def test_get_prompt_logprobs(
70 hf_logprob[i][-1][token_id].item(),
71 atol=1e-2,
72 rtol=1e-2)
73 - assert isinstance(sample_logprob.decoded_token, str), \
74 - ("The token should be decoded by the time it is returned "
73 + assert isinstance(sample_logprob.decoded_token, str), (
74 + "The token should be decoded by the time it is returned "
75 " to the user.")
76
77
tests/samplers/test_sampler.py
+10 -7
@@ -255,9 +255,10 @@ def test_sampler_mixed(seed: int, device: str):
255 if metadata.sampling_params.use_beam_search:
256 continue
257
258 - if metadata.sampling_params.seed is not None \
259 - and expected_tokens[i] is None:
260 - # Record seeded random result to compare with results of second invocation
258 + if (metadata.sampling_params.seed is not None
259 + and expected_tokens[i] is None):
260 + # Record seeded random result to compare with results of
261 + # second invocation
262 expected_tokens[i] = [
263 nth_output.output_token
264 for nth_output in sequence_output.samples
@@ -265,11 +266,13 @@ def test_sampler_mixed(seed: int, device: str):
266 continue
267
268 for n, nth_output in enumerate(sequence_output.samples):
268 - if metadata.sampling_params.temperature == 0 or metadata.sampling_params.seed is not None:
269 + if (metadata.sampling_params.temperature == 0
270 + or metadata.sampling_params.seed is not None):
271 # Ensure exact matches for greedy or random with seed
272 assert nth_output.output_token == expected_tokens[i][n]
273 else:
272 - # For non-seeded random check that one of the high-logit tokens were chosen
274 + # For non-seeded random check that one of the high-logit
275 + # tokens were chosen
276 assert nth_output.output_token in expected_tokens[i]
277
278 # Test batch
@@ -284,8 +287,8 @@ def test_sampler_mixed(seed: int, device: str):
287 input_tensor.data = input_tensor.index_select(0, target_index)
288 fake_logits.data = fake_logits.index_select(0, target_index)
289
287 - # This time, results of seeded random samples will be compared with the corresponding
288 - # sample in the pre-shuffled batch
290 + # This time, results of seeded random samples will be compared with
291 + # the corresponding sample in the pre-shuffled batch
292 test_sampling(model_runner)
293
294 del model_runner
tests/spec_decode/test_metrics.py
+4 -2
@@ -150,8 +150,10 @@ def test_initial_metrics_has_correct_values(has_data: bool):
150 assert metrics.emitted_tokens == num_emitted_tokens
151
152 if has_data:
153 - assert metrics.draft_acceptance_rate == num_accepted_tokens / num_draft_tokens
154 - assert metrics.system_efficiency == num_emitted_tokens / num_possible_tokens
153 + assert (metrics.draft_acceptance_rate == num_accepted_tokens /
154 + num_draft_tokens)
155 + assert (metrics.system_efficiency == num_emitted_tokens /
156 + num_possible_tokens)
157 else:
158 assert math.isnan(metrics.draft_acceptance_rate)
159 assert math.isnan(metrics.system_efficiency)
tests/spec_decode/test_multi_step_worker.py
+2 -1
@@ -3,7 +3,8 @@ import random
3 import pytest
4 from unittest.mock import MagicMock
5
6 -from vllm.spec_decode.multi_step_worker import MultiStepWorker, DraftModelTop1Proposer
6 +from vllm.spec_decode.multi_step_worker import (MultiStepWorker,
7 + DraftModelTop1Proposer)
8 from vllm.worker.worker import Worker
9 from vllm.model_executor.utils import set_random_seed
10 from vllm.sequence import SamplerOutput
tests/spec_decode/test_spec_decode_worker.py
+12 -6
@@ -4,12 +4,15 @@ import pytest
4 from unittest.mock import MagicMock
5
6 from vllm.spec_decode.multi_step_worker import MultiStepWorker
7 -from vllm.spec_decode.spec_decode_worker import SpecDecodeWorker, split_num_cache_blocks_evenly
7 +from vllm.spec_decode.spec_decode_worker import (SpecDecodeWorker,
8 + split_num_cache_blocks_evenly)
9 from vllm.spec_decode.interfaces import SpeculativeProposals
10 from vllm.model_executor.utils import set_random_seed
11 from vllm.model_executor.layers.rejection_sampler import RejectionSampler
11 -from .utils import mock_worker, create_batch, ExecuteModelData, create_sampler_output_list
12 -from vllm.spec_decode.metrics import SpecDecodeWorkerMetrics, AsyncMetricsCollector
12 +from .utils import (mock_worker, create_batch, ExecuteModelData,
13 + create_sampler_output_list)
14 +from vllm.spec_decode.metrics import (SpecDecodeWorkerMetrics,
15 + AsyncMetricsCollector)
16
17
18 @pytest.mark.parametrize('k', [1, 2, 6])
@@ -391,13 +394,15 @@ def test_collects_metrics(k: int, batch_size: int, returns_metrics: bool):
394
395 mock_rejsample_metrics = MagicMock(
396 spec=SpecDecodeWorkerMetrics) if returns_metrics else None
394 - metrics_collector.maybe_collect_rejsample_metrics.return_value = mock_rejsample_metrics
397 + metrics_collector.maybe_collect_rejsample_metrics.return_value = (
398 + mock_rejsample_metrics)
399
400 output = worker.execute_model(**execute_model_data.to_dict(),
401 num_spec_tokens=k)
402 assert output[0].spec_decode_worker_metrics == mock_rejsample_metrics
403
400 - call_args_list = metrics_collector.maybe_collect_rejsample_metrics.call_args_list
404 + call_args_list = (
405 + metrics_collector.maybe_collect_rejsample_metrics.call_args_list)
406 assert len(call_args_list) == 1
407 args, kwargs = call_args_list[0]
408 assert args[0] == k or kwargs.get('k', -1) == k
@@ -547,7 +552,8 @@ def test_profile_num_available_blocks(available_gpu_blocks: int,
552
553 target_worker.profile_num_available_blocks.return_value = (
554 available_gpu_blocks, available_cpu_blocks)
550 - target_worker.get_cache_block_size_bytes.return_value = target_cache_block_size_bytes
555 + target_worker.get_cache_block_size_bytes.return_value = (
556 + target_cache_block_size_bytes)
557 draft_worker.get_cache_block_size_bytes.return_value = draft_kv_size_bytes
558
559 worker = SpecDecodeWorker(draft_worker, target_worker, rejection_sampler,
vllm/config.py
+8 -6
@@ -45,7 +45,7 @@ class ModelConfig:
45 a tag name, or a commit id. If unspecified, will use the default
46 version.
47 code_revision: The specific revision to use for the model code on
48 - Hugging Face Hub. It can be a branch name, a tag name, or a
48 + Hugging Face Hub. It can be a branch name, a tag name, or a
49 commit id. If unspecified, will use the default version.
50 tokenizer_revision: The specific tokenizer version to use. It can be a
51 branch name, a tag name, or a commit id. If unspecified, will use
@@ -189,8 +189,8 @@ class ModelConfig:
189 if is_hip(
190 ) and self.quantization in rocm_not_supported_quantization:
191 raise ValueError(
192 - f"{self.quantization} quantization is currently not supported "
193 - f"in ROCm.")
192 + f"{self.quantization} quantization is currently not "
193 + f"supported in ROCm.")
194 if self.quantization != "marlin":
195 logger.warning(
196 f"{self.quantization} quantization is not fully "
@@ -321,7 +321,8 @@ class CacheConfig:
321 self.num_cpu_blocks = None
322
323 def metrics_info(self):
324 - # convert cache_config to dict(key: str, value: str) for prometheus metrics info
324 + # convert cache_config to dict(key: str, value: str) for prometheus
325 + # metrics info
326 return {key: str(value) for key, value in self.__dict__.items()}
327
328 def _verify_args(self) -> None:
@@ -399,8 +400,9 @@ class ParallelConfig:
400 ) -> None:
401 self.pipeline_parallel_size = pipeline_parallel_size
402 if is_neuron():
402 - # For Neuron device support, here we assign TP=1 to avoid sharding within vLLM directly.
403 - # Transformer-neuronx would take neuron_tp_degree attribute, and distribute the workload
403 + # For Neuron device support, here we assign TP=1 to avoid sharding
404 + # within vLLM directly. Transformer-neuronx would take
405 + # neuron_tp_degree attribute, and distribute the workload
406 # to multiple NeuronCores.
407 self.tensor_parallel_size = 1
408 self.neuron_tp_degree = tensor_parallel_size
vllm/core/block_manager.py
+10 -5
@@ -95,13 +95,15 @@ class BlockAllocator:
95 del self.cached_blocks[block.block_hash]
96
97 def get_num_free_blocks(self) -> int:
98 - return self.num_blocks - self.current_num_blocks + self.evictor.num_blocks
98 + return (self.num_blocks - self.current_num_blocks +
99 + self.evictor.num_blocks)
100
101 def contains_block(self, block_hash: int) -> bool:
102 return block_hash in self.cached_blocks or block_hash in self.evictor
103
104 def update_hash(self, block_hash: int, block: PhysicalTokenBlock):
104 - # If caching is enabled, update the hash of block and the cached_blocks dictionary.
105 + # If caching is enabled, update the hash of block and the
106 + # cached_blocks dictionary.
107 if self.enable_caching:
108 assert not self.contains_block(block_hash)
109 old_hash = block.block_hash
@@ -218,10 +220,12 @@ class BlockSpaceManager:
220 seq: Sequence,
221 last_block: PhysicalTokenBlock,
222 ) -> PhysicalTokenBlock:
221 - # Compute a new hash for the block so that it can be shared by other Sequences
223 + # Compute a new hash for the block so that it can be shared by
224 + # other Sequences
225 new_hash = seq.hash_of_block(len(seq.logical_token_blocks) - 1)
226
224 - # if new_hash is already in the cached table, then free last_block and return the cached version
227 + # if new_hash is already in the cached table, then free last_block
228 + # and return the cached version
229 if self.gpu_allocator.contains_block(new_hash):
230 self.gpu_allocator.free(last_block)
231 return self.gpu_allocator.allocate(new_hash)
@@ -289,7 +293,8 @@ class BlockSpaceManager:
293 assert last_block.device == Device.GPU
294 if last_block.ref_count == 1:
295 # Not shared with other sequences. Appendable.
292 - # If the last block is now complete, promote it to a full block so that it can be shared
296 + # If the last block is now complete, promote it to a full block so
297 + # that it can be shared
298 new_block = self._maybe_promote_last_block(seq, last_block)
299 block_table[-1] = new_block
300 return None
vllm/core/evictor.py
+3 -3
@@ -39,9 +39,9 @@ class Evictor(ABC):
39 @abstractmethod
40 def remove(self, block_hash: int) -> PhysicalTokenBlock:
41 """Simply removes the block with the hash value block_hash from the
42 - evictor. Caller is responsible for making sure that block_hash is contained
43 - in the evictor before calling remove. Should be used to "bring back" blocks
44 - that have been freed but not evicted yet.
42 + evictor. Caller is responsible for making sure that block_hash is
43 + contained in the evictor before calling remove. Should be used to
44 + "bring back" blocks that have been freed but not evicted yet.
45 """
46 pass
47
vllm/core/scheduler.py
+4 -4
@@ -214,8 +214,8 @@ class Scheduler:
214 lora_int_id = 0
215 if self.lora_enabled:
216 lora_int_id = seq_group.lora_int_id
217 - if lora_int_id > 0 and lora_int_id not in curr_loras and len(
218 - curr_loras) >= self.lora_config.max_loras:
217 + if (lora_int_id > 0 and lora_int_id not in curr_loras
218 + and len(curr_loras) >= self.lora_config.max_loras):
219 # We don't have a space for another LoRA, so
220 # we ignore this request for now.
221 leftover_waiting_sequences.appendleft(seq_group)
@@ -309,8 +309,8 @@ class Scheduler:
309 lora_int_id = 0
310 if self.lora_enabled:
311 lora_int_id = seq_group.lora_int_id
312 - if lora_int_id > 0 and lora_int_id not in curr_loras and len(
313 - curr_loras) >= self.lora_config.max_loras:
312 + if (lora_int_id > 0 and lora_int_id not in curr_loras
313 + and len(curr_loras) >= self.lora_config.max_loras):
314 # We don't have a space for another LoRA, so
315 # we ignore this request for now.
316 leftover_swapped.appendleft(seq_group)
vllm/engine/llm_engine.py
+15 -12
@@ -100,7 +100,8 @@ class LLMEngine:
100 f"download_dir={model_config.download_dir!r}, "
101 f"load_format={model_config.load_format}, "
102 f"tensor_parallel_size={parallel_config.tensor_parallel_size}, "
103 - f"disable_custom_all_reduce={parallel_config.disable_custom_all_reduce}, "
103 + f"disable_custom_all_reduce="
104 + f"{parallel_config.disable_custom_all_reduce}, "
105 f"quantization={model_config.quantization}, "
106 f"enforce_eager={model_config.enforce_eager}, "
107 f"kv_cache_dtype={cache_config.cache_dtype}, "
@@ -929,7 +930,8 @@ class LLMEngine:
930 # Latency Timings.
931 time_last_iters = []
932 for seq_group in scheduler_outputs.scheduled_seq_groups:
932 - # Time since last token. (n.b. updates seq_group.metrics.last_token_time)
933 + # Time since last token.
934 + # (n.b. updates seq_group.metrics.last_token_time)
935 time_last_iters.append(seq_group.get_last_latency(now))
936 # Time since arrival for all finished requests.
937 if seq_group.is_finished():
@@ -961,16 +963,17 @@ class LLMEngine:
963 for token_id, sample_logprob in logprobs.items():
964 if (sample_logprob.decoded_token is None and token_id != -1):
965 all_input_ids_with_logprob = all_input_ids[:-1] + [token_id]
964 - _, new_text, prefix_offset, read_offset = detokenize_incrementally(
965 - self.get_tokenizer_for_seq(seq),
966 - all_input_ids=all_input_ids_with_logprob,
967 - prev_tokens=seq.tokens,
968 - prefix_offset=seq.prefix_offset,
969 - read_offset=seq.read_offset,
970 - skip_special_tokens=prms.skip_special_tokens,
971 - spaces_between_special_tokens=prms.
972 - spaces_between_special_tokens,
973 - )
966 + (_, new_text, prefix_offset,
967 + read_offset) = detokenize_incrementally(
968 + self.get_tokenizer_for_seq(seq),
969 + all_input_ids=all_input_ids_with_logprob,
970 + prev_tokens=seq.tokens,
971 + prefix_offset=seq.prefix_offset,
972 + read_offset=seq.read_offset,
973 + skip_special_tokens=prms.skip_special_tokens,
974 + spaces_between_special_tokens=prms.
975 + spaces_between_special_tokens,
976 + )
977 sample_logprob.decoded_token = new_text
978
979 def _decode_sequence(self, seq: Sequence, prms: SamplingParams) -> None:
vllm/engine/metrics.py
+13 -9
@@ -1,5 +1,6 @@
1 from vllm.logger import init_logger
2 -from prometheus_client import Counter, Gauge, Histogram, Info, REGISTRY, disable_created_metrics
2 +from prometheus_client import (Counter, Gauge, Histogram, Info, REGISTRY,
3 + disable_created_metrics)
4
5 import time
6 import numpy as np
@@ -177,10 +178,12 @@ class StatLogger:
178 def _log_prometheus_interval(self, prompt_throughput: float,
179 generation_throughput: float) -> None:
180 # Logs metrics to prometheus that are computed every logging_interval.
180 - # Support legacy gauge metrics that make throughput calculations on the vLLM side.
181 - # Moving forward, we should use counters like counter_prompt_tokens, counter_generation_tokens
182 - # Which log raw data and calculate summaries using rate() on the grafana/prometheus side.
183 - # See https://github.com/vllm-project/vllm/pull/2316#discussion_r1464204666
181 + # Support legacy gauge metrics that make throughput calculations on
182 + # the vLLM side. Moving forward, we should use counters like
183 + # counter_prompt_tokens, counter_generation_tokens
184 + # Which log raw data and calculate summaries using rate() on the
185 + # grafana/prometheus side. See
186 + # https://github.com/vllm-project/vllm/pull/2316#discussion_r1464204666
187 self.metrics.gauge_avg_prompt_throughput.labels(
188 **self.labels).set(prompt_throughput)
189 self.metrics.gauge_avg_generation_throughput.labels(
@@ -188,7 +191,7 @@ class StatLogger:
191
192 def log(self, stats: Stats) -> None:
193 """Called by LLMEngine.
191 - Logs to prometheus and tracked stats every iteration.
194 + Logs to prometheus and tracked stats every iteration.
195 Logs to Stdout every self.local_interval seconds."""
196
197 # Log to prometheus.
@@ -200,8 +203,8 @@ class StatLogger:
203
204 # Log locally every local_interval seconds.
205 if self._local_interval_elapsed(stats.now):
203 -
204 - # Compute summary metrics for tracked stats (and log them to promethus if applicable).
206 + # Compute summary metrics for tracked stats (and log them
207 + # to promethus if applicable).
208 prompt_throughput = self._get_throughput(self.num_prompt_tokens,
209 now=stats.now)
210 generation_throughput = self._get_throughput(
@@ -213,7 +216,8 @@ class StatLogger:
216 # Log to stdout.
217 logger.info(
218 f"Avg prompt throughput: {prompt_throughput:.1f} tokens/s, "
216 - f"Avg generation throughput: {generation_throughput:.1f} tokens/s, "
219 + f"Avg generation throughput: "
220 + f"{generation_throughput:.1f} tokens/s, "
221 f"Running: {stats.num_running} reqs, "
222 f"Swapped: {stats.num_swapped} reqs, "
223 f"Pending: {stats.num_waiting} reqs, "
vllm/entrypoints/api_server.py
+5 -3
@@ -1,7 +1,9 @@
1 """
2 -NOTE: This API server is used only for demonstrating usage of AsyncEngine and simple performance benchmarks.
3 -It is not intended for production use. For production use, we recommend using our OpenAI compatible server.
4 -We are also not going to accept PRs modifying this file, please change `vllm/entrypoints/openai/api_server.py` instead.
2 +NOTE: This API server is used only for demonstrating usage of AsyncEngine
3 +and simple performance benchmarks. It is not intended for production use.
4 +For production use, we recommend using our OpenAI compatible server.
5 +We are also not going to accept PRs modifying this file, please
6 +change `vllm/entrypoints/openai/api_server.py` instead.
7 """
8
9 import argparse
vllm/entrypoints/openai/api_server.py
+16 -17
@@ -18,7 +18,9 @@ from fastapi.responses import JSONResponse, StreamingResponse, Response
18 import vllm
19 from vllm.engine.arg_utils import AsyncEngineArgs
20 from vllm.engine.async_llm_engine import AsyncLLMEngine
21 -from vllm.entrypoints.openai.protocol import CompletionRequest, ChatCompletionRequest, ErrorResponse
21 +from vllm.entrypoints.openai.protocol import (CompletionRequest,
22 + ChatCompletionRequest,
23 + ErrorResponse)
24 from vllm.logger import init_logger
25 from vllm.entrypoints.openai.serving_chat import OpenAIServingChat
26 from vllm.entrypoints.openai.serving_completion import OpenAIServingCompletion
@@ -84,13 +86,11 @@ def parse_args():
86 type=json.loads,
87 default=["*"],
88 help="allowed headers")
87 - parser.add_argument(
88 - "--api-key",
89 - type=str,
90 - default=None,
91 - help=
92 - "If provided, the server will require this key to be presented in the header."
93 - )
89 + parser.add_argument("--api-key",
90 + type=str,
91 + default=None,
92 + help="If provided, the server will require this key "
93 + "to be presented in the header.")
94 parser.add_argument("--served-model-name",
95 type=str,
96 default=None,
@@ -103,9 +103,8 @@ def parse_args():
103 default=None,
104 nargs='+',
105 action=LoRAParserAction,
106 - help=
107 - "LoRA module configurations in the format name=path. Multiple modules can be specified."
108 - )
106 + help="LoRA module configurations in the format name=path. "
107 + "Multiple modules can be specified.")
108 parser.add_argument("--chat-template",
109 type=str,
110 default=None,
@@ -138,9 +137,10 @@ def parse_args():
137 help="Additional ASGI middleware to apply to the app. "
138 "We accept multiple --middleware arguments. "
139 "The value should be an import path. "
141 - "If a function is provided, vLLM will add it to the server using @app.middleware('http'). "
142 - "If a class is provided, vLLM will add it to the server using app.add_middleware(). "
143 - )
140 + "If a function is provided, vLLM will add it to the server "
141 + "using @app.middleware('http'). "
142 + "If a class is provided, vLLM will add it to the server "
143 + "using app.add_middleware(). ")
144
145 parser = AsyncEngineArgs.add_cli_args(parser)
146 return parser.parse_args()
@@ -235,9 +235,8 @@ if __name__ == "__main__":
235 elif inspect.iscoroutinefunction(imported):
236 app.middleware("http")(imported)
237 else:
238 - raise ValueError(
239 - f"Invalid middleware {middleware}. Must be a function or a class."
240 - )
238 + raise ValueError(f"Invalid middleware {middleware}. "
239 + f"Must be a function or a class.")
240
241 logger.info(f"vLLM API server version {vllm.__version__}")
242 logger.info(f"args: {args}")
vllm/entrypoints/openai/serving_chat.py
+15 -10
@@ -12,7 +12,8 @@ from vllm.entrypoints.openai.protocol import (
12 UsageInfo)
13 from vllm.outputs import RequestOutput
14 from vllm.entrypoints.openai.serving_engine import OpenAIServing, LoRA
15 -from vllm.model_executor.guided_decoding import get_guided_decoding_logits_processor
15 +from vllm.model_executor.guided_decoding import (
16 + get_guided_decoding_logits_processor)
17
18 logger = init_logger(__name__)
19
@@ -37,8 +38,9 @@ class OpenAIServingChat(OpenAIServing):
38 ChatCompletionResponse]:
39 """Completion API similar to OpenAI's API.
40
40 - See https://platform.openai.com/docs/api-reference/chat/create
41 - for the API specification. This API mimics the OpenAI ChatCompletion API.
41 + See https://platform.openai.com/docs/api-reference/chat/create
42 + for the API specification. This API mimics the OpenAI
43 + ChatCompletion API.
44
45 NOTE: Currently we do not support the following feature:
46 - function_call (Users should implement this by themselves)
@@ -116,7 +118,8 @@ class OpenAIServingChat(OpenAIServing):
118 # the result_generator, it needs to be sent as the FIRST
119 # response (by the try...catch).
120 if first_iteration:
119 - # Send first response for each request.n (index) with the role
121 + # Send first response for each request.n (index) with
122 + # the role
123 role = self.get_chat_request_role(request)
124 for i in range(request.n):
125 choice_data = ChatCompletionResponseStreamChoice(
@@ -133,7 +136,8 @@ class OpenAIServingChat(OpenAIServing):
136 data = chunk.model_dump_json(exclude_unset=True)
137 yield f"data: {data}\n\n"
138
136 - # Send response to echo the input portion of the last message
139 + # Send response to echo the input portion of the
140 + # last message
141 if request.echo:
142 last_msg_content = ""
143 if request.messages and isinstance(
@@ -145,11 +149,12 @@ class OpenAIServingChat(OpenAIServing):
149
150 if last_msg_content:
151 for i in range(request.n):
148 - choice_data = ChatCompletionResponseStreamChoice(
149 - index=i,
150 - delta=DeltaMessage(
151 - content=last_msg_content),
152 - finish_reason=None)
152 + choice_data = (
153 + ChatCompletionResponseStreamChoice(
154 + index=i,
155 + delta=DeltaMessage(
156 + content=last_msg_content),
157 + finish_reason=None))
158 chunk = ChatCompletionStreamResponse(
159 id=request_id,
160 object=chunk_object_type,
vllm/entrypoints/openai/serving_completion.py
+17 -11
@@ -1,7 +1,8 @@
1 import asyncio
2 import time
3 from fastapi import Request
4 -from typing import AsyncGenerator, AsyncIterator, Callable, List, Optional, Dict, Tuple
4 +from typing import (AsyncGenerator, AsyncIterator, Callable, List, Optional,
5 + Dict, Tuple)
6 from vllm.logger import init_logger
7 from vllm.utils import random_uuid
8 from vllm.engine.async_llm_engine import AsyncLLMEngine
@@ -16,7 +17,8 @@ from vllm.entrypoints.openai.protocol import (
17 )
18 from vllm.outputs import RequestOutput
19 from vllm.entrypoints.openai.serving_engine import OpenAIServing, LoRA
19 -from vllm.model_executor.guided_decoding import get_guided_decoding_logits_processor
20 +from vllm.model_executor.guided_decoding import (
21 + get_guided_decoding_logits_processor)
22
23 logger = init_logger(__name__)
24
@@ -44,9 +46,8 @@ def parse_prompt_format(prompt) -> Tuple[bool, list]:
46 prompt_is_tokens = True
47 prompts = prompt # case 4: array of token arrays
48 else:
47 - raise ValueError(
48 - "prompt must be a string, array of strings, array of tokens, or array of token arrays"
49 - )
49 + raise ValueError("prompt must be a string, array of strings, "
50 + "array of tokens, or array of token arrays")
51 return prompt_is_tokens, prompts
52
53
@@ -156,7 +157,8 @@ class OpenAIServingCompletion(OpenAIServing):
157 int, RequestOutput]] = merge_async_iterators(*generators)
158
159 # Similar to the OpenAI API, when n != best_of, we do not stream the
159 - # results. In addition, we do not stream the results when use beam search.
160 + # results. In addition, we do not stream the results when use
161 + # beam search.
162 stream = (request.stream
163 and (request.best_of is None or request.n == request.best_of)
164 and not request.use_beam_search)
@@ -223,7 +225,8 @@ class OpenAIServingCompletion(OpenAIServing):
225
226 for output in res.outputs:
227 i = output.index + prompt_idx * request.n
226 - # TODO(simon): optimize the performance by avoiding full text O(n^2) sending.
228 + # TODO(simon): optimize the performance by avoiding full
229 + # text O(n^2) sending.
230
231 if request.echo and request.max_tokens == 0:
232 # only return the prompt
@@ -231,11 +234,12 @@ class OpenAIServingCompletion(OpenAIServing):
234 delta_token_ids = res.prompt_token_ids
235 top_logprobs = res.prompt_logprobs
236 has_echoed[i] = True
234 - elif request.echo and request.max_tokens > 0 and not has_echoed[
235 - i]:
237 + elif (request.echo and request.max_tokens > 0
238 + and not has_echoed[i]):
239 # echo the prompt and first token
240 delta_text = res.prompt + output.text
238 - delta_token_ids = res.prompt_token_ids + output.token_ids
241 + delta_token_ids = (res.prompt_token_ids +
242 + output.token_ids)
243 top_logprobs = res.prompt_logprobs + (output.logprobs
244 or [])
245 has_echoed[i] = True
@@ -248,7 +252,9 @@ class OpenAIServingCompletion(OpenAIServing):
252 i]:] if output.logprobs else None
253
254 if request.logprobs is not None:
251 - assert top_logprobs is not None, "top_logprobs must be provided when logprobs is requested"
255 + assert top_logprobs is not None, (
256 + "top_logprobs must be provided when logprobs "
257 + "is requested")
258 logprobs = self._create_logprobs(
259 token_ids=delta_token_ids,
260 top_logprobs=top_logprobs,
vllm/entrypoints/openai/serving_engine.py
+8 -5
@@ -50,10 +50,12 @@ class OpenAIServing:
50 except RuntimeError:
51 event_loop = None
52
53 - if event_loop is not None and event_loop.is_running(
54 - ): # If the current is instanced by Ray Serve, there is already a running event loop
53 + if event_loop is not None and event_loop.is_running():
54 + # If the current is instanced by Ray Serve,
55 + # there is already a running event loop
56 event_loop.create_task(self._post_init())
56 - else: # When using single vLLM without engine_use_ray
57 + else:
58 + # When using single vLLM without engine_use_ray
59 asyncio.run(self._post_init())
60
61 async def _post_init(self):
@@ -178,8 +180,9 @@ class OpenAIServing:
180
181 if token_num + request.max_tokens > self.max_model_len:
182 raise ValueError(
181 - f"This model's maximum context length is {self.max_model_len} tokens. "
182 - f"However, you requested {request.max_tokens + token_num} tokens "
183 + f"This model's maximum context length is "
184 + f"{self.max_model_len} tokens. However, you requested "
185 + f"{request.max_tokens + token_num} tokens "
186 f"({token_num} in the messages, "
187 f"{request.max_tokens} in the completion). "
188 f"Please reduce the length of the messages or completion.", )
vllm/lora/layers.py
+8 -6
@@ -20,10 +20,12 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
20 RowParallelLinear,
21 QKVParallelLinear,
22 MergedColumnParallelLinear)
23 -from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding, ParallelLMHead
23 +from vllm.model_executor.layers.vocab_parallel_embedding import (
24 + VocabParallelEmbedding, ParallelLMHead)
25 from vllm.model_executor.parallel_utils.parallel_state import (
26 get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
26 -from vllm.model_executor.parallel_utils.utils import split_tensor_along_last_dim
27 +from vllm.model_executor.parallel_utils.utils import (
28 + split_tensor_along_last_dim)
29
30 if TYPE_CHECKING:
31 pass
@@ -84,7 +86,8 @@ def _apply_lora_packed_nslice(
86 lora_b_stacked: 3 element tuple of (num_loras, output_dim, lora_rank)
87 indices: (batch_size)
88 output: (batch_size, q_slice_size + 2*kv_slice_size)
87 - output_slices: n-1 element tuple of (slice_size...), where n is number of slices
89 + output_slices: n-1 element tuple of (slice_size...),
90 + where n is number of slices
91 """
92 org_output = output
93 x = x.view(-1, x.shape[-1])
@@ -819,9 +822,8 @@ class SamplerWithLoRA(BaseLayerWithLoRA):
822 ) -> None:
823 # Keep this in sync with csrc/punica/bgmv/bgmv_config.h
824 if 32000 < self.base_layer.vocab_size > 33024:
822 - raise ValueError(
823 - "When using LoRA, vocab size must be 32000 >= vocab_size <= 33024"
824 - )
825 + raise ValueError("When using LoRA, vocab size must be "
826 + "32000 >= vocab_size <= 33024")
827 self.lora_a_stacked = torch.zeros(
828 (
829 max_loras,
vllm/lora/models.py
+2 -1
@@ -13,7 +13,8 @@ from torch import nn
13 from vllm.config import LoRAConfig
14 from vllm.utils import LRUCache, in_wsl
15
16 -from vllm.lora.layers import BaseLayerWithLoRA, LoRAMapping, from_layer, from_layer_sampler
16 +from vllm.lora.layers import (BaseLayerWithLoRA, LoRAMapping, from_layer,
17 + from_layer_sampler)
18 from vllm.lora.lora import LoRALayerWeights, PackedLoRALayerWeights
19 from vllm.lora.utils import parse_fine_tuned_lora_name, replace_submodule
20
vllm/lora/worker_manager.py
+3 -4
@@ -154,10 +154,9 @@ class WorkerLoRAManager(AbstractWorkerLoRAManager):
154 f"LoRA rank {lora.rank} is greater than max_lora_rank "
155 f"{self.lora_config.max_lora_rank}.")
156 if lora.extra_vocab_size > self.lora_config.lora_extra_vocab_size:
157 - raise ValueError(
158 - f"LoRA added vocab size {lora.extra_vocab_size} is greater than "
159 - f"lora_extra_vocab_size {self.lora_config.lora_extra_vocab_size}."
160 - )
157 + raise ValueError(f"LoRA added vocab size {lora.extra_vocab_size} "
158 + f"is greater than lora_extra_vocab_size "
159 + f"{self.lora_config.lora_extra_vocab_size}.")
160 return lora
161
162 def add_dummy_lora(self, lora_request: LoRARequest, rank: int) -> bool:
vllm/model_executor/guided_decoding.py
+4 -2
@@ -8,8 +8,10 @@ from re import escape as regex_escape
8 from typing import Union, Tuple
9 from pydantic import BaseModel
10
11 -from vllm.entrypoints.openai.protocol import CompletionRequest, ChatCompletionRequest
12 -from vllm.model_executor.guided_logits_processors import JSONLogitsProcessor, RegexLogitsProcessor
11 +from vllm.entrypoints.openai.protocol import (CompletionRequest,
12 + ChatCompletionRequest)
13 +from vllm.model_executor.guided_logits_processors import (JSONLogitsProcessor,
14 + RegexLogitsProcessor)
15
16
17 class GuidedDecodingMode(Enum):
vllm/model_executor/guided_logits_processors.py
+9 -6
@@ -107,12 +107,15 @@ class JSONLogitsProcessor(RegexLogitsProcessor):
107 Parameters
108 ----------
109 schema
110 - A JSON schema that encodes the structure we want the model to generate
110 + A JSON schema that encodes the structure we want the model to
111 + generate
112 tokenizer
113 The model's tokenizer
114 whitespace_pattern
114 - Pattern to use for JSON syntactic whitespace (doesn't impact string literals)
115 - Example: allow only a single space or newline with `whitespace_pattern=r"[\n ]?"`
115 + Pattern to use for JSON syntactic whitespace (doesn't impact
116 + string literals)
117 + Example: allow only a single space or newline with
118 + `whitespace_pattern=r"[\n ]?"`
119 """
120 if isinstance(schema, type(BaseModel)):
121 schema_str = json.dumps(schema.model_json_schema())
@@ -122,8 +125,8 @@ class JSONLogitsProcessor(RegexLogitsProcessor):
125 schema_str = schema
126 else:
127 raise ValueError(
125 - f"Cannot parse schema {schema}. The schema must be either " +
126 - "a Pydantic object, a dictionary or a string that contains the JSON "
127 - + "Schema specification")
128 + f"Cannot parse schema {schema}. The schema must be either "
129 + f"a Pydantic object, a dictionary or a string that contains "
130 + f"the JSON Schema specification")
131 regex_string = build_regex_from_schema(schema_str, whitespace_pattern)
132 super().__init__(regex_string, tokenizer)
vllm/model_executor/layers/attention/attention.py
+2 -2
@@ -35,12 +35,12 @@ class Attention(nn.Module):
35 ) -> None:
36 super().__init__()
37 if _use_flash_attn():
38 - from vllm.model_executor.layers.attention.backends.flash_attn import FlashAttentionBackend
38 + from vllm.model_executor.layers.attention.backends.flash_attn import FlashAttentionBackend # noqa: E501
39 self.backend = FlashAttentionBackend(num_heads, head_size, scale,
40 num_kv_heads, alibi_slopes,
41 sliding_window)
42 else:
43 - from vllm.model_executor.layers.attention.backends.xformers import XFormersBackend
43 + from vllm.model_executor.layers.attention.backends.xformers import XFormersBackend # noqa: E501
44 self.backend = XFormersBackend(num_heads, head_size, scale,
45 num_kv_heads, alibi_slopes,
46 sliding_window)
vllm/model_executor/layers/fused_moe/fused_moe.py
+70 -37
@@ -30,9 +30,10 @@ def fused_moe_kernel(
30 K,
31 EM,
32 num_valid_tokens,
33 - # The stride variables represent how much to increase the ptr by when moving by 1
34 - # element in a particular dimension. E.g. `stride_am` is how much to increase `a_ptr`
35 - # by to get the element one row down (A has M rows).
33 + # The stride variables represent how much to increase the ptr by when
34 + # moving by 1 element in a particular dimension. E.g. `stride_am` is
35 + # how much to increase `a_ptr` by to get the element one row down
36 + # (A has M rows).
37 stride_am,
38 stride_ak,
39 stride_be,
@@ -50,17 +51,30 @@ def fused_moe_kernel(
51 compute_type: tl.constexpr,
52 ):
53 """
53 - Implements the fused computation for a Mixture of Experts (MOE) using token and expert matrices.
54 + Implements the fused computation for a Mixture of Experts (MOE) using
55 + token and expert matrices.
56
57 Key Parameters:
56 - - A: The input tensor representing tokens with shape (*, K), where '*' can be any shape representing batches and K is the feature dimension of each token.
57 - - B: The stacked MOE weight tensor with shape (E, N, K), where E is the number of experts, K is the input feature dimension, and N is the output feature dimension.
58 - - C: The output cache tensor with shape (M, topk, N), where M is the total number of tokens post padding, topk is the number of times each token is repeated,
59 - and N is the output feature dimension.
60 - - sorted_token_ids: A tensor containing the sorted indices of tokens, repeated topk times and arranged by the expert index they are assigned to.
61 - - expert_ids: A tensor containing the indices of the expert for each block. It determines which expert matrix from B should be used for each block in A.
62 - This kernel performs the multiplication of a token by its corresponding expert matrix as determined by `expert_ids`. The sorting of `sorted_token_ids`
63 - by expert index and padding ensures divisibility by BLOCK_SIZE_M, which is necessary to maintain consistency in block matrix multiplication across different blocks processed by the same expert.
58 + - A: The input tensor representing tokens with shape (*, K), where '*' can
59 + be any shape representing batches and K is the feature dimension of
60 + each token.
61 + - B: The stacked MOE weight tensor with shape (E, N, K), where E is
62 + the number of experts, K is the input feature dimension, and N is
63 + the output feature dimension.
64 + - C: The output cache tensor with shape (M, topk, N), where M is the
65 + total number of tokens post padding, topk is the number of times
66 + each token is repeated, and N is the output feature dimension.
67 + - sorted_token_ids: A tensor containing the sorted indices of tokens,
68 + repeated topk times and arranged by the expert index they are
69 + assigned to.
70 + - expert_ids: A tensor containing the indices of the expert for each
71 + block. It determines which expert matrix from B should be used for
72 + each block in A.
73 + This kernel performs the multiplication of a token by its corresponding
74 + expert matrix as determined by `expert_ids`. The sorting of
75 + `sorted_token_ids` by expert index and padding ensures divisibility by
76 + BLOCK_SIZE_M, which is necessary to maintain consistency in block matrix
77 + multiplication across different blocks processed by the same expert.
78 """
79 # -----------------------------------------------------------
80 # Map program ids `pid` to the block of C it should compute.
@@ -105,7 +119,8 @@ def fused_moe_kernel(
119 accumulator = tl.zeros((BLOCK_SIZE_M, BLOCK_SIZE_N), dtype=tl.float32)
120
121 for k in range(0, tl.cdiv(K, BLOCK_SIZE_K)):
108 - # Load the next block of A and B, generate a mask by checking the K dimension.
122 + # Load the next block of A and B, generate a mask by checking the
123 + # K dimension.
124 a = tl.load(a_ptrs,
125 mask=token_mask[:, None] &
126 (offs_k[None, :] < K - k * BLOCK_SIZE_K),
@@ -139,30 +154,41 @@ def moe_align_block_size(
154 topk_ids: torch.Tensor, block_size: int,
155 num_experts: int) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
156 """
142 - Aligns the token distribution across experts to be compatible with block size for matrix multiplication.
157 + Aligns the token distribution across experts to be compatible with block
158 + size for matrix multiplication.
159
160 Parameters:
145 - - topk_ids: A tensor of shape [total_tokens, top_k] representing the top-k expert indices for each token.
161 + - topk_ids: A tensor of shape [total_tokens, top_k] representing the
162 + top-k expert indices for each token.
163 - block_size: The block size used in block matrix multiplication.
164 - num_experts: The total number of experts.
165
166 Returns:
150 - - sorted_token_ids: A tensor containing the sorted token indices according to their allocated expert.
167 + - sorted_token_ids: A tensor containing the sorted token indices according
168 + to their allocated expert.
169 - expert_ids: A tensor indicating the assigned expert index for each block.
152 - - num_tokens_post_padded: The total number of tokens after padding, ensuring divisibility by block_size.
170 + - num_tokens_post_padded: The total number of tokens after padding,
171 + ensuring divisibility by block_size.
172
154 - This function pads the number of tokens that each expert needs to process so that it is divisible by block_size.
155 - Padding ensures that during block matrix multiplication, the dimensions align correctly.
173 + This function pads the number of tokens that each expert needs to process
174 + so that it is divisible by block_size.
175 + Padding ensures that during block matrix multiplication, the dimensions
176 + align correctly.
177
178 Example:
158 - Given topk_ids = [[2, 3, 4], [1, 2, 4], [1, 3, 4], [1, 2, 3]], block_size = 4, and num_experts = 4:
159 - - We initially have 12 tokens (after repeating 'top_k' times) and 4 experts, with each expert needing to process 3 tokens.
179 + Given topk_ids = [[2, 3, 4], [1, 2, 4], [1, 3, 4], [1, 2, 3]],
180 + block_size = 4, and num_experts = 4:
181 + - We initially have 12 tokens (after repeating 'top_k' times) and 4 experts,
182 + with each expert needing to process 3 tokens.
183 - As block_size is 4, we pad 1 token for each expert.
184 - First, flatten topk_ids to [2, 3, 4, 1, 2, 4, 1, 3, 4, 1, 2, 3].
185 - Then append padding tokens [12, 12, 12, 12] for each block.
163 - - After sorting by expert index, we obtain token_ids [3, 6, 9, 12, 0, 4, 10, 12, 1, 7, 11, 12, 2, 5, 8, 12].
164 - Tokens 12 are non-existent (padding) and are ignored in the subsequent matrix multiplication.
165 - - The padding ensures that the total number of tokens is now divisible by block_size for proper block matrix operations.
186 + - After sorting by expert index, we obtain token_ids
187 + [3, 6, 9, 12, 0, 4, 10, 12, 1, 7, 11, 12, 2, 5, 8, 12].
188 + Tokens 12 are non-existent (padding) and are ignored in
189 + the subsequent matrix multiplication.
190 + - The padding ensures that the total number of tokens is now divisible
191 + by block_size for proper block matrix operations.
192 """
193 sorted_ids = torch.empty(
194 (topk_ids.numel() + num_experts * (block_size - 1), ),
@@ -224,13 +250,14 @@ def get_moe_configs(E: int, N: int) -> Optional[Dict[int, Any]]:
250 """
251 Return optimized configurations for the fused MoE kernel.
252
227 - The return value will be a dictionary that maps an irregular grid of batch sizes
228 - to configurations of the fused_moe kernel. To evaluate the kernel on a given batch
229 - size bs, the closest batch size in the grid should be picked and the associated
230 - configuration chosen to invoke the kernel.
253 + The return value will be a dictionary that maps an irregular grid of
254 + batch sizes to configurations of the fused_moe kernel. To evaluate the
255 + kernel on a given batch size bs, the closest batch size in the grid should
256 + be picked and the associated configuration chosen to invoke the kernel.
257 """
258
233 - # First look up if an optimized configuration is available in the configs directory
259 + # First look up if an optimized configuration is available in the configs
260 + # directory
261 device_name = torch.cuda.get_device_name().replace(" ", "_")
262
263 config_file_path = os.path.join(
@@ -243,7 +270,8 @@ def get_moe_configs(E: int, N: int) -> Optional[Dict[int, Any]]:
270 # If a configuration has been found, return it
271 return {int(key): val for key, val in json.load(f).items()}
272
246 - # If no optimized configuration is available, we will use the default configuration
273 + # If no optimized configuration is available, we will use the default
274 + # configuration
275 return None
276
277
@@ -258,18 +286,22 @@ def fused_moe(
286 override_config: Optional[Dict[str, Any]] = None,
287 ) -> torch.Tensor:
288 """
261 - This function computes a Mixture of Experts (MoE) layer using two sets of weights, w1 and w2, and top-k gating mechanism.
262 -
289 + This function computes a Mixture of Experts (MoE) layer using two sets of
290 + weights, w1 and w2, and top-k gating mechanism.
291 +
292 Parameters:
293 - hidden_states (torch.Tensor): The input tensor to the MoE layer.
294 - w1 (torch.Tensor): The first set of expert weights.
295 - w2 (torch.Tensor): The second set of expert weights.
267 - - gating_output (torch.Tensor): The output of the gating operation (before softmax).
296 + - gating_output (torch.Tensor): The output of the gating operation
297 + (before softmax).
298 - topk (int): The number of top-k experts to select.
299 - renormalize (bool): If True, renormalize the top-k weights to sum to 1.
270 - - inplace (bool): If True, perform the operation in-place. Defaults to False.
271 - - override_config (Optional[Dict[str, Any]]): Optional override for the kernel configuration.
272 -
300 + - inplace (bool): If True, perform the operation in-place.
301 + Defaults to False.
302 + - override_config (Optional[Dict[str, Any]]): Optional override
303 + for the kernel configuration.
304 +
305 Returns:
306 - torch.Tensor: The output tensor after applying the MoE layer.
307 """
@@ -325,7 +357,8 @@ def fused_moe(
357 configs = get_moe_configs(E, w2.shape[2])
358
359 if configs:
328 - # If an optimal configuration map has been found, look up the optimal config
360 + # If an optimal configuration map has been found, look up the
361 + # optimal config
362 config = configs[min(configs.keys(), key=lambda x: abs(x - M))]
363 else:
364 # Else use the default config
vllm/model_executor/layers/linear.py
+8 -4
@@ -285,7 +285,8 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
285 shard_size = shard_size // param.pack_factor
286 shard_offset = shard_offset // param.pack_factor
287
288 - # If marlin, we need to adjust the offset and size to account for the tiling.
288 + # If marlin, we need to adjust the offset and size to
289 + # account for the tiling.
290 shard_size, shard_offset = adjust_marlin_shard(
291 param, shard_size, shard_offset)
292
@@ -307,7 +308,8 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
308 shard_size = shard_size // param.pack_factor
309 shard_offset = shard_offset // param.pack_factor
310
310 - # If marlin, we need to adjust the offset and size to account for the tiling.
311 + # If marlin, we need to adjust the offset and size to
312 + # account for the tiling.
313 shard_size, shard_offset = adjust_marlin_shard(
314 param, shard_size, shard_offset)
315
@@ -413,7 +415,8 @@ class QKVParallelLinear(ColumnParallelLinear):
415 shard_size = shard_size // param.pack_factor
416 shard_offset = shard_offset // param.pack_factor
417
416 - # If marlin, we need to adjust the offset and size to account for the tiling.
418 + # If marlin, we need to adjust the offset and size to
419 + # account for the tiling.
420 shard_size, shard_offset = adjust_marlin_shard(
421 param, shard_size, shard_offset)
422
@@ -442,7 +445,8 @@ class QKVParallelLinear(ColumnParallelLinear):
445 shard_size = shard_size // param.pack_factor
446 shard_offset = shard_offset // param.pack_factor
447
445 - # If marlin, we need to adjust the offset and size to account for the tiling.
448 + # If marlin, we need to adjust the offset and size to
449 + # account for the tiling.
450 shard_size, shard_offset = adjust_marlin_shard(
451 param, shard_size, shard_offset)
452
vllm/model_executor/layers/quantization/__init__.py
+2 -1
@@ -1,6 +1,7 @@
1 from typing import Type
2
3 -from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
3 +from vllm.model_executor.layers.quantization.base_config import (
4 + QuantizationConfig)
5 from vllm.model_executor.layers.quantization.awq import AWQConfig
6 from vllm.model_executor.layers.quantization.gptq import GPTQConfig
7 from vllm.model_executor.layers.quantization.squeezellm import SqueezeLLMConfig
vllm/model_executor/layers/quantization/awq.py
+4 -2
@@ -6,7 +6,8 @@ from torch.nn.parameter import Parameter
6 from vllm._C import ops
7 from vllm.model_executor.layers.linear import (LinearMethodBase,
8 set_weight_attrs)
9 -from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
9 +from vllm.model_executor.layers.quantization.base_config import (
10 + QuantizationConfig)
11
12
13 class AWQConfig(QuantizationConfig):
@@ -50,7 +51,8 @@ class AWQConfig(QuantizationConfig):
51 def get_config_filenames() -> List[str]:
52 return [
53 "quant_config.json", # E.g., casperhansen/vicuna-7b-v1.5-awq
53 - "quantize_config.json", # E.g., abhinavkulkarni/mosaicml-mpt-7b-instruct-w4-g128-awq
54 + # E.g., abhinavkulkarni/mosaicml-mpt-7b-instruct-w4-g128-awq
55 + "quantize_config.json",
56 ]
57
58 @classmethod
vllm/model_executor/layers/quantization/gptq.py
+6 -4
@@ -31,8 +31,8 @@ class GPTQConfig(QuantizationConfig):
31 self.pack_factor = Fraction(32, self.weight_bits)
32 if self.weight_bits not in [2, 3, 4, 8]:
33 raise ValueError(
34 - "Currently, only 2/3/4/8-bit weight quantization is supported for "
35 - f"GPTQ, but got {self.weight_bits} bits.")
34 + "Currently, only 2/3/4/8-bit weight quantization is "
35 + f"supported for GPTQ, but got {self.weight_bits} bits.")
36
37 def __repr__(self) -> str:
38 return (f"GPTQConfig(weight_bits={self.weight_bits}, "
@@ -101,7 +101,8 @@ class GPTQLinearMethod(LinearMethodBase):
101 "The input size is not aligned with the quantized "
102 "weight shape. This can be caused by too large "
103 "tensor parallel size.")
104 - if output_size_per_partition % self.quant_config.pack_factor.numerator != 0:
104 + if (output_size_per_partition % self.quant_config.pack_factor.numerator
105 + != 0):
106 raise ValueError(
107 "The output size is not aligned with the quantized "
108 "weight shape. This can be caused by too large "
@@ -114,7 +115,8 @@ class GPTQLinearMethod(LinearMethodBase):
115 exllama_state = ExllamaState.UNINITIALIZED
116 scale_and_zero_size = input_size // group_size
117 scale_and_zero_input_dim = None
117 - if input_size != input_size_per_partition and self.quant_config.group_size != -1:
118 + if (input_size != input_size_per_partition
119 + and self.quant_config.group_size != -1):
120 # For act-order models, we cannot use Exllama for row parallel layer
121 if self.quant_config.desc_act:
122 exllama_state = ExllamaState.UNUSED
vllm/model_executor/layers/quantization/marlin.py
+24 -15
@@ -5,7 +5,8 @@ from torch.nn.parameter import Parameter
5
6 from vllm._C import ops
7 from vllm.model_executor.layers.linear import LinearMethodBase, set_weight_attrs
8 -from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
8 +from vllm.model_executor.layers.quantization.base_config import (
9 + QuantizationConfig)
10
11
12 class MarlinConfig(QuantizationConfig):
@@ -22,8 +23,9 @@ class MarlinConfig(QuantizationConfig):
23 self.group_size = group_size
24 if self.group_size != 128 and self.group_size != -1:
25 raise ValueError(
25 - "Currently, only group size 128 and -1 (channelwise) is supported for "
26 - f"Marlin, but got group_size of {self.group_size}")
26 + "Currently, only group size 128 and -1 (channelwise) "
27 + "is supported for Marlin, but got group_size of "
28 + f"{self.group_size}")
29
30 # 4 Bits packed into 32 bit datatype.
31 self.pack_factor = 32 // 4
@@ -37,7 +39,8 @@ class MarlinConfig(QuantizationConfig):
39 # Min in_features dim
40 self.min_k_threads = 128
41
40 - # Max parallel problems to solve at once (improves large batch performance)
42 + # Max parallel problems to solve at once (improves large
43 + # batch performance)
44 self.max_parallel = 16
45
46 # Permutation length used by the marlin kernels.
@@ -102,22 +105,26 @@ class MarlinLinearMethod(LinearMethodBase):
105 # Validate output_size_per_partition
106 if output_size_per_partition % self.quant_config.min_n_threads != 0:
107 raise ValueError(
105 - f"Weight output_size_per_partition = {output_size_per_partition} is not divisible by min_n_threads = {self.quant_config.min_n_threads}."
106 - )
108 + f"Weight output_size_per_partition = "
109 + f"{output_size_per_partition} is not divisible by "
110 + f"min_n_threads = {self.quant_config.min_n_threads}.")
111 if output_size_per_partition % self.quant_config.pack_factor != 0:
112 raise ValueError(
109 - f"Weight output_size_per_partition = {output_size_per_partition} is not divisible by pack_factor = {self.quant_config.pack_factor}."
110 - )
113 + f"Weight output_size_per_partition = "
114 + f"{output_size_per_partition} is not divisible by "
115 + f"pack_factor = {self.quant_config.pack_factor}.")
116
117 # Validate input_size_per_partition
118 if input_size_per_partition % self.quant_config.min_k_threads != 0:
119 raise ValueError(
115 - f"Weight input_size_per_partition = {input_size_per_partition} is not divisible by min_k_threads = {self.quant_config.min_k_threads}."
116 - )
117 - if self.quant_config.group_size != -1 and input_size_per_partition % self.quant_config.group_size != 0:
118 - raise ValueError(
119 - f"Weight input_size_per_partition = f{input_size_per_partition} is not divisible by group_size = {self.quant_config.group_size}."
120 - )
120 + f"Weight input_size_per_partition = "
121 + f"{input_size_per_partition} is not divisible by "
122 + f"min_k_threads = {self.quant_config.min_k_threads}.")
123 + if (self.quant_config.group_size != -1 and
124 + input_size_per_partition % self.quant_config.group_size != 0):
125 + raise ValueError(f"Weight input_size_per_partition = "
126 + f"{input_size_per_partition} is not divisible by "
127 + f"group_size = {self.quant_config.group_size}.")
128
129 # Check that we have at least 4 tiles horizontally in the shard
130 num_tiles_per_perm = self.quant_config.perm_len // (
@@ -149,7 +156,9 @@ class MarlinLinearMethod(LinearMethodBase):
156 )
157
158 # Determine if channelwise or not
152 - input_groups = 1 if self.quant_config.group_size == -1 else input_size_per_partition // self.quant_config.group_size
159 + input_groups = (1 if self.quant_config.group_size == -1 else
160 + input_size_per_partition //
161 + self.quant_config.group_size)
162
163 scales = Parameter(
164 torch.empty(
vllm/model_executor/layers/quantization/squeezellm.py
+2 -1
@@ -6,7 +6,8 @@ from torch.nn.parameter import Parameter
6 from vllm._C import ops
7 from vllm.model_executor.layers.linear import (LinearMethodBase,
8 set_weight_attrs)
9 -from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
9 +from vllm.model_executor.layers.quantization.base_config import (
10 + QuantizationConfig)
11 from vllm.utils import is_hip
12
13
vllm/model_executor/layers/sampler.py
+2 -1
@@ -6,7 +6,8 @@ import torch.nn as nn
6
7 from vllm.model_executor.parallel_utils.communication_op import (
8 tensor_model_parallel_gather)
9 -from vllm.model_executor.sampling_metadata import SamplingMetadata, SamplingTensors
9 +from vllm.model_executor.sampling_metadata import (SamplingMetadata,
10 + SamplingTensors)
11 from vllm.sampling_params import SamplingParams, SamplingType
12 from vllm.sequence import (Logprob, PromptLogprobs, SampleLogprobs,
13 SamplerOutput, SequenceData, SequenceGroupOutput,
vllm/model_executor/models/baichuan.py
+2 -1
@@ -333,7 +333,8 @@ class BaiChuanBaseForCausalLM(nn.Module):
333 if "rotary_emb.inv_freq" in name:
334 continue
335 if name == "lm_head.weight":
336 - # Unlike Baichuan, Baichuan2 normalizes the head weights. Refer to:
336 + # Unlike Baichuan, Baichuan2 normalizes the head weights.
337 + # Refer to:
338 # https://huggingface.co/baichuan-inc/Baichuan2-7B-Chat/blob/84603cde5ebffb6084e476cfaeceaf0b8b91fe54/modeling_baichuan.py#L508
339 # Distinguish between Baichuan and Baichuan2 by checking the
340 # vocab size. This is suggested by
vllm/model_executor/models/deepseek.py
+5 -3
@@ -119,7 +119,8 @@ class DeepseekMoE(nn.Module):
119 linear_method=None)
120
121 if config.n_shared_experts is not None:
122 - intermediate_size = config.moe_intermediate_size * config.n_shared_experts
122 + intermediate_size = (config.moe_intermediate_size *
123 + config.n_shared_experts)
124 self.shared_experts = DeepseekMLP(
125 hidden_size=config.hidden_size,
126 intermediate_size=intermediate_size,
@@ -273,8 +274,9 @@ class DeepseekDecoderLayer(nn.Module):
274 max_position_embeddings=max_position_embeddings,
275 linear_method=linear_method,
276 )
276 - if (config.n_routed_experts is not None and \
277 - layer_idx >= config.first_k_dense_replace and layer_idx % config.moe_layer_freq == 0):
277 + if (config.n_routed_experts is not None
278 + and layer_idx >= config.first_k_dense_replace
279 + and layer_idx % config.moe_layer_freq == 0):
280 self.mlp = DeepseekMoE(config=config, linear_method=linear_method)
281 else:
282 self.mlp = DeepseekMLP(
vllm/model_executor/models/gpt_j.py
+2 -1
@@ -143,7 +143,8 @@ class GPTJBlock(nn.Module):
143 linear_method: Optional[LinearMethodBase] = None,
144 ):
145 super().__init__()
146 - inner_dim = 4 * config.n_embd if config.n_inner is None else config.n_inner
146 + inner_dim = (4 * config.n_embd
147 + if config.n_inner is None else config.n_inner)
148 self.ln_1 = nn.LayerNorm(config.n_embd, eps=config.layer_norm_epsilon)
149 self.attn = GPTJAttention(config, linear_method)
150 self.mlp = GPTJMLP(inner_dim, config, linear_method)
vllm/model_executor/models/internlm2.py
+2 -1
@@ -305,7 +305,8 @@ class InternLM2ForCausalLM(nn.Module):
305 param = params_dict[name]
306 if "wqkv" in name:
307 config = self.config
308 - kv_groups = config.num_attention_heads // config.num_key_value_heads
308 + kv_groups = (config.num_attention_heads //
309 + config.num_key_value_heads)
310 head_dim = config.hidden_size // config.num_attention_heads
311 loaded_weight = loaded_weight.view(-1, 2 + kv_groups,
312 head_dim,
vllm/model_executor/models/olmo.py
+12 -7
@@ -52,7 +52,8 @@ from vllm.model_executor.layers.linear import (
52 )
53 from vllm.model_executor.layers.rotary_embedding import get_rope
54 from vllm.model_executor.layers.sampler import Sampler
55 -from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding
55 +from vllm.model_executor.layers.vocab_parallel_embedding import (
56 + VocabParallelEmbedding)
57 from vllm.model_executor.parallel_utils.parallel_state import (
58 get_tensor_model_parallel_world_size, )
59 from vllm.model_executor.sampling_metadata import SamplingMetadata
@@ -81,7 +82,8 @@ class SwiGLU(nn.Module):
82
83 class OlmoAttention(nn.Module):
84 """
84 - This is the attention block where the output is computed as ``Attention(LN(x))`` in ``MLP(LN(x + Attention(LN(x))))``
85 + This is the attention block where the output is computed as
86 + ``Attention(LN(x))`` in ``MLP(LN(x + Attention(LN(x))))``
87 (plus another skip connection).
88 """
89
@@ -94,11 +96,12 @@ class OlmoAttention(nn.Module):
96 self.config = config
97 self.hidden_size = config.d_model
98 assert config.d_model % config.n_heads == 0
97 - tensor_model_parallel_world_size = get_tensor_model_parallel_world_size(
98 - )
99 + tensor_model_parallel_world_size = (
100 + get_tensor_model_parallel_world_size())
101 self.total_num_heads = self.config.n_heads
102 assert self.total_num_heads % tensor_model_parallel_world_size == 0
101 - self.num_heads = self.total_num_heads // tensor_model_parallel_world_size
103 + self.num_heads = (self.total_num_heads //
104 + tensor_model_parallel_world_size)
105 self.head_dim = self.hidden_size // self.total_num_heads
106
107 # Layer norms.
@@ -158,7 +161,8 @@ class OlmoAttention(nn.Module):
161
162 class OlmoMLP(nn.Module):
163 """
161 - This is the MLP block where the output is computed as ``MLP(LN(x))`` in ``MLP(LN(x + Attention(LN(x))))``
164 + This is the MLP block where the output is computed as
165 + ``MLP(LN(x))`` in ``MLP(LN(x + Attention(LN(x))))``
166 (plus another skip connection).
167 """
168
@@ -217,7 +221,8 @@ class OlmoMLP(nn.Module):
221
222 class OlmoBlock(nn.Module):
223 """
220 - This is a typical transformer block where the output is computed as ``MLP(LN(x + Attention(LN(x))))``
224 + This is a typical transformer block where the output is
225 + computed as ``MLP(LN(x + Attention(LN(x))))``
226 (plus another skip connection).
227 """
228
vllm/model_executor/models/qwen2.py
+2 -1
@@ -170,7 +170,8 @@ class Qwen2DecoderLayer(nn.Module):
170 self.hidden_size = config.hidden_size
171 # Requires transformers > 4.32.0
172 rope_theta = getattr(config, "rope_theta", 1000000)
173 - use_sliding_window = config.use_sliding_window and layer_idx < config.max_window_layers
173 + use_sliding_window = (config.use_sliding_window
174 + and layer_idx < config.max_window_layers)
175 self.self_attn = Qwen2Attention(
176 hidden_size=self.hidden_size,
177 num_heads=config.num_attention_heads,
vllm/model_executor/models/stablelm.py
+7 -6
@@ -1,5 +1,6 @@
1 # coding=utf-8
2 -# Copyright 2023 Stability AI, EleutherAI, and The HuggingFace Inc. team. All rights reserved.
2 +# Copyright 2023 Stability AI, EleutherAI, and The HuggingFace Inc. team.
3 +# All rights reserved.
4 #
5 # Licensed under the Apache License, Version 2.0 (the "License");
6 # you may not use this file except in compliance with the License.
@@ -16,7 +17,8 @@
17 # This code is based off the following work:
18 # https://huggingface.co/stabilityai/stablelm-3b-4e1t/blob/main/modeling_stablelm_epoch.py
19 # https://huggingface.co/stabilityai/stablelm-3b-4e1t/blob/main/config.json
19 -"""Inference-only StabeLM (https://github.com/Stability-AI/StableLM) model compatible with HuggingFace weights."""
20 +"""Inference-only StabeLM (https://github.com/Stability-AI/StableLM)
21 +model compatible with HuggingFace weights."""
22 from typing import List, Optional, Tuple
23
24 import torch
@@ -102,9 +104,9 @@ class StablelmAttention(nn.Module):
104 self.kv_size = self.num_key_value_heads * self.head_dim
105 self.qkv_bias = getattr(config, "use_qkv_bias", False)
106 if (self.head_dim * self.num_heads * tp_size) != self.hidden_size:
105 - raise ValueError(
106 - f"hidden_size must be divisible by num_heads (got `hidden_size`: {self.hidden_size}"
107 - f" and `num_heads`: {self.num_heads}).")
107 + raise ValueError(f"hidden_size must be divisible by num_heads "
108 + f"(got `hidden_size`: {self.hidden_size}"
109 + f" and `num_heads`: {self.num_heads}).")
110
111 self.qkv_proj = QKVParallelLinear(self.hidden_size,
112 self.head_dim,
@@ -192,7 +194,6 @@ class StableLMEpochModel(nn.Module):
194 config: PretrainedConfig,
195 linear_method: Optional[LinearMethodBase] = None) -> None:
196 super().__init__()
195 - # self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size, config.pad_token_id)
197 self.embed_tokens = VocabParallelEmbedding(
198 config.vocab_size,
199 config.hidden_size,
vllm/model_executor/models/starcoder2.py
+2 -1
@@ -35,7 +35,8 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
35 from vllm.model_executor.layers.sampler import Sampler
36 from vllm.model_executor.layers.vocab_parallel_embedding import (
37 VocabParallelEmbedding, ParallelLMHead, DEFAULT_VOCAB_PADDING_SIZE)
38 -from vllm.model_executor.parallel_utils.parallel_state import get_tensor_model_parallel_world_size
38 +from vllm.model_executor.parallel_utils.parallel_state import (
39 + get_tensor_model_parallel_world_size)
40 from vllm.model_executor.weight_utils import (default_weight_loader,
41 hf_model_weights_iterator)
42 from vllm.sequence import SamplerOutput
vllm/model_executor/neuron_model_loader.py
+2 -1
@@ -34,7 +34,8 @@ def _get_model_architecture(config: PretrainedConfig) -> Type[nn.Module]:
34
35 def get_model(model_config: ModelConfig, device_config: DeviceConfig,
36 **kwargs) -> nn.Module:
37 - from transformers_neuronx.config import NeuronConfig, ContinuousBatchingConfig
37 + from transformers_neuronx.config import (NeuronConfig,
38 + ContinuousBatchingConfig)
39
40 parallel_config = kwargs.get("parallel_config")
41 scheduler_config = kwargs.get("scheduler_config")
vllm/model_executor/parallel_utils/communication_op.py
+3 -2
@@ -11,7 +11,8 @@ from vllm.model_executor.parallel_utils.parallel_state import (
11 get_tensor_model_parallel_group,
12 is_cupy_nccl_enabled_for_all_reduce,
13 )
14 -from vllm.model_executor.parallel_utils.custom_all_reduce import custom_all_reduce
14 +from vllm.model_executor.parallel_utils.custom_all_reduce import (
15 + custom_all_reduce)
16
17
18 def tensor_model_parallel_all_reduce(input_: torch.Tensor) -> torch.Tensor:
@@ -24,7 +25,7 @@ def tensor_model_parallel_all_reduce(input_: torch.Tensor) -> torch.Tensor:
25 and GPU topology.
26
27 TLDR: always assume this function modifies its input, but use the return
27 - value as the output.
28 + value as the output.
29 """
30 # Bypass the function if we are using only 1 GPU.
31 if get_tensor_model_parallel_world_size() == 1:
vllm/model_executor/sampling_metadata.py
+2 -1
@@ -114,7 +114,8 @@ class SamplingTensors:
114 do_penalties = True
115 if (i < sampling_metadata.num_prompts
116 and sampling_params.prompt_logprobs is not None):
117 - # For tokens in the prompt that we only need to get their logprobs
117 + # For tokens in the prompt that we only need to get
118 + # their logprobs
119 prompt_len = sampling_metadata.prompt_lens[i]
120 temperatures += [temperature] * (prompt_len - 1)
121 top_ps += [top_p] * (prompt_len - 1)
vllm/sampling_params.py
+2 -2
@@ -74,8 +74,8 @@ class SamplingParams:
74 stop_token_ids: List of tokens that stop the generation when they are
75 generated. The returned output will contain the stop tokens unless
76 the stop tokens are special tokens.
77 - include_stop_str_in_output: Whether to include the stop strings in output
78 - text. Defaults to False.
77 + include_stop_str_in_output: Whether to include the stop strings in
78 + output text. Defaults to False.
79 ignore_eos: Whether to ignore the EOS token and continue generating
80 tokens after the EOS token is generated.
81 max_tokens: Maximum number of tokens to generate per output sequence.
vllm/sequence.py
+2 -1
@@ -351,7 +351,8 @@ class SequenceGroup:
351 self.metrics.first_token_time = time
352
353 def maybe_set_first_scheduled_time(self, time: float) -> None:
354 - """Sets the first scheduled time and time in queue for Request level timings."""
354 + """Sets the first scheduled time and time in queue for Request
355 + level timings."""
356 if self.metrics.first_scheduled_time is None:
357 self.metrics.first_scheduled_time = time
358 self.metrics.time_in_queue = time - self.metrics.arrival_time
vllm/spec_decode/batch_expansion.py
+18 -11
@@ -5,8 +5,12 @@ import torch
5
6 from vllm.sequence import (SamplerOutput, SequenceGroupMetadata, SequenceData)
7 from vllm.worker.worker import Worker
8 -from vllm.spec_decode.util import nvtx_range, sampler_output_to_torch, get_all_seq_ids, split_batch_by_proposal_len
9 -from vllm.spec_decode.interfaces import SpeculativeScorer, SpeculativeProposals, SpeculativeScores
8 +from vllm.spec_decode.util import (nvtx_range, sampler_output_to_torch,
9 + get_all_seq_ids,
10 + split_batch_by_proposal_len)
11 +from vllm.spec_decode.interfaces import (SpeculativeScorer,
12 + SpeculativeProposals,
13 + SpeculativeScores)
14
15 SeqId = int
16 TargetSeqId = int
@@ -68,11 +72,12 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
72 proposal_lens_list = proposals.proposal_lens.tolist()
73 proposal_token_ids_list = proposals.proposal_token_ids.tolist()
74
71 - spec_indices, non_spec_indices, target_seq_group_metadata_list, num_scoring_tokens = self._expand_batch(
72 - seq_group_metadata_list=seq_group_metadata_list,
73 - proposal_token_ids_list=proposal_token_ids_list,
74 - proposal_lens_list=proposal_lens_list,
75 - )
75 + (spec_indices, non_spec_indices, target_seq_group_metadata_list,
76 + num_scoring_tokens) = self._expand_batch(
77 + seq_group_metadata_list=seq_group_metadata_list,
78 + proposal_token_ids_list=proposal_token_ids_list,
79 + proposal_lens_list=proposal_lens_list,
80 + )
81
82 target_sampler_output = self._scorer_worker.execute_model(
83 seq_group_metadata_list=target_seq_group_metadata_list,
@@ -125,7 +130,8 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
130 num_scoring_tokens = len(target_seq_group_metadata_list)
131 target_seq_group_metadata_list.extend(non_spec_seqs)
132
128 - return spec_indices, non_spec_indices, target_seq_group_metadata_list, num_scoring_tokens
133 + return (spec_indices, non_spec_indices, target_seq_group_metadata_list,
134 + num_scoring_tokens)
135
136 def _contract_batch(self, original_bs: int,
137 target_sampler_output: List[SamplerOutput],
@@ -306,10 +312,11 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
312 # Convert non-speculative output tokens to tensors.
313 sampler_output.sampled_token_probs = non_spec_probs
314 sampler_output.sampled_token_ids = non_spec_sampled_tokens
309 - non_spec_target_token_ids, non_spec_target_probs = sampler_output_to_torch(
310 - [sampler_output])
315 + non_spec_target_token_ids, non_spec_target_probs = (
316 + sampler_output_to_torch([sampler_output]))
317
312 - return target_token_ids, target_probs, non_spec_target_token_ids, non_spec_target_probs
318 + return (target_token_ids, target_probs, non_spec_target_token_ids,
319 + non_spec_target_probs)
320
321 def _create_target_seq_id_iterator(
322 self, seq_ids: List[SeqId]) -> Iterator[TargetSeqId]:
vllm/spec_decode/multi_step_worker.py
+9 -5
@@ -5,7 +5,8 @@ import torch
5
6 from vllm.sequence import SamplerOutput, SequenceGroupMetadata
7 from vllm.worker.worker import Worker
8 -from vllm.spec_decode.interfaces import SpeculativeProposals, SpeculativeProposer
8 +from vllm.spec_decode.interfaces import (SpeculativeProposals,
9 + SpeculativeProposer)
10 from vllm.spec_decode.util import sampler_output_to_torch
11
12
@@ -247,8 +248,9 @@ class DraftModelTop1Proposer(SpeculativeProposer):
248 """
249
250 # Split speculative- and non-speculative- sequences.
250 - proposal_lens, nonzero_proposal_len_seqs, nonzero_proposal_len_indices = self._split_by_max_model_len(
251 - seq_group_metadata_list, max_proposal_len)
251 + (proposal_lens, nonzero_proposal_len_seqs,
252 + nonzero_proposal_len_indices) = self._split_by_max_model_len(
253 + seq_group_metadata_list, max_proposal_len)
254
255 if nonzero_proposal_len_seqs:
256 # Speculate tokens using the draft worker for the speculative
@@ -306,7 +308,8 @@ class DraftModelTop1Proposer(SpeculativeProposer):
308 else:
309 proposal_lens.append(0)
310
309 - return proposal_lens, nonzero_proposal_len_seqs, nonzero_proposal_len_indices
311 + return (proposal_lens, nonzero_proposal_len_seqs,
312 + nonzero_proposal_len_indices)
313
314 def _merge_outputs(
315 self,
@@ -356,7 +359,8 @@ class DraftModelTop1Proposer(SpeculativeProposer):
359 device=self._device)
360 entire_proposal_probs[nonzero_proposal_len_indices] = proposal_probs
361
359 - proposal_tokens, proposal_probs = entire_proposal_tokens, entire_proposal_probs
362 + proposal_tokens, proposal_probs = (entire_proposal_tokens,
363 + entire_proposal_probs)
364
365 proposal_lens = torch.zeros(batch_size,
366 dtype=torch.long,
vllm/spec_decode/spec_decode_worker.py
+11 -8
@@ -10,7 +10,8 @@ from vllm.worker.worker import Worker
10 from vllm.spec_decode.multi_step_worker import MultiStepWorker
11 from vllm.model_executor.layers.rejection_sampler import RejectionSampler
12 from vllm.config import CacheConfig
13 -from vllm.spec_decode.util import nvtx_range, get_all_seq_ids, split_batch_by_proposal_len
13 +from vllm.spec_decode.util import (nvtx_range, get_all_seq_ids,
14 + split_batch_by_proposal_len)
15 from vllm.spec_decode.interfaces import SpeculativeProposals, SpeculativeScores
16 from vllm.spec_decode.batch_expansion import BatchExpansionTop1Scorer
17 from vllm.spec_decode.interfaces import SpeculativeScorer
@@ -25,7 +26,7 @@ class SpecDecodeWorker:
26 LLM, after which some verification routine determines which (if any) of the
27 speculative tokens are accepted by the larger LLM.
28
28 - See https://github.com/vllm-project/vllm/pull/2188 and
29 + See https://github.com/vllm-project/vllm/pull/2188 and
30 https://github.com/vllm-project/vllm/pull/3103 for more info.
31
32 The current implementation has the following limitations:
@@ -109,10 +110,12 @@ class SpecDecodeWorker:
110 block_size, gpu_memory_utilization, cpu_swap_space,
111 cache_dtype))
112
112 - scorer_cache_block_size_bytes = self.scorer_worker.get_cache_block_size_bytes(
113 - block_size, cache_dtype)
114 - proposer_cache_block_size_bytes = self.proposer_worker.get_cache_block_size_bytes(
115 - block_size, cache_dtype)
113 + scorer_cache_block_size_bytes = (
114 + self.scorer_worker.get_cache_block_size_bytes(
115 + block_size, cache_dtype))
116 + proposer_cache_block_size_bytes = (
117 + self.proposer_worker.get_cache_block_size_bytes(
118 + block_size, cache_dtype))
119
120 new_num_gpu_blocks = split_num_cache_blocks_evenly(
121 scorer_cache_block_size_bytes, proposer_cache_block_size_bytes,
@@ -320,8 +323,8 @@ class SpecDecodeWorker:
323 sampler_output_list.append(
324 SamplerOutput(outputs=step_output_token_ids))
325
323 - maybe_rejsample_metrics = self._metrics.maybe_collect_rejsample_metrics(
324 - k)
326 + maybe_rejsample_metrics = (
327 + self._metrics.maybe_collect_rejsample_metrics(k))
328 if maybe_rejsample_metrics is not None:
329 sampler_output_list[
330 0].spec_decode_worker_metrics = maybe_rejsample_metrics
vllm/transformers_utils/configs/mpt.py
+17 -72
@@ -62,62 +62,6 @@ class MPTConfig(PretrainedConfig):
62 fc_type: str = 'torch',
63 verbose: Optional[int] = None,
64 **kwargs: Any):
65 - """The MPT configuration class.
66 - Args:
67 - d_model (int): The size of the embedding dimension of the model.
68 - n_heads (int): The number of attention heads.
69 - n_layers (int): The number of layers in the model.
70 - expansion_ratio (int): The ratio of the up/down scale in the ffn.
71 - max_seq_len (int): The maximum sequence length of the model.
72 - vocab_size (int): The size of the vocabulary.
73 - resid_pdrop (float): The dropout probability applied to the attention output before combining with residual.
74 - emb_pdrop (float): The dropout probability for the embedding layer.
75 - learned_pos_emb (bool): Whether to use learned positional embeddings
76 - attn_config (Dict): A dictionary used to configure the model's attention module:
77 - attn_type (str): type of attention to use. Options: multihead_attention, multiquery_attention, grouped_query_attention
78 - attn_pdrop (float): The dropout probability for the attention layers.
79 - attn_impl (str): The attention implementation to use. One of 'torch', 'flash', or 'triton'.
80 - qk_ln (bool): Whether to apply layer normalization to the queries and keys in the attention layer.
81 - clip_qkv (Optional[float]): If not None, clip the queries, keys, and values in the attention layer to
82 - this value.
83 - softmax_scale (Optional[float]): If not None, scale the softmax in the attention layer by this value. If None,
84 - use the default scale of ``1/sqrt(d_keys)``.
85 - prefix_lm (Optional[bool]): Whether the model should operate as a Prefix LM. This requires passing an
86 - extra `prefix_mask` argument which indicates which tokens belong to the prefix. Tokens in the prefix
87 - can attend to one another bi-directionally. Tokens outside the prefix use causal attention.
88 - attn_uses_sequence_id (Optional[bool]): Whether to restrict attention to tokens that have the same sequence_id.
89 - When the model is in `train` mode, this requires passing an extra `sequence_id` argument which indicates
90 - which sub-sequence each token belongs to.
91 - Defaults to ``False`` meaning any provided `sequence_id` will be ignored.
92 - alibi (bool): Whether to use the alibi bias instead of position embeddings.
93 - alibi_bias_max (int): The maximum value of the alibi bias.
94 - kv_n_heads (Optional[int]): For grouped_query_attention only, allow user to specify number of kv heads.
95 - ffn_config (Dict): A dictionary used to configure the model's ffn module:
96 - ffn_type (str): type of ffn to use. Options: mptmlp, te_ln_mlp
97 - init_device (str): The device to use for parameter initialization.
98 - logit_scale (Optional[Union[float, str]]): If not None, scale the logits by this value.
99 - no_bias (bool): Whether to use bias in all layers.
100 - verbose (int): The verbosity level. 0 is silent.
101 - embedding_fraction (float): The fraction to scale the gradients of the embedding layer by.
102 - norm_type (str): choose type of norm to use
103 - use_cache (bool): Whether or not the model should return the last key/values attentions
104 - init_config (Dict): A dictionary used to configure the model initialization:
105 - init_config.name: The parameter initialization scheme to use. Options: 'default_', 'baseline_',
106 - 'kaiming_uniform_', 'kaiming_normal_', 'neox_init_', 'small_init_', 'xavier_uniform_', or
107 - 'xavier_normal_'. These mimic the parameter initialization methods in PyTorch.
108 - init_div_is_residual (Union[int, float, str, bool]): Value to divide initial weights by if ``module._is_residual`` is True.
109 - emb_init_std (Optional[float]): The standard deviation of the normal distribution used to initialize the embedding layer.
110 - emb_init_uniform_lim (Optional[Union[Tuple[float, float], float]]): The lower and upper limits of the uniform distribution
111 - used to initialize the embedding layer. Mutually exclusive with ``emb_init_std``.
112 - init_std (float): The standard deviation of the normal distribution used to initialize the model,
113 - if using the baseline_ parameter initialization scheme.
114 - init_gain (float): The gain to use for parameter initialization with kaiming or xavier initialization schemes.
115 - fan_mode (str): The fan mode to use for parameter initialization with kaiming initialization schemes.
116 - init_nonlinearity (str): The nonlinearity to use for parameter initialization with kaiming initialization schemes.
117 - ---
118 - See llmfoundry.models.utils.param_init_fns.py for info on other param init config options
119 - fc_type (str): choose fc layer implementation. Options: torch and te. te layers support fp8 when using H100 GPUs.
120 - """
65 self.d_model = d_model
66 self.n_heads = n_heads
67 self.n_layers = n_layers
@@ -139,8 +83,8 @@ class MPTConfig(PretrainedConfig):
83 self.fc_type = fc_type
84 if verbose is not None:
85 warnings.warn(DeprecationWarning(
142 - 'verbose argument for MPTConfig is now ignored and will be removed. Use python_log_level instead.'
143 - ),
86 + 'verbose argument for MPTConfig is now ignored and '
87 + 'will be removed. Use python_log_level instead.'),
88 stacklevel=2)
89 if 'name' in kwargs:
90 del kwargs['name']
@@ -149,7 +93,8 @@ class MPTConfig(PretrainedConfig):
93 if self.attn_config.get('alibi', False):
94 self.learned_pos_emb = False
95 warnings.warn(
152 - f'alibi is turned on, setting `learned_pos_emb` to {self.learned_pos_emb}`',
96 + f'alibi is turned on, setting `learned_pos_emb` '
97 + f'to {self.learned_pos_emb}`',
98 stacklevel=2)
99 super().__init__(**kwargs)
100 self._validate_config()
@@ -176,8 +121,8 @@ class MPTConfig(PretrainedConfig):
121 [self.attn_config['attn_pdrop'], self.resid_pdrop, self.emb_pdrop]
122 )):
123 raise ValueError(
179 - "self.attn_config['attn_pdrop'], resid_pdrop, emb_pdrop are probabilities and must be between 0 and 1" # pylint: disable=line-too-long
180 - )
124 + "self.attn_config['attn_pdrop'], resid_pdrop, emb_pdrop are "
125 + "probabilities and must be between 0 and 1")
126 if self.attn_config['attn_impl'] not in ['torch', 'flash', 'triton']:
127 raise ValueError(
128 f"Unknown attn_impl={self.attn_config['attn_impl']}")
@@ -193,17 +138,17 @@ class MPTConfig(PretrainedConfig):
138 if self.attn_config['attn_uses_sequence_id'] and self.attn_config[
139 'attn_impl'] not in ['torch', 'triton']:
140 raise NotImplementedError(
196 - 'attn_uses_sequence_id only implemented with torch and triton attention.' # pylint: disable=line-too-long
197 - )
141 + 'attn_uses_sequence_id only implemented with torch '
142 + 'and triton attention.')
143 if self.embedding_fraction > 1 or self.embedding_fraction <= 0:
144 raise ValueError(
200 - 'model.embedding_fraction must be between 0 (exclusive) and 1 (inclusive)!' # pylint: disable=line-too-long
201 - )
145 + 'model.embedding_fraction must be between 0 (exclusive) '
146 + 'and 1 (inclusive)!')
147 if isinstance(self.logit_scale,
148 str) and self.logit_scale != 'inv_sqrt_d_model':
149 raise ValueError(
205 - f"self.logit_scale={self.logit_scale!r} is not recognized as an option; use numeric value or 'inv_sqrt_d_model'." # pylint: disable=line-too-long
206 - )
150 + f"self.logit_scale={self.logit_scale!r} is not recognized as "
151 + "an option; use numeric value or 'inv_sqrt_d_model'.")
152 if self.init_config.get('name', None) is None:
153 raise ValueError(
154 f"self.init_config={self.init_config!r} 'name' needs to be set."
@@ -219,11 +164,11 @@ class MPTConfig(PretrainedConfig):
164 del te
165 except Exception as exc:
166 raise ImportError(
222 - # pylint: disable=line-too-long
223 - 'TransformerEngine import fail. `fc_type: te` requires TransformerEngine be installed. '
224 - +
225 - 'The required version of transformer_engine also requires FlashAttention v1.0.6 is installed:\n'
226 - + 'pip install flash-attn==1.0.6 --no-build-isolation \n' +
167 + 'TransformerEngine import fail. `fc_type: te` requires '
168 + 'TransformerEngine be installed. '
169 + 'The required version of transformer_engine also requires '
170 + 'FlashAttention v1.0.6 is installed:\n'
171 + 'pip install flash-attn==1.0.6 --no-build-isolation \n'
172 'pip install git+https://github.com/NVIDIA/TransformerEngine.git@144e4888b2cdd60bd52e706d5b7a79cb9c1a7156'
173 ) from exc
174 if self.ffn_config['ffn_type'] == 'mptmlp':
vllm/transformers_utils/configs/starcoder2.py
-72
@@ -2,78 +2,6 @@ from transformers import PretrainedConfig
2
3
4 class Starcoder2Config(PretrainedConfig):
5 - r"""
6 - This is the configuration class to store the configuration of a [`Starcoder2Model`]. It is used to instantiate a
7 - Starcoder2 model according to the specified arguments, defining the model architecture. Instantiating a configuration
8 - with the defaults will yield a similar configuration to that of the [bigcode/starcoder2-7b_16k](https://huggingface.co/bigcode/starcoder2-7b_16k) model.
9 -
10 -
11 - Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
12 - documentation from [`PretrainedConfig`] for more information.
13 -
14 -
15 - Args:
16 - vocab_size (`int`, *optional*, defaults to 49152):
17 - Vocabulary size of the Starcoder2 model. Defines the number of different tokens that can be represented by the
18 - `inputs_ids` passed when calling [`Starcoder2Model`]
19 - hidden_size (`int`, *optional*, defaults to 3072):
20 - Dimension of the hidden representations.
21 - intermediate_size (`int`, *optional*, defaults to 12288):
22 - Dimension of the MLP representations.
23 - num_hidden_layers (`int`, *optional*, defaults to 30):
24 - Number of hidden layers in the Transformer encoder.
25 - num_attention_heads (`int`, *optional*, defaults to 24):
26 - Number of attention heads for each attention layer in the Transformer encoder.
27 - num_key_value_heads (`int`, *optional*, defaults to 2):
28 - This is the number of key_value heads that should be used to implement Grouped Query Attention. If
29 - `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if
30 - `num_key_value_heads=1 the model will use Multi Query Attention (MQA) otherwise GQA is used. When
31 - converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed
32 - by meanpooling all the original heads within that group. For more details checkout [this
33 - paper](https://arxiv.org/pdf/2305.13245.pdf). If it is not specified, will default to `8`.
34 - hidden_act (`str` or `function`, *optional*, defaults to `"gelu_pytorch_tanh"`):
35 - The non-linear activation function (function or string) in the decoder.
36 - max_position_embeddings (`int`, *optional*, defaults to 4096):
37 - The maximum sequence length that this model might ever be used with. Starcoder2's sliding window attention
38 - allows sequence of up to 4096*32 tokens.
39 - initializer_range (`float`, *optional*, defaults to 0.02):
40 - The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
41 - norm_epsilon (`float`, *optional*, defaults to 1e-05):
42 - Epsilon value for the layer norm
43 - use_cache (`bool`, *optional*, defaults to `True`):
44 - Whether or not the model should return the last key/values attentions (not used by all models). Only
45 - relevant if `config.is_decoder=True`.
46 - bos_token_id (`int`, *optional*, defaults to 50256):
47 - The id of the "beginning-of-sequence" token.
48 - eos_token_id (`int`, *optional*, defaults to 50256):
49 - The id of the "end-of-sequence" token.
50 - rope_theta (`float`, *optional*, defaults to 10000.0):
51 - The base period of the RoPE embeddings.
52 - sliding_window (`int`, *optional*):
53 - Sliding window attention window size. If not specified, will default to `None` (no sliding window).
54 - attention_dropout (`float`, *optional*, defaults to 0.0):
55 - The dropout ratio for the attention probabilities.
56 - residual_dropout (`float`, *optional*, defaults to 0.0):
57 - Residual connection dropout value.
58 - embedding_dropout (`float`, *optional*, defaults to 0.0):
59 - Embedding dropout.
60 - use_bias (`bool`, *optional*, defaults to `True`):
61 - Whether to use bias term on linear layers of the model.
62 -
63 -
64 - ```python
65 - >>> from transformers import Starcoder2Model, Starcoder2Config
66 -
67 - >>> # Initializing a Starcoder2 7B style configuration
68 - >>> configuration = Starcoder2Config()
69 -
70 - >>> # Initializing a model from the Starcoder2 7B style configuration
71 - >>> model = Starcoder2Model(configuration)
72 -
73 - >>> # Accessing the model configuration
74 - >>> configuration = model.config
75 - ```"""
76 -
5 model_type = "starcoder2"
6 keys_to_ignore_at_inference = ["past_key_values"]
7
vllm/transformers_utils/tokenizers/baichuan.py
+42 -50
@@ -1,4 +1,3 @@
1 -# yapf: disable
1 # Adapted from
2 # https://huggingface.co/baichuan-inc/Baichuan2-13B-Chat/blob/8f6e343d545c503b91429582231d1d354dac2740/tokenization_baichuan.py
3 # This includes a fix suggested in
@@ -13,7 +12,6 @@ import sentencepiece as spm
12 from transformers.tokenization_utils import AddedToken, PreTrainedTokenizer
13 from transformers.utils import logging
14
16 -
15 logger = logging.get_logger(__name__)
16
17 VOCAB_FILES_NAMES = {"vocab_file": "tokenizer.model"}
@@ -52,27 +50,16 @@ class BaichuanTokenizer(PreTrainedTokenizer):
50 clean_up_tokenization_spaces=False,
51 **kwargs,
52 ):
55 - self.sp_model_kwargs = {} if sp_model_kwargs is None else sp_model_kwargs
56 - bos_token = (
57 - AddedToken(bos_token, lstrip=False, rstrip=False)
58 - if isinstance(bos_token, str)
59 - else bos_token
60 - )
61 - eos_token = (
62 - AddedToken(eos_token, lstrip=False, rstrip=False)
63 - if isinstance(eos_token, str)
64 - else eos_token
65 - )
66 - unk_token = (
67 - AddedToken(unk_token, lstrip=False, rstrip=False)
68 - if isinstance(unk_token, str)
69 - else unk_token
70 - )
71 - pad_token = (
72 - AddedToken(pad_token, lstrip=False, rstrip=False)
73 - if isinstance(pad_token, str)
74 - else pad_token
75 - )
53 + self.sp_model_kwargs = ({} if sp_model_kwargs is None else
54 + sp_model_kwargs)
55 + bos_token = (AddedToken(bos_token, lstrip=False, rstrip=False)
56 + if isinstance(bos_token, str) else bos_token)
57 + eos_token = (AddedToken(eos_token, lstrip=False, rstrip=False)
58 + if isinstance(eos_token, str) else eos_token)
59 + unk_token = (AddedToken(unk_token, lstrip=False, rstrip=False)
60 + if isinstance(unk_token, str) else unk_token)
61 + pad_token = (AddedToken(pad_token, lstrip=False, rstrip=False)
62 + if isinstance(pad_token, str) else pad_token)
63 self.vocab_file = vocab_file
64 self.add_bos_token = add_bos_token
65 self.add_eos_token = add_eos_token
@@ -107,7 +94,10 @@ class BaichuanTokenizer(PreTrainedTokenizer):
94
95 def get_vocab(self):
96 """Returns vocab as a dict"""
110 - vocab = {self.convert_ids_to_tokens(i): i for i in range(self.vocab_size)}
97 + vocab = {
98 + self.convert_ids_to_tokens(i): i
99 + for i in range(self.vocab_size)
100 + }
101 vocab.update(self.added_tokens_encoder)
102 return vocab
103
@@ -130,7 +120,8 @@ class BaichuanTokenizer(PreTrainedTokenizer):
120 out_string = ""
121 prev_is_special = False
122 for i, token in enumerate(tokens):
133 - # make sure that special tokens are not decoded using sentencepiece model
123 + # make sure that special tokens are not decoded using
124 + # sentencepiece model
125 if token in self.all_special_tokens:
126 if not prev_is_special and i != 0:
127 out_string += " "
@@ -143,9 +134,9 @@ class BaichuanTokenizer(PreTrainedTokenizer):
134 out_string += self.sp_model.decode(current_sub_tokens)
135 return out_string
136
146 - def save_vocabulary(
147 - self, save_directory, filename_prefix: Optional[str] = None
148 - ) -> Tuple[str]:
137 + def save_vocabulary(self,
138 + save_directory,
139 + filename_prefix: Optional[str] = None) -> Tuple[str]:
140 """
141 Save the vocabulary and special tokens file to a directory.
142
@@ -157,24 +148,24 @@ class BaichuanTokenizer(PreTrainedTokenizer):
148 `Tuple(str)`: Paths to the files saved.
149 """
150 if not os.path.isdir(save_directory):
160 - logger.error(f"Vocabulary path ({save_directory}) should be a directory")
151 + logger.error(f"Vocabulary path ({save_directory}) "
152 + "should be a directory")
153 return
154 out_vocab_file = os.path.join(
155 save_directory,
164 - (filename_prefix + "-" if filename_prefix else "")
165 - + VOCAB_FILES_NAMES["vocab_file"],
156 + (filename_prefix + "-" if filename_prefix else "") +
157 + VOCAB_FILES_NAMES["vocab_file"],
158 )
159
160 if os.path.abspath(self.vocab_file) != os.path.abspath(
169 - out_vocab_file
170 - ) and os.path.isfile(self.vocab_file):
161 + out_vocab_file) and os.path.isfile(self.vocab_file):
162 copyfile(self.vocab_file, out_vocab_file)
163 elif not os.path.isfile(self.vocab_file):
164 with open(out_vocab_file, "wb") as fi:
165 content_spiece_model = self.sp_model.serialized_model_proto()
166 fi.write(content_spiece_model)
167
177 - return (out_vocab_file,)
168 + return (out_vocab_file, )
169
170 def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None):
171 bos_token_id = [self.bos_token_id] if self.add_bos_token else []
@@ -194,7 +185,8 @@ class BaichuanTokenizer(PreTrainedTokenizer):
185 already_has_special_tokens: bool = False,
186 ) -> List[int]:
187 """
197 - Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding
188 + Retrieve sequence ids from a token list that has no special tokens
189 + added. This method is called when adding
190 special tokens using the tokenizer `prepare_for_model` method.
191
192 Args:
@@ -202,11 +194,14 @@ class BaichuanTokenizer(PreTrainedTokenizer):
194 List of IDs.
195 token_ids_1 (`List[int]`, *optional*):
196 Optional second list of IDs for sequence pairs.
205 - already_has_special_tokens (`bool`, *optional*, defaults to `False`):
206 - Whether or not the token list is already formatted with special tokens for the model.
197 + already_has_special_tokens (`bool`, *optional*, defaults to
198 + `False`):
199 + Whether or not the token list is already formatted with
200 + special tokens for the model.
201
202 Returns:
209 - `List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
203 + `List[int]`: A list of integers in the range [0, 1]:
204 + 1 for a special token, 0 for a sequence token.
205 """
206 if already_has_special_tokens:
207 return super().get_special_tokens_mask(
@@ -220,20 +215,16 @@ class BaichuanTokenizer(PreTrainedTokenizer):
215
216 if token_ids_1 is None:
217 return bos_token_id + ([0] * len(token_ids_0)) + eos_token_id
223 - return (
224 - bos_token_id
225 - + ([0] * len(token_ids_0))
226 - + eos_token_id
227 - + bos_token_id
228 - + ([0] * len(token_ids_1))
229 - + eos_token_id
230 - )
218 + return (bos_token_id + ([0] * len(token_ids_0)) + eos_token_id +
219 + bos_token_id + ([0] * len(token_ids_1)) + eos_token_id)
220
221 def create_token_type_ids_from_sequences(
233 - self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
234 - ) -> List[int]:
222 + self,
223 + token_ids_0: List[int],
224 + token_ids_1: Optional[List[int]] = None) -> List[int]:
225 """
236 - Creates a mask from the two sequences passed to be used in a sequence-pair classification task. An ALBERT
226 + Creates a mask from the two sequences passed to be used in a
227 + sequence-pair classification task. An ALBERT
228 sequence pair mask has the following format:
229
230 ```
@@ -250,7 +241,8 @@ class BaichuanTokenizer(PreTrainedTokenizer):
241 Optional second list of IDs for sequence pairs.
242
243 Returns:
253 - `List[int]`: List of [token type IDs](../glossary#token-type-ids) according to the given sequence(s).
244 + `List[int]`: List of [token type IDs](../glossary#token-type-ids)
245 + according to the given sequence(s).
246 """
247 bos_token_id = [self.bos_token_id] if self.add_bos_token else []
248 eos_token_id = [self.eos_token_id] if self.add_eos_token else []
vllm/utils.py
+6 -6
@@ -133,9 +133,10 @@ def get_max_shared_memory_bytes(gpu: int = 0) -> int:
133 # the Neuron-X backend does not have the `cuda_utils` module.
134 from vllm._C import cuda_utils
135
136 - max_shared_mem = cuda_utils.get_max_shared_memory_per_block_device_attribute(
137 - gpu)
138 - # value 0 will cause MAX_SEQ_LEN become negative and test_attention.py will fail
136 + max_shared_mem = (
137 + cuda_utils.get_max_shared_memory_per_block_device_attribute(gpu))
138 + # value 0 will cause MAX_SEQ_LEN become negative and test_attention.py
139 + # will fail
140 assert max_shared_mem > 0, "max_shared_mem can not be zero"
141 return int(max_shared_mem)
142
@@ -209,9 +210,8 @@ def get_nvcc_cuda_version() -> Optional[Version]:
210 if not cuda_home:
211 cuda_home = '/usr/local/cuda'
212 if os.path.isfile(cuda_home + '/bin/nvcc'):
212 - logger.info(
213 - f'CUDA_HOME is not found in the environment. Using {cuda_home} as CUDA_HOME.'
214 - )
213 + logger.info(f'CUDA_HOME is not found in the environment. '
214 + f'Using {cuda_home} as CUDA_HOME.')
215 else:
216 logger.warning(
217 f'Not found nvcc in {cuda_home}. Skip cuda version check!')
vllm/worker/model_runner.py
+5 -6
@@ -93,14 +93,13 @@ class ModelRunner:
93 scheduler_config=self.scheduler_config)
94
95 self.model_memory_usage = m.consumed_memory
96 - logger.info(
97 - f"Loading model weights took {self.model_memory_usage / float(2**30):.4f} GB"
98 - )
96 + logger.info(f"Loading model weights took "
97 + f"{self.model_memory_usage / float(2**30):.4f} GB")
98
99 if self.lora_config:
101 - assert hasattr(
102 - self.model, "supported_lora_modules"
103 - ) and self.model.supported_lora_modules, "Model does not support LoRA"
100 + assert hasattr(self.model, "supported_lora_modules"
101 + ) and self.model.supported_lora_modules, (
102 + "Model does not support LoRA")
103 assert hasattr(
104 self.model,
105 "embedding_modules"), "Model does not have embedding_modules"
vllm/worker/neuron_worker.py
+4 -2
@@ -79,7 +79,8 @@ class Worker:
79 cpu_swap_space: int = 0,
80 cache_dtype: str = "float16",
81 ) -> Tuple[int, int]:
82 - """Simply returns max_num_seqs as num_gpu_blocks, 0 as num_cpu_blocks."""
82 + """Simply returns max_num_seqs as num_gpu_blocks, 0 as
83 + num_cpu_blocks."""
84 num_gpu_blocks = self.scheduler_config.max_num_seqs
85 num_cpu_blocks = 0
86 return num_gpu_blocks, num_cpu_blocks
@@ -177,7 +178,8 @@ def _init_distributed_environment(
178 "distributed_init_method must be set if torch.distributed "
179 "is not already initialized")
180 else:
180 - distributed_backend = distributed_backend if distributed_backend else "nccl"
181 + distributed_backend = (distributed_backend
182 + if distributed_backend else "nccl")
183 torch.distributed.init_process_group(
184 backend=distributed_backend,
185 world_size=parallel_config.world_size,