A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
[CI] Try introducing isort. (#3495)
SangBin Cho committed
Mar 25, 2024 at 23:59 UTC
01bfb22b4112ee813185366ab26985d172661a61
144 files changed
+506
-459
.github/workflows/ruff.yml
+5
-2
@@ -25,10 +25,13 @@ jobs:
25
- name: Install dependencies
26
run: |
27
python -m pip install --upgrade pip
28
- pip install ruff==0.1.5 codespell==2.2.6 tomli==2.0.1
28
+ pip install ruff==0.1.5 codespell==2.2.6 tomli==2.0.1 isort==5.13.2
29
- name: Analysing the code with ruff
30
run: |
31
ruff .
32
- name: Spelling check with codespell
33
run: |
34
- codespell --toml pyproject.toml
\ No newline at end of file
34
+ codespell --toml pyproject.toml
35
+ - name: Run isort
36
+ run: |
37
+ isort . --check-only
benchmarks/benchmark_prefix_caching.py
+1
-2
@@ -1,8 +1,7 @@
1
import argparse
2
import time
3
4
-from vllm import LLM
5
-from vllm import SamplingParams
4
+from vllm import LLM, SamplingParams
5
6
PROMPT = "You are a helpful assistant in recognizes the content of tables in markdown format. Here is a table as fellows. You need to answer my question about the table.\n# Table\n|Opening|Opening|Sl. No.|Film|Cast|Director|Music Director|Notes|\n|----|----|----|----|----|----|----|----|\n|J A N|9|1|Agni Pushpam|Jayabharathi, Kamalahasan|Jeassy|M. K. Arjunan||\n|J A N|16|2|Priyamvada|Mohan Sharma, Lakshmi, KPAC Lalitha|K. S. Sethumadhavan|V. Dakshinamoorthy||\n|J A N|23|3|Yakshagaanam|Madhu, Sheela|Sheela|M. S. Viswanathan||\n|J A N|30|4|Paalkkadal|Sheela, Sharada|T. K. Prasad|A. T. Ummer||\n|F E B|5|5|Amma|Madhu, Srividya|M. Krishnan Nair|M. K. Arjunan||\n|F E B|13|6|Appooppan|Thikkurissi Sukumaran Nair, Kamal Haasan|P. Bhaskaran|M. S. Baburaj||\n|F E B|20|7|Srishti|Chowalloor Krishnankutty, Ravi Alummoodu|K. T. Muhammad|M. S. Baburaj||\n|F E B|20|8|Vanadevatha|Prem Nazir, Madhubala|Yusufali Kechery|G. Devarajan||\n|F E B|27|9|Samasya|Madhu, Kamalahaasan|K. Thankappan|Shyam||\n|F E B|27|10|Yudhabhoomi|K. P. Ummer, Vidhubala|Crossbelt Mani|R. K. Shekhar||\n|M A R|5|11|Seemantha Puthran|Prem Nazir, Jayabharathi|A. B. Raj|M. K. Arjunan||\n|M A R|12|12|Swapnadanam|Rani Chandra, Dr. Mohandas|K. G. George|Bhaskar Chandavarkar||\n|M A R|19|13|Thulavarsham|Prem Nazir, sreedevi, Sudheer|N. Sankaran Nair|V. Dakshinamoorthy||\n|M A R|20|14|Aruthu|Kaviyoor Ponnamma, Kamalahasan|Ravi|G. Devarajan||\n|M A R|26|15|Swimming Pool|Kamal Haasan, M. G. Soman|J. Sasikumar|M. K. Arjunan||\n\n# Question\nWhat' s the content in the (1,1) cells\n" # noqa: E501
7
benchmarks/benchmark_serving.py
+3
-6
@@ -25,15 +25,12 @@ from datetime import datetime
25
from typing import AsyncGenerator, List, Tuple
26
27
import numpy as np
28
+from backend_request_func import (ASYNC_REQUEST_FUNCS, RequestFuncInput,
29
+ RequestFuncOutput)
30
from tqdm.asyncio import tqdm
31
from transformers import PreTrainedTokenizerBase
30
-from vllm.transformers_utils.tokenizer import get_tokenizer
32
32
-from backend_request_func import (
33
- ASYNC_REQUEST_FUNCS,
34
- RequestFuncInput,
35
- RequestFuncOutput,
36
-)
33
+from vllm.transformers_utils.tokenizer import get_tokenizer
34
35
36
@dataclass
benchmarks/benchmark_throughput.py
+1
-1
@@ -6,9 +6,9 @@ import time
6
from typing import List, Optional, Tuple
7
8
import torch
9
+from tqdm import tqdm
10
from transformers import (AutoModelForCausalLM, AutoTokenizer,
11
PreTrainedTokenizerBase)
11
-from tqdm import tqdm
12
13
14
def sample_requests(
benchmarks/kernels/benchmark_mixtral_moe.py
+3
-1
@@ -2,11 +2,13 @@ import json
2
import os
3
import sys
4
5
-from vllm.model_executor.layers.fused_moe import fused_moe, get_config_file_name
5
import torch
6
import torch.nn.functional as F
7
import triton
8
9
+from vllm.model_executor.layers.fused_moe import (fused_moe,
10
+ get_config_file_name)
11
+
12
os.environ['CUDA_VISIBLE_DEVICES'] = '0'
13
14
benchmarks/kernels/benchmark_paged_attention.py
+2
-2
@@ -1,12 +1,12 @@
1
-from typing import Optional
1
import argparse
2
import random
3
import time
4
+from typing import Optional
5
6
import torch
7
8
-from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE, create_kv_caches_with_random
8
from vllm._C import ops
9
+from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE, create_kv_caches_with_random
10
11
NUM_BLOCKS = 1024
12
PARTITION_SIZE = 512
benchmarks/kernels/benchmark_rope.py
+4
-3
@@ -1,9 +1,10 @@
1
+import argparse
2
+from itertools import accumulate
3
from typing import Optional
4
3
-import argparse
4
-import torch
5
import nvtx
6
-from itertools import accumulate
6
+import torch
7
+
8
from vllm.model_executor.layers.rotary_embedding import get_rope
9
10
cmake/hipify.py
+1
-1
@@ -9,8 +9,8 @@
9
#
10
11
import argparse
12
-import shutil
12
import os
13
+import shutil
14
15
from torch.utils.hipify.hipify_python import hipify
16
collect_env.py
+1
-1
@@ -6,10 +6,10 @@
6
# Run it with `python collect_env.py` or `python -m torch.utils.collect_env`
7
import datetime
8
import locale
9
+import os
10
import re
11
import subprocess
12
import sys
12
-import os
13
from collections import namedtuple
14
15
try:
docs/source/conf.py
+2
-1
@@ -10,10 +10,11 @@
10
# add these directories to sys.path here. If the directory is relative to the
11
# documentation root, use os.path.abspath to make it absolute, like shown here.
12
13
+import logging
14
import os
15
import sys
16
+
17
from sphinx.ext import autodoc
16
-import logging
18
19
sys.path.insert(0, os.path.abspath(os.path.join('..', '..')))
20
examples/gradio_openai_chatbot_webserver.py
+2
-1
@@ -1,6 +1,7 @@
1
import argparse
2
-from openai import OpenAI
2
+
3
import gradio as gr
4
+from openai import OpenAI
5
6
# Argument parser setup
7
parser = argparse.ArgumentParser(
examples/llm_engine_example.py
+1
-1
@@ -1,7 +1,7 @@
1
import argparse
2
from typing import List, Tuple
3
4
-from vllm import EngineArgs, LLMEngine, SamplingParams, RequestOutput
4
+from vllm import EngineArgs, LLMEngine, RequestOutput, SamplingParams
5
6
7
def create_test_prompts() -> List[Tuple[str, SamplingParams]]:
examples/multilora_inference.py
+2
-2
@@ -5,11 +5,11 @@ for offline inference.
5
Requires HuggingFace credentials for access to Llama2.
6
"""
7
8
-from typing import Optional, List, Tuple
8
+from typing import List, Optional, Tuple
9
10
from huggingface_hub import snapshot_download
11
12
-from vllm import EngineArgs, LLMEngine, SamplingParams, RequestOutput
12
+from vllm import EngineArgs, LLMEngine, RequestOutput, SamplingParams
13
from vllm.lora.request import LoRARequest
14
15
examples/offline_inference_distributed.py
+3
-1
@@ -5,11 +5,13 @@ distributively on a multi-nodes cluster.
5
Learn more about Ray Data in https://docs.ray.io/en/latest/data/data.html
6
"""
7
8
-from vllm import LLM, SamplingParams
8
from typing import Dict
9
+
10
import numpy as np
11
import ray
12
13
+from vllm import LLM, SamplingParams
14
+
15
# Create a sampling params object.
16
sampling_params = SamplingParams(temperature=0.8, top_p=0.95)
17
format.sh
+42
@@ -25,6 +25,7 @@ YAPF_VERSION=$(yapf --version | awk '{print $2}')
25
RUFF_VERSION=$(ruff --version | awk '{print $2}')
26
MYPY_VERSION=$(mypy --version | awk '{print $2}')
27
CODESPELL_VERSION=$(codespell --version)
28
+ISORT_VERSION=$(isort --vn)
29
30
# # params: tool name, tool version, required version
31
tool_version_check() {
@@ -37,6 +38,7 @@ tool_version_check() {
38
tool_version_check "yapf" $YAPF_VERSION "$(grep yapf requirements-dev.txt | cut -d'=' -f3)"
39
tool_version_check "ruff" $RUFF_VERSION "$(grep "ruff==" requirements-dev.txt | cut -d'=' -f3)"
40
tool_version_check "mypy" "$MYPY_VERSION" "$(grep mypy requirements-dev.txt | cut -d'=' -f3)"
41
+tool_version_check "isort" "$ISORT_VERSION" "$(grep isort requirements-dev.txt | cut -d'=' -f3)"
42
tool_version_check "codespell" "$CODESPELL_VERSION" "$(grep codespell requirements-dev.txt | cut -d'=' -f3)"
43
44
YAPF_FLAGS=(
@@ -178,6 +180,46 @@ else
180
lint_changed
181
fi
182
183
+# check spelling of specified files
184
+isort_check() {
185
+ isort "$@"
186
+}
187
+
188
+isort_check_all(){
189
+ isort .
190
+}
191
+
192
+# Spelling check of files that differ from main branch.
193
+isort_check_changed() {
194
+ # The `if` guard ensures that the list of filenames is not empty, which
195
+ # could cause ruff to receive 0 positional arguments, making it hang
196
+ # waiting for STDIN.
197
+ #
198
+ # `diff-filter=ACM` and $MERGEBASE is to ensure we only lint files that
199
+ # exist on both branches.
200
+ MERGEBASE="$(git merge-base origin/main HEAD)"
201
+
202
+ if ! git diff --diff-filter=ACM --quiet --exit-code "$MERGEBASE" -- '*.py' '*.pyi' &>/dev/null; then
203
+ git diff --name-only --diff-filter=ACM "$MERGEBASE" -- '*.py' '*.pyi' | xargs \
204
+ isort
205
+ fi
206
+}
207
+
208
+# Run Isort
209
+# This flag runs spell check of individual files. --files *must* be the first command line
210
+# arg to use this option.
211
+if [[ "$1" == '--files' ]]; then
212
+ isort_check "${@:2}"
213
+ # If `--all` is passed, then any further arguments are ignored and the
214
+ # entire python directory is linted.
215
+elif [[ "$1" == '--all' ]]; then
216
+ isort_check_all
217
+else
218
+ # Check spelling only of the files that changed in last commit.
219
+ isort_check_changed
220
+fi
221
+echo 'vLLM isort: Done'
222
+
223
if ! git diff --quiet &>/dev/null; then
224
echo 'Reformatted files. Please review and stage the changes.'
225
echo 'Changes not staged for commit:'
pyproject.toml
+4
@@ -51,3 +51,7 @@ exclude = "vllm/model_executor/parallel_utils/|vllm/model_executor/models/"
51
[tool.codespell]
52
ignore-words-list = "dout, te, indicies"
53
skip = "./tests/prompts"
54
+
55
+[tool.isort]
56
+use_parentheses = true
57
+skip_gitignore = true
requirements-dev.txt
+1
@@ -4,6 +4,7 @@ toml==0.10.2
4
tomli==2.0.1
5
ruff==0.1.5
6
codespell==2.2.6
7
+isort==5.13.2
8
9
# type checking
10
mypy==0.991
setup.py
+5
-5
@@ -1,16 +1,16 @@
1
import io
2
+import logging
3
import os
4
import re
4
-import logging
5
import subprocess
6
import sys
7
+from shutil import which
8
from typing import List
9
9
-from packaging.version import parse, Version
10
-from setuptools import setup, find_packages, Extension
11
-from setuptools.command.build_ext import build_ext
12
-from shutil import which
10
import torch
11
+from packaging.version import Version, parse
12
+from setuptools import Extension, find_packages, setup
13
+from setuptools.command.build_ext import build_ext
14
from torch.utils.cpp_extension import CUDA_HOME
15
16
ROOT_DIR = os.path.dirname(__file__)
tests/async_engine/test_chat_template.py
+3
-3
@@ -1,12 +1,12 @@
1
-from dataclasses import dataclass
1
import os
2
import pathlib
3
+from dataclasses import dataclass
4
5
import pytest
6
7
-from vllm.transformers_utils.tokenizer import get_tokenizer
8
-from vllm.entrypoints.openai.serving_chat import OpenAIServingChat
7
from vllm.entrypoints.openai.protocol import ChatCompletionRequest
8
+from vllm.entrypoints.openai.serving_chat import OpenAIServingChat
9
+from vllm.transformers_utils.tokenizer import get_tokenizer
10
11
chatml_jinja_path = pathlib.Path(os.path.dirname(os.path.abspath(
12
__file__))).parent.parent / "examples/template_chatml.jinja"
tests/conftest.py
+1
-1
@@ -6,8 +6,8 @@ import torch
6
from transformers import AutoModelForCausalLM
7
8
from vllm import LLM, SamplingParams
9
-from vllm.transformers_utils.tokenizer import get_tokenizer
9
from vllm.config import TokenizerPoolConfig
10
+from vllm.transformers_utils.tokenizer import get_tokenizer
11
12
_TEST_DIR = os.path.dirname(__file__)
13
_TEST_PROMPTS = [os.path.join(_TEST_DIR, "prompts", "example.txt")]
tests/core/test_block_manager.py
+5
-4
@@ -1,13 +1,14 @@
1
-import pytest
1
import time
2
from typing import List
3
4
+import pytest
5
+
6
from vllm import SamplingParams
7
from vllm.block import PhysicalTokenBlock
7
-from vllm.core.block_manager import (UncachedBlockAllocator, BlockSpaceManager,
8
- AllocStatus)
8
+from vllm.core.block_manager import (AllocStatus, BlockSpaceManager,
9
+ UncachedBlockAllocator)
10
+from vllm.sequence import Logprob, Sequence, SequenceGroup, SequenceStatus
11
from vllm.utils import Device
10
-from vllm.sequence import Sequence, SequenceGroup, SequenceStatus, Logprob
12
13
from .utils import create_dummy_prompt
14
tests/core/test_scheduler.py
+3
-2
@@ -1,10 +1,11 @@
1
+import time
2
from typing import List
3
+
4
import pytest # noqa
3
-import time
5
6
from vllm.config import CacheConfig, SchedulerConfig
7
from vllm.core.scheduler import Scheduler
7
-from vllm.sequence import SequenceGroup, Logprob
8
+from vllm.sequence import Logprob, SequenceGroup
9
10
from .utils import create_dummy_prompt
11
tests/distributed/test_comm_ops.py
+3
-5
@@ -3,14 +3,12 @@
3
Run `pytest tests/distributed/test_comm_ops.py --forked`.
4
"""
5
import pytest
6
-import torch
6
import ray
7
+import torch
8
9
from vllm.model_executor.parallel_utils.communication_op import (
10
- tensor_model_parallel_all_reduce,
11
- tensor_model_parallel_all_gather,
12
- broadcast_tensor_dict,
13
-)
10
+ broadcast_tensor_dict, tensor_model_parallel_all_gather,
11
+ tensor_model_parallel_all_reduce)
12
from vllm.test_utils import (init_test_distributed_environment,
13
multi_process_tensor_parallel)
14
tests/distributed/test_custom_all_reduce.py
+1
-1
@@ -1,6 +1,6 @@
1
+import os
2
import random
3
3
-import os
4
import pytest
5
import ray
6
import torch
tests/entrypoints/test_guided_processors.py
+3
-3
@@ -1,11 +1,11 @@
1
# This unit test should be moved to a new
2
# tests/test_guided_decoding directory.
3
4
-from transformers import AutoTokenizer
4
import torch
5
+from transformers import AutoTokenizer
6
7
-from vllm.model_executor.guided_logits_processors import (RegexLogitsProcessor,
8
- JSONLogitsProcessor)
7
+from vllm.model_executor.guided_logits_processors import (JSONLogitsProcessor,
8
+ RegexLogitsProcessor)
9
10
TEST_SCHEMA = {
11
"type": "object",
tests/entrypoints/test_openai_server.py
+7
-8
@@ -1,22 +1,21 @@
1
+# imports for guided decoding tests
2
+import json
3
import os
4
+import re
5
import subprocess
6
+import sys
7
import time
8
5
-import sys
9
+import jsonschema
10
+import openai # use the official client for correctness check
11
import pytest
7
-import requests
12
# using Ray for overall ease of process management, parallel requests,
13
# and debugging.
14
import ray
11
-import openai # use the official client for correctness check
15
+import requests
16
# downloading lora to test lora requests
17
from huggingface_hub import snapshot_download
18
15
-# imports for guided decoding tests
16
-import json
17
-import jsonschema
18
-import re
19
-
19
from vllm.transformers_utils.tokenizer import get_tokenizer
20
21
MAX_SERVER_START_WAIT_S = 600 # wait for server to start for 60 seconds
tests/kernels/conftest.py
+1
@@ -1,4 +1,5 @@
1
import pytest
2
+
3
from vllm.utils import create_kv_caches_with_random
4
5
tests/kernels/test_activation.py
+1
-1
@@ -2,10 +2,10 @@ from typing import Type
2
3
import pytest
4
import torch
5
+from allclose_default import get_default_atol, get_default_rtol
6
7
from vllm.model_executor.layers.activation import (FastGELU, GeluAndMul,
8
NewGELU, SiluAndMul)
8
-from allclose_default import get_default_atol, get_default_rtol
9
10
DTYPES = [torch.half, torch.bfloat16, torch.float]
11
NUM_TOKENS = [7, 83, 2048] # Arbitrary values for testing
tests/kernels/test_attention.py
+3
-4
@@ -3,13 +3,12 @@ from typing import List, Optional, Tuple
3
4
import pytest
5
import torch
6
+from allclose_default import get_default_atol, get_default_rtol
7
from xformers import ops as xops
8
from xformers.ops.fmha.attn_bias import BlockDiagonalCausalMask
9
9
-from vllm._C import ops, cache_ops
10
-from vllm.utils import get_max_shared_memory_bytes
11
-from vllm.utils import is_hip
12
-from allclose_default import get_default_atol, get_default_rtol
10
+from vllm._C import cache_ops, ops
11
+from vllm.utils import get_max_shared_memory_bytes, is_hip
12
13
FLOAT32_BYTES = torch.finfo(torch.float).bits // 8
14
# This will change depending on the compute capability.
tests/kernels/test_cache.py
+1
-2
@@ -1,10 +1,9 @@
1
import random
2
+from typing import Tuple
3
4
import pytest
5
import torch
6
6
-from typing import Tuple
7
-
7
from vllm._C import cache_ops
8
9
COPYING_DIRECTION = [('cuda', 'cpu'), ('cuda', 'cuda'), ('cpu', 'cuda')]
tests/kernels/test_moe.py
+1
-1
@@ -7,8 +7,8 @@ import torch
7
from transformers import MixtralConfig
8
from transformers.models.mixtral.modeling_mixtral import MixtralSparseMoeBlock
9
10
-from vllm.model_executor.layers.fused_moe import fused_moe
10
from vllm.model_executor.layers.activation import SiluAndMul
11
+from vllm.model_executor.layers.fused_moe import fused_moe
12
from vllm.model_executor.models.mixtral import MixtralMoE
13
14
tests/kernels/test_pos_encoding.py
+2
-1
@@ -1,9 +1,10 @@
1
+from itertools import accumulate
2
from typing import List, Optional
3
4
import pytest
5
import torch
6
from allclose_default import get_default_atol, get_default_rtol
6
-from itertools import accumulate
7
+
8
from vllm.model_executor.layers.rotary_embedding import get_rope
9
10
IS_NEOX_STYLE = [True, False]
tests/kernels/test_prefix_prefill.py
+3
-2
@@ -1,12 +1,13 @@
1
import random
2
-import pytest
2
import time
3
4
+import pytest
5
import torch
6
-from vllm.attention.ops.prefix_prefill import context_attention_fwd
6
from xformers import ops as xops
7
from xformers.ops.fmha.attn_bias import BlockDiagonalCausalFromBottomRightMask
8
9
+from vllm.attention.ops.prefix_prefill import context_attention_fwd
10
+
11
NUM_HEADS = [64]
12
NUM_QUERIES_PER_KV = [1, 8, 64]
13
HEAD_SIZES = [128]
tests/kernels/test_rand.py
+3
-2
@@ -1,7 +1,8 @@
1
-import torch
2
-import pytest
1
import random
2
3
+import pytest
4
+import torch
5
+
6
from vllm.model_executor.layers.ops.rand import seeded_uniform
7
from vllm.model_executor.utils import set_random_seed
8
tests/kernels/test_sampler.py
+4
-4
@@ -1,15 +1,15 @@
1
import gc
2
3
-import torch
3
import pytest
4
+import torch
5
import triton
6
import triton.language as tl
7
8
from vllm.model_executor.layers.ops.sample import (
9
- _uniform_to_exponential, sample, get_num_triton_sampler_splits,
10
- MAX_TRITON_N_COLS)
11
-from vllm.model_executor.utils import set_random_seed
9
+ MAX_TRITON_N_COLS, _uniform_to_exponential, get_num_triton_sampler_splits,
10
+ sample)
11
from vllm.model_executor.sampling_metadata import SamplingTensors
12
+from vllm.model_executor.utils import set_random_seed
13
14
SINGLE_SPLIT_VOCAB_SIZE = 32000 # llama/mistral/mixtral vocab size
15
MULTI_SPLIT_VOCAB_SIZE = MAX_TRITON_N_COLS + 100
tests/lora/conftest.py
+4
-4
@@ -2,7 +2,7 @@ import contextlib
2
import gc
3
import tempfile
4
from collections import OrderedDict
5
-from unittest.mock import patch, MagicMock
5
+from unittest.mock import MagicMock, patch
6
7
import pytest
8
import ray
@@ -12,13 +12,13 @@ from huggingface_hub import snapshot_download
12
13
import vllm
14
from vllm.config import LoRAConfig
15
-from vllm.model_executor.layers.sampler import Sampler
16
-from vllm.model_executor.layers.logits_processor import LogitsProcessor
17
-from vllm.model_executor.model_loader import get_model
15
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
16
MergedColumnParallelLinear,
17
RowParallelLinear)
18
+from vllm.model_executor.layers.logits_processor import LogitsProcessor
19
+from vllm.model_executor.layers.sampler import Sampler
20
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
21
+from vllm.model_executor.model_loader import get_model
22
from vllm.model_executor.parallel_utils.parallel_state import (
23
destroy_model_parallel, initialize_model_parallel)
24
tests/lora/test_layer_variation.py
+4
-2
@@ -1,12 +1,14 @@
1
+import tempfile
2
+from random import sample
3
from typing import List, Optional
4
+
5
import peft
6
import pytest
4
-from random import sample
5
-import tempfile
7
from transformers import AutoModelForCausalLM
8
9
import vllm
10
from vllm.lora.request import LoRARequest
11
+
12
from .conftest import cleanup
13
14
MODEL_PATH = "Felladrin/Llama-68M-Chat-v1"
tests/lora/test_layers.py
+14
-18
@@ -1,32 +1,28 @@
1
-import pytest
1
import random
2
from copy import deepcopy
3
from dataclasses import dataclass
5
-from typing import List, Optional, Dict, Tuple
4
+from typing import Dict, List, Optional, Tuple
5
6
+import pytest
7
import torch
8
import torch.nn.functional as F
9
10
-from vllm.lora.layers import (
11
- ColumnParallelLinearWithLoRA,
12
- MergedColumnParallelLinearWithLoRA,
13
- QKVParallelLinearWithLora,
14
- VocabParallelEmbeddingWithLoRA,
15
- RowParallelLinearWithLoRA,
16
- LogitsProcessorWithLoRA,
17
- LoRAMapping,
18
- BaseLayerWithLoRA,
19
-)
20
-from vllm.lora.models import (LoRALayerWeights, convert_mapping,
21
- PackedLoRALayerWeights)
10
from vllm.config import LoRAConfig
23
-from vllm.model_executor.layers.logits_processor import LogitsProcessor
11
+from vllm.lora.layers import (BaseLayerWithLoRA, ColumnParallelLinearWithLoRA,
12
+ LogitsProcessorWithLoRA, LoRAMapping,
13
+ MergedColumnParallelLinearWithLoRA,
14
+ QKVParallelLinearWithLora,
15
+ RowParallelLinearWithLoRA,
16
+ VocabParallelEmbeddingWithLoRA)
17
+from vllm.lora.models import (LoRALayerWeights, PackedLoRALayerWeights,
18
+ convert_mapping)
19
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
20
MergedColumnParallelLinear,
26
- RowParallelLinear,
27
- QKVParallelLinear)
21
+ QKVParallelLinear,
22
+ RowParallelLinear)
23
+from vllm.model_executor.layers.logits_processor import LogitsProcessor
24
from vllm.model_executor.layers.vocab_parallel_embedding import (
29
- VocabParallelEmbedding, ParallelLMHead)
25
+ ParallelLMHead, VocabParallelEmbedding)
26
from vllm.model_executor.utils import set_random_seed
27
28
from .utils import DummyLoRAManager
tests/lora/test_llama.py
+1
@@ -3,6 +3,7 @@ import ray
3
4
import vllm
5
from vllm.lora.request import LoRARequest
6
+
7
from .conftest import cleanup
8
9
MODEL_PATH = "meta-llama/Llama-2-7b-hf"
tests/lora/test_lora_manager.py
+4
-4
@@ -8,11 +8,11 @@ from torch import nn
8
9
from vllm.config import LoRAConfig
10
from vllm.lora.layers import (ColumnParallelLinearWithLoRA,
11
- RowParallelLinearWithLoRA,
12
- MergedColumnParallelLinearWithLoRA)
11
+ MergedColumnParallelLinearWithLoRA,
12
+ RowParallelLinearWithLoRA)
13
from vllm.lora.lora import LoRALayerWeights, PackedLoRALayerWeights
14
-from vllm.lora.models import (LoRAModel, LoRAModelManager,
15
- LRUCacheLoRAModelManager, LoRAMapping)
14
+from vllm.lora.models import (LoRAMapping, LoRAModel, LoRAModelManager,
15
+ LRUCacheLoRAModelManager)
16
from vllm.lora.request import LoRARequest
17
from vllm.lora.worker_manager import (LRUCacheWorkerLoRAManager,
18
WorkerLoRAManager)
tests/lora/test_tokenizer_group.py
+3
-1
@@ -1,8 +1,10 @@
1
import pytest
2
from transformers import AutoTokenizer, PreTrainedTokenizerBase
3
+
4
from vllm.lora.request import LoRARequest
4
-from vllm.transformers_utils.tokenizer_group import get_tokenizer_group
5
from vllm.transformers_utils.tokenizer import get_lora_tokenizer
6
+from vllm.transformers_utils.tokenizer_group import get_tokenizer_group
7
+
8
from ..conftest import get_tokenizer_pool_config
9
10
tests/lora/test_utils.py
+1
-1
@@ -2,8 +2,8 @@ from collections import OrderedDict
2
3
from torch import nn
4
5
+from vllm.lora.utils import parse_fine_tuned_lora_name, replace_submodule
6
from vllm.utils import LRUCache
6
-from vllm.lora.utils import (parse_fine_tuned_lora_name, replace_submodule)
7
8
9
def test_parse_fine_tuned_lora_name():
tests/lora/test_worker.py
+2
-2
@@ -3,10 +3,10 @@ import random
3
import tempfile
4
from unittest.mock import patch
5
6
+from vllm.config import (DeviceConfig, LoRAConfig, ModelConfig, ParallelConfig,
7
+ SchedulerConfig)
8
from vllm.lora.models import LoRAMapping
9
from vllm.lora.request import LoRARequest
8
-from vllm.config import (ModelConfig, ParallelConfig, SchedulerConfig,
9
- DeviceConfig, LoRAConfig)
10
from vllm.worker.worker import Worker
11
12
tests/models/test_marlin.py
+3
-1
@@ -11,9 +11,11 @@ up to 3 times to see if we pass.
11
Run `pytest tests/models/test_marlin.py --forked`.
12
"""
13
14
+from dataclasses import dataclass
15
+
16
import pytest
17
import torch
16
-from dataclasses import dataclass
18
+
19
from vllm.model_executor.layers.quantization import (
20
_QUANTIZATION_CONFIG_REGISTRY)
21
tests/samplers/test_logprobs.py
+1
-1
@@ -1,7 +1,7 @@
1
import pytest
2
import torch
3
-from tests.conftest import VllmRunner
3
4
+from tests.conftest import VllmRunner
5
from vllm import SamplingParams
6
7
MODELS = ["facebook/opt-125m"]
tests/samplers/test_rejection_sampler.py
+2
-3
@@ -1,13 +1,12 @@
1
"""Tests for rejection sampling."""
2
-import pytest
2
from typing import List, Tuple
3
4
+import pytest
5
import torch
6
import torch.nn.functional as F
7
8
-from vllm.model_executor.utils import set_random_seed
9
-
8
from vllm.model_executor.layers.rejection_sampler import RejectionSampler
9
+from vllm.model_executor.utils import set_random_seed
10
11
CUDA_DEVICES = [
12
f"cuda:{i}" for i in range(1 if torch.cuda.device_count() == 1 else 2)
tests/samplers/test_sampler.py
+1
-2
@@ -1,11 +1,10 @@
1
import random
2
-from typing import Tuple, List
2
+from typing import List, Optional, Tuple
3
from unittest.mock import patch
4
5
import pytest
6
import torch
7
from transformers import GenerationConfig, GenerationMixin
8
-from typing import Optional
8
9
from vllm.model_executor.layers.sampler import Sampler
10
from vllm.model_executor.utils import set_random_seed
tests/samplers/test_seeded_generate.py
+1
-1
@@ -8,8 +8,8 @@ from itertools import combinations
8
9
import pytest
10
11
-from vllm.model_executor.utils import set_random_seed
11
from vllm import SamplingParams
12
+from vllm.model_executor.utils import set_random_seed
13
14
MODEL = "facebook/opt-125m"
15
RANDOM_SEEDS = list(range(5))
tests/spec_decode/test_batch_expansion.py
+2
-2
@@ -1,9 +1,9 @@
1
-import torch
1
import pytest
2
+import torch
3
4
from vllm.spec_decode.batch_expansion import BatchExpansionTop1Scorer
5
6
-from .utils import mock_worker, create_seq_group_metadata_from_prompts
6
+from .utils import create_seq_group_metadata_from_prompts, mock_worker
7
8
9
@pytest.mark.parametrize('num_target_seq_ids', [100])
tests/spec_decode/test_metrics.py
+3
-3
@@ -1,9 +1,9 @@
1
-import torch
1
import math
3
-import pytest
4
-
2
from unittest.mock import MagicMock
3
4
+import pytest
5
+import torch
6
+
7
from vllm.spec_decode.metrics import AsyncMetricsCollector
8
9
tests/spec_decode/test_multi_step_worker.py
+10
-9
@@ -1,18 +1,19 @@
1
-import torch
1
import random
3
-import pytest
2
from unittest.mock import MagicMock
3
6
-from vllm.spec_decode.multi_step_worker import (MultiStepWorker,
7
- DraftModelTop1Proposer)
8
-from vllm.worker.worker import Worker
4
+import pytest
5
+import torch
6
+
7
from vllm.model_executor.utils import set_random_seed
8
from vllm.sequence import SamplerOutput
9
+from vllm.spec_decode.multi_step_worker import (DraftModelTop1Proposer,
10
+ MultiStepWorker)
11
+from vllm.worker.worker import Worker
12
12
-from .utils import (create_execute_model_data, create_worker,
13
- create_seq_group_metadata_from_prompts, zero_kv_cache,
14
- patch_execute_model_with_seeds,
15
- assert_logprobs_dict_allclose, create_batch)
13
+from .utils import (assert_logprobs_dict_allclose, create_batch,
14
+ create_execute_model_data,
15
+ create_seq_group_metadata_from_prompts, create_worker,
16
+ patch_execute_model_with_seeds, zero_kv_cache)
17
18
19
@pytest.mark.parametrize('num_steps', list(range(1, 17)))
tests/spec_decode/test_spec_decode_worker.py
+11
-9
@@ -1,18 +1,20 @@
1
-import torch
1
import random
3
-import pytest
2
from unittest.mock import MagicMock
3
4
+import pytest
5
+import torch
6
+
7
+from vllm.model_executor.layers.rejection_sampler import RejectionSampler
8
+from vllm.model_executor.utils import set_random_seed
9
+from vllm.spec_decode.interfaces import SpeculativeProposals
10
+from vllm.spec_decode.metrics import (AsyncMetricsCollector,
11
+ SpecDecodeWorkerMetrics)
12
from vllm.spec_decode.multi_step_worker import MultiStepWorker
13
from vllm.spec_decode.spec_decode_worker import (SpecDecodeWorker,
14
split_num_cache_blocks_evenly)
9
-from vllm.spec_decode.interfaces import SpeculativeProposals
10
-from vllm.model_executor.utils import set_random_seed
11
-from vllm.model_executor.layers.rejection_sampler import RejectionSampler
12
-from .utils import (mock_worker, create_batch, ExecuteModelData,
13
- create_sampler_output_list)
14
-from vllm.spec_decode.metrics import (SpecDecodeWorkerMetrics,
15
- AsyncMetricsCollector)
15
+
16
+from .utils import (ExecuteModelData, create_batch, create_sampler_output_list,
17
+ mock_worker)
18
19
20
@pytest.mark.parametrize('k', [1, 2, 6])
tests/spec_decode/test_utils.py
+4
-4
@@ -1,9 +1,9 @@
1
-from vllm.spec_decode.util import get_all_seq_ids
2
-from vllm.sequence import SequenceGroupMetadata
3
-from vllm.spec_decode.util import split_batch_by_proposal_len
1
+from unittest.mock import MagicMock
2
3
import pytest
6
-from unittest.mock import MagicMock
4
+
5
+from vllm.sequence import SequenceGroupMetadata
6
+from vllm.spec_decode.util import get_all_seq_ids, split_batch_by_proposal_len
7
8
9
def test_get_all_seq_ids():
tests/spec_decode/utils.py
+11
-9
@@ -1,17 +1,19 @@
1
-import torch
2
-from typing import List, Optional, Dict, Iterable, Union
1
+from dataclasses import dataclass, fields
2
+from itertools import count
3
+from typing import Dict, Iterable, List, Optional, Union
4
from unittest.mock import MagicMock
5
5
-from vllm.worker.worker import Worker
6
-from vllm.utils import get_distributed_init_method, get_ip, get_open_port
6
+import torch
7
+
8
from vllm.engine.arg_utils import EngineArgs
8
-from vllm.sequence import (Logprob, SequenceGroupMetadata, SequenceData,
9
- SamplerOutput, SequenceGroupOutput, SequenceOutput)
9
+from vllm.model_executor.utils import set_random_seed
10
from vllm.sampling_params import SamplingParams
11
+from vllm.sequence import (Logprob, SamplerOutput, SequenceData,
12
+ SequenceGroupMetadata, SequenceGroupOutput,
13
+ SequenceOutput)
14
+from vllm.utils import get_distributed_init_method, get_ip, get_open_port
15
from vllm.worker.cache_engine import CacheEngine
12
-from vllm.model_executor.utils import set_random_seed
13
-from itertools import count
14
-from dataclasses import dataclass, fields
16
+from vllm.worker.worker import Worker
17
18
19
@dataclass
tests/test_cache_block_hashing.py
+1
-1
@@ -7,8 +7,8 @@ from typing import List, Optional
7
import pytest
8
9
from vllm.lora.request import LoRARequest
10
-from vllm.transformers_utils.tokenizer_group import TokenizerGroup
10
from vllm.sequence import Sequence
11
+from vllm.transformers_utils.tokenizer_group import TokenizerGroup
12
13
# Make two prefixes with different first blocks.
14
prefix_start = [("You are an expert"), ("You are a")]
tests/test_sequence.py
+1
-1
@@ -1,6 +1,6 @@
1
import pytest
2
3
-from vllm.sequence import SequenceGroupOutput, SamplerOutput, SequenceOutput
3
+from vllm.sequence import SamplerOutput, SequenceGroupOutput, SequenceOutput
4
5
6
@pytest.fixture
tests/tokenization/test_cached_tokenizer.py
+3
-1
@@ -1,7 +1,9 @@
1
from copy import deepcopy
2
-from vllm.transformers_utils.tokenizer import get_cached_tokenizer
2
+
3
from transformers import AutoTokenizer
4
5
+from vllm.transformers_utils.tokenizer import get_cached_tokenizer
6
+
7
8
def test_cached_tokenizer():
9
reference_tokenizer = AutoTokenizer.from_pretrained("gpt2")
tests/tokenization/test_detokenize.py
+5
-5
@@ -1,12 +1,12 @@
1
-import pytest
1
+from typing import Dict, List
2
3
+import pytest
4
from transformers import AutoTokenizer
4
-from typing import List, Dict
5
6
-from vllm.sequence import Sequence, Logprob, SamplingParams, SequenceGroup
7
-from vllm.transformers_utils.tokenizer_group import get_tokenizer_group
8
-from vllm.transformers_utils.tokenizer import detokenize_incrementally
6
+from vllm.sequence import Logprob, SamplingParams, Sequence, SequenceGroup
7
from vllm.transformers_utils.detokenizer import Detokenizer
8
+from vllm.transformers_utils.tokenizer import detokenize_incrementally
9
+from vllm.transformers_utils.tokenizer_group import get_tokenizer_group
10
11
TRUTH = [
12
"Hello here, this is a simple test",
tests/tokenization/test_tokenizer_group.py
+4
-2
@@ -1,14 +1,16 @@
1
-import os
2
-import pytest
1
import asyncio
2
+import os
3
from unittest.mock import patch
4
5
+import pytest
6
from transformers import AutoTokenizer, PreTrainedTokenizerBase
7
+
8
from vllm.transformers_utils.tokenizer_group import get_tokenizer_group
9
from vllm.transformers_utils.tokenizer_group.ray_tokenizer_group import (
10
RayTokenizerGroupPool)
11
from vllm.transformers_utils.tokenizer_group.tokenizer_group import (
12
TokenizerGroup)
13
+
14
from ..conftest import get_tokenizer_pool_config
15
16
tests/worker/test_swap.py
+1
-1
@@ -1,8 +1,8 @@
1
import torch
2
3
from vllm.engine.arg_utils import EngineArgs
4
-from vllm.worker.worker import Worker
4
from vllm.utils import get_distributed_init_method, get_ip, get_open_port
5
+from vllm.worker.worker import Worker
6
7
8
def test_swap() -> None:
vllm/attention/__init__.py
+2
-1
@@ -1,4 +1,5 @@
1
-from vllm.attention.backends.abstract import AttentionBackend, AttentionMetadata
1
+from vllm.attention.backends.abstract import (AttentionBackend,
2
+ AttentionMetadata)
3
from vllm.attention.layer import Attention
4
from vllm.attention.selector import get_attn_backend
5
vllm/attention/backends/flash_attn.py
+3
-2
@@ -7,12 +7,13 @@ flashinfer for all the attention operations.
7
from dataclasses import dataclass
8
from typing import Dict, List, Optional, Tuple, Type
9
10
-from flash_attn import flash_attn_varlen_func
10
import torch
11
+from flash_attn import flash_attn_varlen_func
12
13
from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
14
AttentionMetadata)
15
-from vllm.attention.ops.paged_attn import PagedAttention, PagedAttentionMetadata
15
+from vllm.attention.ops.paged_attn import (PagedAttention,
16
+ PagedAttentionMetadata)
17
18
19
class FlashAttentionBackend(AttentionBackend):
vllm/attention/backends/xformers.py
+2
-1
@@ -11,7 +11,8 @@ from xformers.ops.fmha.attn_bias import (AttentionBias,
11
12
from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
13
AttentionMetadata)
14
-from vllm.attention.ops.paged_attn import PagedAttention, PagedAttentionMetadata
14
+from vllm.attention.ops.paged_attn import (PagedAttention,
15
+ PagedAttentionMetadata)
16
from vllm.logger import init_logger
17
from vllm.utils import is_hip
18
vllm/attention/ops/paged_attn.py
+1
-2
@@ -3,8 +3,7 @@ from typing import Dict, List, Optional, Tuple
3
4
import torch
5
6
-from vllm._C import cache_ops
7
-from vllm._C import ops
6
+from vllm._C import cache_ops, ops
7
from vllm.attention.ops.prefix_prefill import context_attention_fwd
8
9
# Should be the same as PARTITION_SIZE in `paged_attention_v2_launcher`.
vllm/attention/selector.py
+4
-2
@@ -13,11 +13,13 @@ logger = init_logger(__name__)
13
def get_attn_backend(dtype: torch.dtype) -> AttentionBackend:
14
if _can_use_flash_attn(dtype):
15
logger.info("Using FlashAttention backend.")
16
- from vllm.attention.backends.flash_attn import FlashAttentionBackend # noqa: F401
16
+ from vllm.attention.backends.flash_attn import ( # noqa: F401
17
+ FlashAttentionBackend)
18
return FlashAttentionBackend
19
else:
20
logger.info("Using XFormers backend.")
20
- from vllm.attention.backends.xformers import XFormersBackend # noqa: F401
21
+ from vllm.attention.backends.xformers import ( # noqa: F401
22
+ XFormersBackend)
23
return XFormersBackend
24
25
vllm/config.py
+7
-6
@@ -1,15 +1,15 @@
1
-from typing import TYPE_CHECKING, Optional, Union, ClassVar
2
-from dataclasses import dataclass
1
+import json
2
import os
4
-from packaging.version import Version
3
+from dataclasses import dataclass
4
+from typing import TYPE_CHECKING, ClassVar, Optional, Union
5
6
-import json
6
import torch
7
+from packaging.version import Version
8
from transformers import PretrainedConfig
9
10
from vllm.logger import init_logger
11
from vllm.transformers_utils.config import get_config
12
-from vllm.utils import get_cpu_memory, is_hip, is_neuron, get_nvcc_cuda_version
12
+from vllm.utils import get_cpu_memory, get_nvcc_cuda_version, is_hip, is_neuron
13
14
if TYPE_CHECKING:
15
from ray.util.placement_group import PlacementGroup
@@ -103,7 +103,8 @@ class ModelConfig:
103
if os.environ.get("VLLM_USE_MODELSCOPE", "False").lower() == "true":
104
# download model from ModelScope hub,
105
# lazy import so that modelscope is not required for normal use.
106
- from modelscope.hub.snapshot_download import snapshot_download # pylint: disable=C
106
+ # pylint: disable=C.
107
+ from modelscope.hub.snapshot_download import snapshot_download
108
109
if not os.path.exists(model):
110
model_path = snapshot_download(model_id=model,
vllm/core/block_manager.py
+3
-3
@@ -1,15 +1,15 @@
1
"""A block manager that manages token blocks."""
2
import enum
3
+from abc import ABC, abstractmethod
4
from itertools import count, takewhile
5
from os.path import commonprefix
6
from typing import Dict, List, Optional, Set, Tuple
6
-from abc import ABC, abstractmethod
7
8
from vllm.block import BlockTable, PhysicalTokenBlock
9
+from vllm.core.evictor import EvictionPolicy, Evictor, make_evictor
10
+from vllm.logger import init_logger
11
from vllm.sequence import Sequence, SequenceGroup, SequenceStatus
12
from vllm.utils import Device
11
-from vllm.core.evictor import Evictor, EvictionPolicy, make_evictor
12
-from vllm.logger import init_logger
13
14
logger = init_logger(__name__)
15
vllm/core/evictor.py
+1
-1
@@ -1,6 +1,6 @@
1
import enum
2
-from typing import OrderedDict
2
from abc import ABC, abstractmethod, abstractproperty
3
+from typing import OrderedDict
4
5
from vllm.block import PhysicalTokenBlock
6
vllm/core/scheduler.py
+3
-3
@@ -1,13 +1,13 @@
1
-from collections import deque
1
import enum
2
import time
4
-from typing import Deque, Dict, Iterable, List, Optional, Tuple, Union, Set
3
+from collections import deque
4
+from typing import Deque, Dict, Iterable, List, Optional, Set, Tuple, Union
5
6
from vllm.config import CacheConfig, LoRAConfig, SchedulerConfig
7
from vllm.core.block_manager import AllocStatus, BlockSpaceManager
8
from vllm.core.policy import PolicyFactory
9
-from vllm.lora.request import LoRARequest
9
from vllm.logger import init_logger
10
+from vllm.lora.request import LoRARequest
11
from vllm.sequence import (Sequence, SequenceData, SequenceGroup,
12
SequenceGroupMetadata, SequenceStatus)
13
vllm/engine/arg_utils.py
+2
-3
@@ -3,9 +3,8 @@ import dataclasses
3
from dataclasses import dataclass
4
from typing import Optional, Tuple
5
6
-from vllm.config import (CacheConfig, DeviceConfig, ModelConfig,
7
- ParallelConfig, SchedulerConfig, LoRAConfig,
8
- TokenizerPoolConfig)
6
+from vllm.config import (CacheConfig, DeviceConfig, LoRAConfig, ModelConfig,
7
+ ParallelConfig, SchedulerConfig, TokenizerPoolConfig)
8
9
10
@dataclass
vllm/engine/async_llm_engine.py
+3
-3
@@ -2,17 +2,17 @@ import asyncio
2
import os
3
import time
4
from functools import partial
5
-from typing import (Callable, Dict, Iterable, List, Optional, Set, Tuple, Type,
6
- Union, AsyncIterator)
5
+from typing import (AsyncIterator, Callable, Dict, Iterable, List, Optional,
6
+ Set, Tuple, Type, Union)
7
8
from transformers import PreTrainedTokenizer
9
10
-from vllm.lora.request import LoRARequest
10
from vllm.config import ModelConfig
11
from vllm.engine.arg_utils import AsyncEngineArgs
12
from vllm.engine.llm_engine import LLMEngine
13
from vllm.engine.ray_utils import initialize_ray_cluster, ray
14
from vllm.logger import init_logger
15
+from vllm.lora.request import LoRARequest
16
from vllm.outputs import RequestOutput
17
from vllm.sampling_params import SamplingParams
18
vllm/engine/llm_engine.py
+5
-5
@@ -4,22 +4,22 @@ from typing import Iterable, List, Optional, Tuple, Type, Union
4
from transformers import PreTrainedTokenizer
5
6
import vllm
7
-from vllm.lora.request import LoRARequest
8
-from vllm.config import (CacheConfig, DeviceConfig, ModelConfig,
9
- ParallelConfig, SchedulerConfig, LoRAConfig)
7
+from vllm.config import (CacheConfig, DeviceConfig, LoRAConfig, ModelConfig,
8
+ ParallelConfig, SchedulerConfig)
9
from vllm.core.scheduler import Scheduler, SchedulerOutputs
10
from vllm.engine.arg_utils import EngineArgs
12
-from vllm.executor.executor_base import ExecutorBase
11
from vllm.engine.metrics import StatLogger, Stats
12
from vllm.engine.ray_utils import initialize_ray_cluster
13
+from vllm.executor.executor_base import ExecutorBase
14
from vllm.logger import init_logger
15
+from vllm.lora.request import LoRARequest
16
from vllm.outputs import RequestOutput
17
from vllm.sampling_params import SamplingParams
18
from vllm.sequence import (SamplerOutput, Sequence, SequenceGroup,
19
SequenceGroupOutput, SequenceOutput, SequenceStatus)
20
+from vllm.transformers_utils.detokenizer import Detokenizer
21
from vllm.transformers_utils.tokenizer_group import (BaseTokenizerGroup,
22
get_tokenizer_group)
22
-from vllm.transformers_utils.detokenizer import Detokenizer
23
from vllm.utils import Counter
24
25
logger = init_logger(__name__)
vllm/engine/metrics.py
+7
-6
@@ -1,11 +1,12 @@
1
-from vllm.logger import init_logger
2
-from prometheus_client import (Counter, Gauge, Histogram, Info, REGISTRY,
3
- disable_created_metrics)
4
-
1
import time
6
-import numpy as np
7
-from typing import Dict, List
2
from dataclasses import dataclass
3
+from typing import Dict, List
4
+
5
+import numpy as np
6
+from prometheus_client import (REGISTRY, Counter, Gauge, Histogram, Info,
7
+ disable_created_metrics)
8
+
9
+from vllm.logger import init_logger
10
11
logger = init_logger(__name__)
12
vllm/engine/ray_utils.py
+2
-3
@@ -1,10 +1,9 @@
1
import pickle
2
-
3
-from typing import Optional, List, Tuple
2
+from typing import List, Optional, Tuple
3
4
from vllm.config import ParallelConfig
5
from vllm.logger import init_logger
7
-from vllm.utils import is_hip, set_cuda_visible_devices, get_ip
6
+from vllm.utils import get_ip, is_hip, set_cuda_visible_devices
7
8
logger = init_logger(__name__)
9
vllm/entrypoints/api_server.py
+1
-1
@@ -11,9 +11,9 @@ import json
11
import ssl
12
from typing import AsyncGenerator
13
14
+import uvicorn
15
from fastapi import FastAPI, Request
16
from fastapi.responses import JSONResponse, Response, StreamingResponse
16
-import uvicorn
17
18
from vllm.engine.arg_utils import AsyncEngineArgs
19
from vllm.engine.async_llm_engine import AsyncLLMEngine
vllm/entrypoints/llm.py
+1
-1
@@ -3,9 +3,9 @@ from typing import List, Optional, Union
3
from tqdm import tqdm
4
from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast
5
6
-from vllm.lora.request import LoRARequest
6
from vllm.engine.arg_utils import EngineArgs
7
from vllm.engine.llm_engine import LLMEngine
8
+from vllm.lora.request import LoRARequest
9
from vllm.outputs import RequestOutput
10
from vllm.sampling_params import SamplingParams
11
from vllm.utils import Counter
vllm/entrypoints/openai/api_server.py
+8
-9
@@ -1,28 +1,27 @@
1
import asyncio
2
-from contextlib import asynccontextmanager
3
-import os
2
import importlib
3
import inspect
4
+import os
5
+from contextlib import asynccontextmanager
6
+from http import HTTPStatus
7
7
-from prometheus_client import make_asgi_app
8
import fastapi
9
import uvicorn
10
-from http import HTTPStatus
10
from fastapi import Request
11
from fastapi.exceptions import RequestValidationError
12
from fastapi.middleware.cors import CORSMiddleware
14
-from fastapi.responses import JSONResponse, StreamingResponse, Response
13
+from fastapi.responses import JSONResponse, Response, StreamingResponse
14
+from prometheus_client import make_asgi_app
15
16
import vllm
17
from vllm.engine.arg_utils import AsyncEngineArgs
18
from vllm.engine.async_llm_engine import AsyncLLMEngine
19
-from vllm.entrypoints.openai.protocol import (CompletionRequest,
20
- ChatCompletionRequest,
21
- ErrorResponse)
22
-from vllm.logger import init_logger
19
from vllm.entrypoints.openai.cli_args import make_arg_parser
20
+from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
21
+ CompletionRequest, ErrorResponse)
22
from vllm.entrypoints.openai.serving_chat import OpenAIServingChat
23
from vllm.entrypoints.openai.serving_completion import OpenAIServingCompletion
24
+from vllm.logger import init_logger
25
26
TIMEOUT_KEEP_ALIVE = 5 # seconds
27
vllm/entrypoints/openai/protocol.py
+2
-3
@@ -3,12 +3,11 @@
3
import time
4
from typing import Dict, List, Literal, Optional, Union
5
6
+import torch
7
from pydantic import BaseModel, Field, model_validator
8
8
-from vllm.utils import random_uuid
9
from vllm.sampling_params import SamplingParams
10
-
11
-import torch
10
+from vllm.utils import random_uuid
11
12
13
class ErrorResponse(BaseModel):
vllm/entrypoints/openai/serving_chat.py
+8
-6
@@ -1,19 +1,21 @@
1
-import time
1
import codecs
2
+import time
3
+from typing import AsyncGenerator, AsyncIterator, List, Optional, Union
4
+
5
from fastapi import Request
4
-from typing import AsyncGenerator, AsyncIterator, Optional, List, Union
5
-from vllm.logger import init_logger
6
-from vllm.utils import random_uuid
6
+
7
from vllm.engine.async_llm_engine import AsyncLLMEngine
8
from vllm.entrypoints.openai.protocol import (
9
ChatCompletionRequest, ChatCompletionResponse,
10
ChatCompletionResponseChoice, ChatCompletionResponseStreamChoice,
11
ChatCompletionStreamResponse, ChatMessage, DeltaMessage, ErrorResponse,
12
UsageInfo)
13
-from vllm.outputs import RequestOutput
14
-from vllm.entrypoints.openai.serving_engine import OpenAIServing, LoRA
13
+from vllm.entrypoints.openai.serving_engine import LoRA, OpenAIServing
14
+from vllm.logger import init_logger
15
from vllm.model_executor.guided_decoding import (
16
get_guided_decoding_logits_processor)
17
+from vllm.outputs import RequestOutput
18
+from vllm.utils import random_uuid
19
20
logger = init_logger(__name__)
21
vllm/entrypoints/openai/serving_completion.py
+14
-15
@@ -1,24 +1,23 @@
1
import asyncio
2
import time
3
+from typing import (AsyncGenerator, AsyncIterator, Callable, Dict, List,
4
+ Optional, Tuple)
5
+
6
from fastapi import Request
4
-from typing import (AsyncGenerator, AsyncIterator, Callable, List, Optional,
5
- Dict, Tuple)
6
-from vllm.logger import init_logger
7
-from vllm.utils import random_uuid
7
+
8
from vllm.engine.async_llm_engine import AsyncLLMEngine
9
-from vllm.entrypoints.openai.protocol import (
10
- CompletionRequest,
11
- CompletionResponse,
12
- CompletionResponseChoice,
13
- CompletionResponseStreamChoice,
14
- CompletionStreamResponse,
15
- LogProbs,
16
- UsageInfo,
17
-)
18
-from vllm.outputs import RequestOutput
19
-from vllm.entrypoints.openai.serving_engine import OpenAIServing, LoRA
9
+from vllm.entrypoints.openai.protocol import (CompletionRequest,
10
+ CompletionResponse,
11
+ CompletionResponseChoice,
12
+ CompletionResponseStreamChoice,
13
+ CompletionStreamResponse,
14
+ LogProbs, UsageInfo)
15
+from vllm.entrypoints.openai.serving_engine import LoRA, OpenAIServing
16
+from vllm.logger import init_logger
17
from vllm.model_executor.guided_decoding import (
18
get_guided_decoding_logits_processor)
19
+from vllm.outputs import RequestOutput
20
+from vllm.utils import random_uuid
21
22
logger = init_logger(__name__)
23
vllm/entrypoints/openai/serving_engine.py
+6
-6
@@ -3,16 +3,16 @@ import json
3
from dataclasses import dataclass
4
from http import HTTPStatus
5
from typing import Dict, List, Optional, Union
6
-from vllm.logger import init_logger
7
-from vllm.transformers_utils.tokenizer import get_tokenizer
6
+
7
from vllm.engine.async_llm_engine import AsyncLLMEngine
9
-from vllm.entrypoints.openai.protocol import (CompletionRequest,
10
- ChatCompletionRequest,
11
- ErrorResponse, LogProbs,
12
- ModelCard, ModelList,
8
+from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
9
+ CompletionRequest, ErrorResponse,
10
+ LogProbs, ModelCard, ModelList,
11
ModelPermission)
12
+from vllm.logger import init_logger
13
from vllm.lora.request import LoRARequest
14
from vllm.sequence import Logprob
15
+from vllm.transformers_utils.tokenizer import get_tokenizer
16
17
logger = init_logger(__name__)
18
vllm/executor/executor_base.py
+2
-2
@@ -1,8 +1,8 @@
1
from abc import ABC, abstractmethod
2
from typing import Dict, List, Optional
3
4
-from vllm.config import (CacheConfig, DeviceConfig, ModelConfig,
5
- ParallelConfig, SchedulerConfig, LoRAConfig)
4
+from vllm.config import (CacheConfig, DeviceConfig, LoRAConfig, ModelConfig,
5
+ ParallelConfig, SchedulerConfig)
6
from vllm.lora.request import LoRARequest
7
from vllm.sequence import SamplerOutput, SequenceGroupMetadata
8
vllm/executor/gpu_executor.py
+4
-4
@@ -1,13 +1,13 @@
1
from typing import Dict, List, Optional
2
3
-from vllm.lora.request import LoRARequest
4
-from vllm.config import (CacheConfig, DeviceConfig, ModelConfig,
5
- ParallelConfig, SchedulerConfig, LoRAConfig)
3
+from vllm.config import (CacheConfig, DeviceConfig, LoRAConfig, ModelConfig,
4
+ ParallelConfig, SchedulerConfig)
5
from vllm.executor.executor_base import ExecutorAsyncBase, ExecutorBase
6
from vllm.executor.utils import check_block_size_valid
7
from vllm.logger import init_logger
8
+from vllm.lora.request import LoRARequest
9
from vllm.sequence import SamplerOutput, SequenceGroupMetadata
10
-from vllm.utils import (get_ip, get_open_port, get_distributed_init_method,
10
+from vllm.utils import (get_distributed_init_method, get_ip, get_open_port,
11
make_async)
12
13
logger = init_logger(__name__)
vllm/executor/neuron_executor.py
+3
-3
@@ -1,10 +1,10 @@
1
from typing import Dict, List, Optional
2
3
-from vllm.lora.request import LoRARequest
4
-from vllm.config import (CacheConfig, DeviceConfig, ModelConfig,
5
- ParallelConfig, SchedulerConfig, LoRAConfig)
3
+from vllm.config import (CacheConfig, DeviceConfig, LoRAConfig, ModelConfig,
4
+ ParallelConfig, SchedulerConfig)
5
from vllm.executor.executor_base import ExecutorBase
6
from vllm.logger import init_logger
7
+from vllm.lora.request import LoRARequest
8
from vllm.sequence import SamplerOutput, SequenceGroupMetadata
9
10
logger = init_logger(__name__)
vllm/executor/ray_gpu_executor.py
+6
-6
@@ -1,20 +1,20 @@
1
import asyncio
2
import copy
3
-from collections import defaultdict
3
import os
4
import pickle
5
+from collections import defaultdict
6
from typing import TYPE_CHECKING, Any, Dict, List, Optional
7
8
-from vllm.config import (CacheConfig, DeviceConfig, ModelConfig,
9
- ParallelConfig, SchedulerConfig, LoRAConfig)
8
+from vllm.config import (CacheConfig, DeviceConfig, LoRAConfig, ModelConfig,
9
+ ParallelConfig, SchedulerConfig)
10
from vllm.engine.ray_utils import RayWorkerVllm, ray
11
from vllm.executor.executor_base import ExecutorAsyncBase, ExecutorBase
12
from vllm.executor.utils import check_block_size_valid
13
from vllm.logger import init_logger
14
from vllm.lora.request import LoRARequest
15
from vllm.sequence import SamplerOutput, SequenceGroupMetadata
16
-from vllm.utils import (set_cuda_visible_devices, get_ip, get_open_port,
17
- get_distributed_init_method, make_async)
16
+from vllm.utils import (get_distributed_init_method, get_ip, get_open_port,
17
+ make_async, set_cuda_visible_devices)
18
19
if ray is not None:
20
from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy
@@ -343,7 +343,7 @@ class RayGPUExecutor(ExecutorBase):
343
raise ValueError(f"Ray version {required_version} or greater is "
344
f"required, but found {current_version}")
345
346
- from ray.dag import MultiOutputNode, InputNode
346
+ from ray.dag import InputNode, MultiOutputNode
347
assert self.parallel_config.worker_use_ray
348
349
# Right now, compiled DAG requires at least 1 arg. We send
vllm/logger.py
+1
-1
@@ -2,8 +2,8 @@
2
# https://github.com/skypilot-org/skypilot/blob/86dc0f6283a335e4aa37b3c10716f90999f48ab6/sky/sky_logging.py
3
"""Logging configuration for vLLM."""
4
import logging
5
-import sys
5
import os
6
+import sys
7
8
VLLM_CONFIGURE_LOGGING = int(os.getenv("VLLM_CONFIGURE_LOGGING", "1"))
9
vllm/lora/layers.py
+6
-8
@@ -10,18 +10,16 @@ from transformers import PretrainedConfig
10
11
from vllm.config import LoRAConfig
12
from vllm.lora.punica import add_lora, add_lora_slice, bgmv
13
-from vllm.model_executor.parallel_utils.communication_op import (
14
- tensor_model_parallel_all_gather,
15
- tensor_model_parallel_all_reduce,
16
- tensor_model_parallel_gather,
17
-)
13
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
19
- RowParallelLinear,
14
+ MergedColumnParallelLinear,
15
QKVParallelLinear,
21
- MergedColumnParallelLinear)
16
+ RowParallelLinear)
17
from vllm.model_executor.layers.logits_processor import LogitsProcessor
18
from vllm.model_executor.layers.vocab_parallel_embedding import (
24
- VocabParallelEmbedding, ParallelLMHead)
19
+ ParallelLMHead, VocabParallelEmbedding)
20
+from vllm.model_executor.parallel_utils.communication_op import (
21
+ tensor_model_parallel_all_gather, tensor_model_parallel_all_reduce,
22
+ tensor_model_parallel_gather)
23
from vllm.model_executor.parallel_utils.parallel_state import (
24
get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
25
from vllm.model_executor.parallel_utils.utils import (
vllm/lora/lora.py
+1
@@ -1,6 +1,7 @@
1
from typing import List, Optional
2
3
import torch
4
+
5
from vllm.utils import is_pin_memory_available
6
7
vllm/lora/models.py
+2
-3
@@ -4,19 +4,18 @@ import logging
4
import math
5
import os
6
import re
7
-from typing import (Callable, Dict, Hashable, List, Optional, Tuple, Type)
7
+from typing import Callable, Dict, Hashable, List, Optional, Tuple, Type
8
9
import safetensors.torch
10
import torch
11
from torch import nn
12
13
from vllm.config import LoRAConfig
14
-from vllm.utils import LRUCache, is_pin_memory_available
15
-
14
from vllm.lora.layers import (BaseLayerWithLoRA, LoRAMapping, from_layer,
15
from_layer_logits_processor)
16
from vllm.lora.lora import LoRALayerWeights, PackedLoRALayerWeights
17
from vllm.lora.utils import parse_fine_tuned_lora_name, replace_submodule
18
+from vllm.utils import LRUCache, is_pin_memory_available
19
20
logger = logging.getLogger(__name__)
21
vllm/lora/worker_manager.py
+2
-2
@@ -4,11 +4,11 @@ from typing import Any, Dict, List, Optional, Set, Type
4
5
import torch
6
7
+from vllm.config import LoRAConfig
8
+from vllm.lora.layers import LoRAMapping
9
from vllm.lora.models import (LoRAModel, LoRAModelManager,
10
LRUCacheLoRAModelManager, create_lora_manager)
11
from vllm.lora.request import LoRARequest
10
-from vllm.lora.layers import LoRAMapping
11
-from vllm.config import LoRAConfig
12
13
logger = logging.getLogger(__name__)
14
vllm/model_executor/guided_decoding.py
+6
-6
@@ -5,16 +5,16 @@ from enum import Enum
5
from functools import lru_cache
6
from json import dumps as json_dumps
7
from re import escape as regex_escape
8
-from typing import Union, Tuple
8
+from typing import Tuple, Union
9
10
from pydantic import BaseModel
11
from transformers import PreTrainedTokenizerBase
12
13
-from vllm.entrypoints.openai.protocol import (CompletionRequest,
14
- ChatCompletionRequest)
15
-from vllm.model_executor.guided_logits_processors import (JSONLogitsProcessor,
16
- RegexLogitsProcessor,
17
- CFGLogitsProcessor)
13
+from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
14
+ CompletionRequest)
15
+from vllm.model_executor.guided_logits_processors import (CFGLogitsProcessor,
16
+ JSONLogitsProcessor,
17
+ RegexLogitsProcessor)
18
19
20
class GuidedDecodingMode(Enum):
vllm/model_executor/guided_logits_processors.py
+3
-3
@@ -16,13 +16,13 @@
16
import json
17
import math
18
from collections import defaultdict
19
-from typing import Union, DefaultDict, Dict, List, Optional, Callable
19
+from typing import Callable, DefaultDict, Dict, List, Optional, Union
20
21
import torch
22
+from outlines.fsm.fsm import CFGFSM, RegexFSM
23
+from outlines.fsm.json_schema import build_regex_from_schema
24
from pydantic import BaseModel
25
from transformers import PreTrainedTokenizerBase
24
-from outlines.fsm.fsm import RegexFSM, CFGFSM
25
-from outlines.fsm.json_schema import build_regex_from_schema
26
27
28
class BaseLogitsProcessor:
vllm/model_executor/layers/fused_moe/__init__.py
+1
-3
@@ -1,7 +1,5 @@
1
from vllm.model_executor.layers.fused_moe.fused_moe import (
2
- fused_moe,
3
- get_config_file_name,
4
-)
2
+ fused_moe, get_config_file_name)
3
4
__all__ = [
5
"fused_moe",
vllm/model_executor/layers/linear.py
+3
-3
@@ -5,14 +5,14 @@ import torch
5
import torch.nn.functional as F
6
from torch.nn.parameter import Parameter
7
8
+from vllm.logger import init_logger
9
+from vllm.model_executor.parallel_utils.communication_op import (
10
+ tensor_model_parallel_all_gather, tensor_model_parallel_all_reduce)
11
from vllm.model_executor.parallel_utils.parallel_state import (
12
get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
10
-from vllm.model_executor.parallel_utils.communication_op import (
11
- tensor_model_parallel_all_reduce, tensor_model_parallel_all_gather)
13
from vllm.model_executor.parallel_utils.utils import (
14
divide, split_tensor_along_last_dim)
15
from vllm.model_executor.utils import set_weight_attrs
15
-from vllm.logger import init_logger
16
17
logger = init_logger(__name__)
18
vllm/model_executor/layers/ops/rand.py
+2
-2
@@ -1,9 +1,9 @@
1
+from typing import Optional, Union
2
+
3
import torch
4
import triton
5
import triton.language as tl
6
5
-from typing import Optional, Union
6
-
7
8
def seeded_uniform(
9
*size,
vllm/model_executor/layers/ops/sample.py
+1
-1
@@ -1,5 +1,5 @@
1
import math
2
-from typing import Tuple, Optional
2
+from typing import Optional, Tuple
3
4
import torch
5
import triton
vllm/model_executor/layers/quantization/__init__.py
+2
-2
@@ -1,11 +1,11 @@
1
from typing import Type
2
3
+from vllm.model_executor.layers.quantization.awq import AWQConfig
4
from vllm.model_executor.layers.quantization.base_config import (
5
QuantizationConfig)
5
-from vllm.model_executor.layers.quantization.awq import AWQConfig
6
from vllm.model_executor.layers.quantization.gptq import GPTQConfig
7
-from vllm.model_executor.layers.quantization.squeezellm import SqueezeLLMConfig
7
from vllm.model_executor.layers.quantization.marlin import MarlinConfig
8
+from vllm.model_executor.layers.quantization.squeezellm import SqueezeLLMConfig
9
10
_QUANTIZATION_CONFIG_REGISTRY = {
11
"awq": AWQConfig,
vllm/model_executor/layers/quantization/gptq.py
+1
-1
@@ -1,7 +1,7 @@
1
import enum
2
from enum import Enum
3
-from typing import Any, Dict, List, Optional
3
from fractions import Fraction
4
+from typing import Any, Dict, List, Optional
5
6
import torch
7
from torch.nn.parameter import Parameter
vllm/model_executor/layers/quantization/marlin.py
+2
-1
@@ -4,7 +4,8 @@ import torch
4
from torch.nn.parameter import Parameter
5
6
from vllm._C import ops
7
-from vllm.model_executor.layers.linear import LinearMethodBase, set_weight_attrs
7
+from vllm.model_executor.layers.linear import (LinearMethodBase,
8
+ set_weight_attrs)
9
from vllm.model_executor.layers.quantization.base_config import (
10
QuantizationConfig)
11
vllm/model_executor/layers/rejection_sampler.py
+2
-2
@@ -1,9 +1,9 @@
1
-from typing import Tuple, Optional
1
from functools import cached_property
2
+from typing import Optional, Tuple
3
4
import torch
5
-import torch.nn as nn
5
import torch.jit
6
+import torch.nn as nn
7
8
9
class RejectionSampler(nn.Module):
vllm/model_executor/layers/vocab_parallel_embedding.py
+3
-5
@@ -4,13 +4,11 @@ import torch
4
import torch.nn.functional as F
5
from torch.nn.parameter import Parameter
6
7
-from vllm.model_executor.parallel_utils.parallel_state import (
8
- get_tensor_model_parallel_rank,
9
- get_tensor_model_parallel_world_size,
10
-)
11
-from vllm.model_executor.parallel_utils.utils import divide
7
from vllm.model_executor.parallel_utils.communication_op import (
8
tensor_model_parallel_all_reduce)
9
+from vllm.model_executor.parallel_utils.parallel_state import (
10
+ get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
11
+from vllm.model_executor.parallel_utils.utils import divide
12
from vllm.model_executor.utils import set_weight_attrs
13
14
DEFAULT_VOCAB_PADDING_SIZE = 64
vllm/model_executor/models/baichuan.py
+2
-2
@@ -32,11 +32,11 @@ from vllm.model_executor.layers.linear import (LinearMethodBase,
32
MergedColumnParallelLinear,
33
QKVParallelLinear,
34
RowParallelLinear)
35
-from vllm.model_executor.layers.rotary_embedding import get_rope
35
from vllm.model_executor.layers.logits_processor import LogitsProcessor
36
+from vllm.model_executor.layers.rotary_embedding import get_rope
37
from vllm.model_executor.layers.sampler import Sampler
38
from vllm.model_executor.layers.vocab_parallel_embedding import (
39
- VocabParallelEmbedding, ParallelLMHead)
39
+ ParallelLMHead, VocabParallelEmbedding)
40
from vllm.model_executor.parallel_utils.parallel_state import (
41
get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
42
from vllm.model_executor.sampling_metadata import SamplingMetadata
vllm/model_executor/models/chatglm.py
+2
-2
@@ -15,11 +15,11 @@ from vllm.model_executor.layers.linear import (LinearMethodBase,
15
MergedColumnParallelLinear,
16
QKVParallelLinear,
17
RowParallelLinear)
18
-from vllm.model_executor.layers.rotary_embedding import get_rope
18
from vllm.model_executor.layers.logits_processor import LogitsProcessor
19
+from vllm.model_executor.layers.rotary_embedding import get_rope
20
from vllm.model_executor.layers.sampler import Sampler
21
from vllm.model_executor.layers.vocab_parallel_embedding import (
22
- VocabParallelEmbedding, ParallelLMHead)
22
+ ParallelLMHead, VocabParallelEmbedding)
23
from vllm.model_executor.parallel_utils.parallel_state import (
24
get_tensor_model_parallel_world_size)
25
from vllm.model_executor.sampling_metadata import SamplingMetadata
vllm/model_executor/models/deepseek.py
+3
-3
@@ -33,14 +33,14 @@ from vllm.model_executor.layers.fused_moe import fused_moe
33
from vllm.model_executor.layers.layernorm import RMSNorm
34
from vllm.model_executor.layers.linear import (LinearMethodBase,
35
MergedColumnParallelLinear,
36
- ReplicatedLinear,
36
QKVParallelLinear,
37
+ ReplicatedLinear,
38
RowParallelLinear)
39
-from vllm.model_executor.layers.rotary_embedding import get_rope
39
from vllm.model_executor.layers.logits_processor import LogitsProcessor
40
+from vllm.model_executor.layers.rotary_embedding import get_rope
41
from vllm.model_executor.layers.sampler import Sampler
42
from vllm.model_executor.layers.vocab_parallel_embedding import (
43
- VocabParallelEmbedding, ParallelLMHead)
43
+ ParallelLMHead, VocabParallelEmbedding)
44
from vllm.model_executor.parallel_utils.communication_op import (
45
tensor_model_parallel_all_reduce)
46
from vllm.model_executor.parallel_utils.parallel_state import (
vllm/model_executor/models/falcon.py
+1
-1
@@ -32,8 +32,8 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
32
LinearMethodBase,
33
QKVParallelLinear,
34
RowParallelLinear)
35
-from vllm.model_executor.layers.rotary_embedding import get_rope
35
from vllm.model_executor.layers.logits_processor import LogitsProcessor
36
+from vllm.model_executor.layers.rotary_embedding import get_rope
37
from vllm.model_executor.layers.sampler import Sampler
38
from vllm.model_executor.layers.vocab_parallel_embedding import (
39
VocabParallelEmbedding)
vllm/model_executor/models/gemma.py
+1
-1
@@ -28,8 +28,8 @@ from vllm.model_executor.layers.linear import (LinearMethodBase,
28
MergedColumnParallelLinear,
29
QKVParallelLinear,
30
RowParallelLinear)
31
-from vllm.model_executor.layers.rotary_embedding import get_rope
31
from vllm.model_executor.layers.logits_processor import LogitsProcessor
32
+from vllm.model_executor.layers.rotary_embedding import get_rope
33
from vllm.model_executor.layers.sampler import Sampler
34
from vllm.model_executor.layers.vocab_parallel_embedding import (
35
VocabParallelEmbedding)
vllm/model_executor/models/gpt_j.py
+2
-2
@@ -28,11 +28,11 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
28
LinearMethodBase,
29
QKVParallelLinear,
30
RowParallelLinear)
31
-from vllm.model_executor.layers.rotary_embedding import get_rope
31
from vllm.model_executor.layers.logits_processor import LogitsProcessor
32
+from vllm.model_executor.layers.rotary_embedding import get_rope
33
from vllm.model_executor.layers.sampler import Sampler
34
from vllm.model_executor.layers.vocab_parallel_embedding import (
35
- VocabParallelEmbedding, ParallelLMHead)
35
+ ParallelLMHead, VocabParallelEmbedding)
36
from vllm.model_executor.parallel_utils.parallel_state import (
37
get_tensor_model_parallel_world_size)
38
from vllm.model_executor.sampling_metadata import SamplingMetadata
vllm/model_executor/models/gpt_neox.py
+2
-2
@@ -28,11 +28,11 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
28
LinearMethodBase,
29
QKVParallelLinear,
30
RowParallelLinear)
31
-from vllm.model_executor.layers.rotary_embedding import get_rope
31
from vllm.model_executor.layers.logits_processor import LogitsProcessor
32
+from vllm.model_executor.layers.rotary_embedding import get_rope
33
from vllm.model_executor.layers.sampler import Sampler
34
from vllm.model_executor.layers.vocab_parallel_embedding import (
35
- VocabParallelEmbedding, ParallelLMHead)
35
+ ParallelLMHead, VocabParallelEmbedding)
36
from vllm.model_executor.parallel_utils.parallel_state import (
37
get_tensor_model_parallel_world_size)
38
from vllm.model_executor.sampling_metadata import SamplingMetadata
vllm/model_executor/models/internlm2.py
+2
-2
@@ -12,11 +12,11 @@ from vllm.model_executor.layers.linear import (LinearMethodBase,
12
MergedColumnParallelLinear,
13
QKVParallelLinear,
14
RowParallelLinear)
15
-from vllm.model_executor.layers.rotary_embedding import get_rope
15
from vllm.model_executor.layers.logits_processor import LogitsProcessor
16
+from vllm.model_executor.layers.rotary_embedding import get_rope
17
from vllm.model_executor.layers.sampler import Sampler
18
from vllm.model_executor.layers.vocab_parallel_embedding import (
19
- VocabParallelEmbedding, ParallelLMHead)
19
+ ParallelLMHead, VocabParallelEmbedding)
20
from vllm.model_executor.parallel_utils.parallel_state import (
21
get_tensor_model_parallel_world_size)
22
from vllm.model_executor.sampling_metadata import SamplingMetadata
vllm/model_executor/models/jais.py
+10
-16
@@ -24,29 +24,23 @@ from typing import List, Optional
24
25
import torch
26
from torch import nn
27
-from vllm.transformers_utils.configs import JAISConfig
27
28
from vllm.attention import Attention, AttentionMetadata
30
-from vllm.model_executor.layers.linear import (
31
- ColumnParallelLinear,
32
- LinearMethodBase,
33
- QKVParallelLinear,
34
- RowParallelLinear,
35
-)
29
+from vllm.model_executor.layers.linear import (ColumnParallelLinear,
30
+ LinearMethodBase,
31
+ QKVParallelLinear,
32
+ RowParallelLinear)
33
from vllm.model_executor.layers.logits_processor import LogitsProcessor
34
from vllm.model_executor.layers.sampler import Sampler
35
from vllm.model_executor.layers.vocab_parallel_embedding import (
39
- VocabParallelEmbedding, )
36
+ VocabParallelEmbedding)
37
from vllm.model_executor.parallel_utils.parallel_state import (
41
- get_tensor_model_parallel_world_size,
42
- get_tensor_model_parallel_rank,
43
-)
44
-from vllm.model_executor.weight_utils import (
45
- default_weight_loader,
46
- hf_model_weights_iterator,
47
-)
48
-from vllm.sequence import SamplerOutput
38
+ get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
39
from vllm.model_executor.sampling_metadata import SamplingMetadata
40
+from vllm.model_executor.weight_utils import (default_weight_loader,
41
+ hf_model_weights_iterator)
42
+from vllm.sequence import SamplerOutput
43
+from vllm.transformers_utils.configs import JAISConfig
44
45
46
class SwiGLUActivation(nn.Module):
vllm/model_executor/models/llama.py
+2
-2
@@ -35,11 +35,11 @@ from vllm.model_executor.layers.linear import (LinearMethodBase,
35
MergedColumnParallelLinear,
36
QKVParallelLinear,
37
RowParallelLinear)
38
-from vllm.model_executor.layers.rotary_embedding import get_rope
38
from vllm.model_executor.layers.logits_processor import LogitsProcessor
39
+from vllm.model_executor.layers.rotary_embedding import get_rope
40
from vllm.model_executor.layers.sampler import Sampler
41
from vllm.model_executor.layers.vocab_parallel_embedding import (
42
- VocabParallelEmbedding, ParallelLMHead, DEFAULT_VOCAB_PADDING_SIZE)
42
+ DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
43
from vllm.model_executor.parallel_utils.parallel_state import (
44
get_tensor_model_parallel_world_size)
45
from vllm.model_executor.sampling_metadata import SamplingMetadata
vllm/model_executor/models/mixtral.py
+2
-2
@@ -35,11 +35,11 @@ from vllm.model_executor.layers.linear import (LinearMethodBase,
35
QKVParallelLinear,
36
ReplicatedLinear,
37
RowParallelLinear)
38
-from vllm.model_executor.layers.rotary_embedding import get_rope
38
from vllm.model_executor.layers.logits_processor import LogitsProcessor
39
+from vllm.model_executor.layers.rotary_embedding import get_rope
40
from vllm.model_executor.layers.sampler import Sampler
41
from vllm.model_executor.layers.vocab_parallel_embedding import (
42
- VocabParallelEmbedding, ParallelLMHead, DEFAULT_VOCAB_PADDING_SIZE)
42
+ DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
43
from vllm.model_executor.parallel_utils.communication_op import (
44
tensor_model_parallel_all_reduce)
45
from vllm.model_executor.parallel_utils.parallel_state import (
vllm/model_executor/models/mixtral_quant.py
+3
-5
@@ -24,24 +24,22 @@
24
from typing import List, Optional
25
26
import numpy as np
27
-
27
import torch
28
import torch.nn.functional as F
30
-
29
from torch import nn
30
from transformers import MixtralConfig
31
32
from vllm.attention import Attention, AttentionMetadata
33
from vllm.model_executor.layers.layernorm import RMSNorm
34
from vllm.model_executor.layers.linear import (LinearMethodBase,
37
- ReplicatedLinear,
35
QKVParallelLinear,
36
+ ReplicatedLinear,
37
RowParallelLinear)
40
-from vllm.model_executor.layers.rotary_embedding import get_rope
38
from vllm.model_executor.layers.logits_processor import LogitsProcessor
39
+from vllm.model_executor.layers.rotary_embedding import get_rope
40
from vllm.model_executor.layers.sampler import Sampler
41
from vllm.model_executor.layers.vocab_parallel_embedding import (
44
- VocabParallelEmbedding, ParallelLMHead)
42
+ ParallelLMHead, VocabParallelEmbedding)
43
from vllm.model_executor.parallel_utils.communication_op import (
44
tensor_model_parallel_all_reduce)
45
from vllm.model_executor.parallel_utils.parallel_state import (
vllm/model_executor/models/olmo.py
+10
-15
@@ -40,32 +40,27 @@ from typing import List, Optional, Tuple
40
41
import torch
42
import torch.nn.functional as F
43
+# this model must need this dependency
44
+from hf_olmo import OLMoConfig
45
from torch import nn
46
47
from vllm.attention import Attention, AttentionMetadata
46
-from vllm.model_executor.layers.linear import (
47
- ColumnParallelLinear,
48
- LinearMethodBase,
49
- QKVParallelLinear,
50
- RowParallelLinear,
51
-)
52
-from vllm.model_executor.layers.rotary_embedding import get_rope
48
+from vllm.model_executor.layers.linear import (ColumnParallelLinear,
49
+ LinearMethodBase,
50
+ QKVParallelLinear,
51
+ RowParallelLinear)
52
from vllm.model_executor.layers.logits_processor import LogitsProcessor
53
+from vllm.model_executor.layers.rotary_embedding import get_rope
54
from vllm.model_executor.layers.sampler import Sampler
55
from vllm.model_executor.layers.vocab_parallel_embedding import (
56
VocabParallelEmbedding)
57
from vllm.model_executor.parallel_utils.parallel_state import (
58
- get_tensor_model_parallel_world_size, )
58
+ get_tensor_model_parallel_world_size)
59
from vllm.model_executor.sampling_metadata import SamplingMetadata
60
-from vllm.model_executor.weight_utils import (
61
- default_weight_loader,
62
- hf_model_weights_iterator,
63
-)
60
+from vllm.model_executor.weight_utils import (default_weight_loader,
61
+ hf_model_weights_iterator)
62
from vllm.sequence import SamplerOutput
63
66
-# this model must need this dependency
67
-from hf_olmo import OLMoConfig
68
-
64
65
class SwiGLU(nn.Module):
66
vllm/model_executor/models/orion.py
+2
-2
@@ -16,11 +16,11 @@ from vllm.model_executor.layers.linear import (LinearMethodBase,
16
MergedColumnParallelLinear,
17
QKVParallelLinear,
18
RowParallelLinear)
19
-from vllm.model_executor.layers.rotary_embedding import get_rope
19
from vllm.model_executor.layers.logits_processor import LogitsProcessor
20
+from vllm.model_executor.layers.rotary_embedding import get_rope
21
from vllm.model_executor.layers.sampler import Sampler
22
from vllm.model_executor.layers.vocab_parallel_embedding import (
23
- VocabParallelEmbedding, ParallelLMHead)
23
+ ParallelLMHead, VocabParallelEmbedding)
24
from vllm.model_executor.parallel_utils.parallel_state import (
25
get_tensor_model_parallel_world_size)
26
from vllm.model_executor.sampling_metadata import SamplingMetadata
vllm/model_executor/models/phi.py
+2
-2
@@ -47,11 +47,11 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
47
LinearMethodBase,
48
QKVParallelLinear,
49
RowParallelLinear)
50
-from vllm.model_executor.layers.rotary_embedding import get_rope
50
from vllm.model_executor.layers.logits_processor import LogitsProcessor
51
+from vllm.model_executor.layers.rotary_embedding import get_rope
52
from vllm.model_executor.layers.sampler import Sampler
53
from vllm.model_executor.layers.vocab_parallel_embedding import (
54
- VocabParallelEmbedding, ParallelLMHead)
54
+ ParallelLMHead, VocabParallelEmbedding)
55
from vllm.model_executor.parallel_utils.parallel_state import (
56
get_tensor_model_parallel_world_size)
57
from vllm.model_executor.sampling_metadata import SamplingMetadata
vllm/model_executor/models/qwen.py
+2
-2
@@ -17,11 +17,11 @@ from vllm.model_executor.layers.linear import (LinearMethodBase,
17
MergedColumnParallelLinear,
18
QKVParallelLinear,
19
RowParallelLinear)
20
-from vllm.model_executor.layers.rotary_embedding import get_rope
20
from vllm.model_executor.layers.logits_processor import LogitsProcessor
21
+from vllm.model_executor.layers.rotary_embedding import get_rope
22
from vllm.model_executor.layers.sampler import Sampler
23
from vllm.model_executor.layers.vocab_parallel_embedding import (
24
- VocabParallelEmbedding, ParallelLMHead)
24
+ ParallelLMHead, VocabParallelEmbedding)
25
from vllm.model_executor.parallel_utils.parallel_state import (
26
get_tensor_model_parallel_world_size)
27
from vllm.model_executor.sampling_metadata import SamplingMetadata
vllm/model_executor/models/qwen2.py
+3
-3
@@ -29,24 +29,24 @@ from torch import nn
29
from transformers import Qwen2Config
30
31
from vllm.attention import Attention, AttentionMetadata
32
+from vllm.config import LoRAConfig
33
from vllm.model_executor.layers.activation import SiluAndMul
34
from vllm.model_executor.layers.layernorm import RMSNorm
35
from vllm.model_executor.layers.linear import (LinearMethodBase,
36
MergedColumnParallelLinear,
37
QKVParallelLinear,
38
RowParallelLinear)
38
-from vllm.model_executor.layers.rotary_embedding import get_rope
39
from vllm.model_executor.layers.logits_processor import LogitsProcessor
40
+from vllm.model_executor.layers.rotary_embedding import get_rope
41
from vllm.model_executor.layers.sampler import Sampler
42
from vllm.model_executor.layers.vocab_parallel_embedding import (
42
- VocabParallelEmbedding, ParallelLMHead)
43
+ ParallelLMHead, VocabParallelEmbedding)
44
from vllm.model_executor.parallel_utils.parallel_state import (
45
get_tensor_model_parallel_world_size)
46
from vllm.model_executor.sampling_metadata import SamplingMetadata
47
from vllm.model_executor.weight_utils import (default_weight_loader,
48
hf_model_weights_iterator)
49
from vllm.sequence import SamplerOutput
49
-from vllm.config import LoRAConfig
50
51
52
class Qwen2MLP(nn.Module):
vllm/model_executor/models/stablelm.py
+2
-2
@@ -31,11 +31,11 @@ from vllm.model_executor.layers.linear import (LinearMethodBase,
31
MergedColumnParallelLinear,
32
QKVParallelLinear,
33
RowParallelLinear)
34
-from vllm.model_executor.layers.rotary_embedding import get_rope
34
from vllm.model_executor.layers.logits_processor import LogitsProcessor
35
+from vllm.model_executor.layers.rotary_embedding import get_rope
36
from vllm.model_executor.layers.sampler import Sampler
37
from vllm.model_executor.layers.vocab_parallel_embedding import (
38
- VocabParallelEmbedding, ParallelLMHead)
38
+ ParallelLMHead, VocabParallelEmbedding)
39
from vllm.model_executor.parallel_utils.parallel_state import (
40
get_tensor_model_parallel_world_size)
41
from vllm.model_executor.sampling_metadata import SamplingMetadata
vllm/model_executor/models/starcoder2.py
+3
-3
@@ -25,19 +25,19 @@ from torch import nn
25
from transformers import Starcoder2Config
26
27
from vllm.attention import Attention, AttentionMetadata
28
-from vllm.model_executor.sampling_metadata import SamplingMetadata
28
from vllm.model_executor.layers.activation import get_act_fn
30
-from vllm.model_executor.layers.rotary_embedding import get_rope
29
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
30
LinearMethodBase,
31
QKVParallelLinear,
32
RowParallelLinear)
33
from vllm.model_executor.layers.logits_processor import LogitsProcessor
34
+from vllm.model_executor.layers.rotary_embedding import get_rope
35
from vllm.model_executor.layers.sampler import Sampler
36
from vllm.model_executor.layers.vocab_parallel_embedding import (
38
- VocabParallelEmbedding, ParallelLMHead, DEFAULT_VOCAB_PADDING_SIZE)
37
+ DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
38
from vllm.model_executor.parallel_utils.parallel_state import (
39
get_tensor_model_parallel_world_size)
40
+from vllm.model_executor.sampling_metadata import SamplingMetadata
41
from vllm.model_executor.weight_utils import (default_weight_loader,
42
hf_model_weights_iterator)
43
from vllm.sequence import SamplerOutput
vllm/model_executor/neuron_model_loader.py
+2
-2
@@ -110,8 +110,8 @@ def _get_model_architecture(config: PretrainedConfig) -> Type[nn.Module]:
110
def get_neuron_model(model_config: ModelConfig,
111
parallel_config: ParallelConfig,
112
scheduler_config: SchedulerConfig) -> nn.Module:
113
- from transformers_neuronx.config import (NeuronConfig,
114
- ContinuousBatchingConfig)
113
+ from transformers_neuronx.config import (ContinuousBatchingConfig,
114
+ NeuronConfig)
115
116
# Create a model instance.
117
model = NeuronCasualLM(model_config.hf_config)
vllm/model_executor/parallel_utils/communication_op.py
+3
-6
@@ -5,14 +5,11 @@ import torch
5
from torch.distributed import ProcessGroup
6
7
from vllm.model_executor.parallel_utils import cupy_utils
8
-from vllm.model_executor.parallel_utils.parallel_state import (
9
- get_tensor_model_parallel_rank,
10
- get_tensor_model_parallel_world_size,
11
- get_tensor_model_parallel_group,
12
- is_cupy_nccl_enabled_for_all_reduce,
13
-)
8
from vllm.model_executor.parallel_utils.custom_all_reduce import (
9
custom_all_reduce)
10
+from vllm.model_executor.parallel_utils.parallel_state import (
11
+ get_tensor_model_parallel_group, get_tensor_model_parallel_rank,
12
+ get_tensor_model_parallel_world_size, is_cupy_nccl_enabled_for_all_reduce)
13
14
15
def tensor_model_parallel_all_reduce(input_: torch.Tensor) -> torch.Tensor:
vllm/model_executor/parallel_utils/custom_all_reduce.py
+3
-2
@@ -6,11 +6,12 @@ import torch.distributed as dist
6
7
from vllm.logger import init_logger
8
from vllm.model_executor.parallel_utils.parallel_state import (
9
- get_tensor_model_parallel_world_size, get_tensor_model_parallel_rank)
9
+ get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
10
11
try:
12
- from vllm._C import custom_ar
12
import pynvml
13
+
14
+ from vllm._C import custom_ar
15
except ImportError:
16
# For AMD GPUs
17
custom_ar = None
vllm/model_executor/sampling_metadata.py
+2
-3
@@ -1,11 +1,10 @@
1
+import random
2
from dataclasses import dataclass
3
from typing import Dict, List, Optional, Tuple
4
5
import torch
5
-import random
6
7
-from vllm.model_executor.layers.ops.sample import (
8
- get_num_triton_sampler_splits)
7
+from vllm.model_executor.layers.ops.sample import get_num_triton_sampler_splits
8
from vllm.sampling_params import SamplingParams, SamplingType
9
from vllm.sequence import SequenceData
10
from vllm.utils import is_pin_memory_available
vllm/model_executor/weight_utils.py
+6
-6
@@ -1,23 +1,23 @@
1
"""Utilities for downloading and initializing model weights."""
2
-import filelock
2
+import fnmatch
3
import glob
4
import hashlib
5
-import fnmatch
5
import json
6
import os
7
from collections import defaultdict
8
from typing import Any, Iterator, List, Optional, Tuple
9
11
-from huggingface_hub import snapshot_download, HfFileSystem
10
+import filelock
11
import numpy as np
13
-from safetensors.torch import load_file, save_file, safe_open
12
import torch
13
+from huggingface_hub import HfFileSystem, snapshot_download
14
+from safetensors.torch import load_file, safe_open, save_file
15
from tqdm.auto import tqdm
16
17
from vllm.config import ModelConfig
18
from vllm.logger import init_logger
19
-from vllm.model_executor.layers.quantization import (get_quantization_config,
20
- QuantizationConfig)
19
+from vllm.model_executor.layers.quantization import (QuantizationConfig,
20
+ get_quantization_config)
21
22
logger = init_logger(__name__)
23
vllm/outputs.py
+3
-3
@@ -1,9 +1,9 @@
1
-from typing import List, Optional
1
import time
2
+from typing import List, Optional
3
4
-from vllm.sequence import (PromptLogprobs, SampleLogprobs, SequenceGroup,
5
- SequenceStatus, RequestMetrics)
4
from vllm.lora.request import LoRARequest
5
+from vllm.sequence import (PromptLogprobs, RequestMetrics, SampleLogprobs,
6
+ SequenceGroup, SequenceStatus)
7
8
9
class CompletionOutput:
vllm/sequence.py
+3
-2
@@ -2,14 +2,15 @@
2
import copy
3
import enum
4
from dataclasses import dataclass
5
-from typing import Dict, List, Optional, Union, TYPE_CHECKING
5
+from typing import TYPE_CHECKING, Dict, List, Optional, Union
6
7
from vllm.block import LogicalTokenBlock
8
-from vllm.sampling_params import SamplingParams
8
from vllm.lora.request import LoRARequest
9
+from vllm.sampling_params import SamplingParams
10
11
if TYPE_CHECKING:
12
import torch
13
+
14
from vllm.spec_decode.metrics import SpecDecodeWorkerMetrics
15
16
vllm/spec_decode/batch_expansion.py
+7
-8
@@ -1,16 +1,15 @@
1
-from typing import Iterator, List, Tuple, Optional, Dict
1
from itertools import chain, count
2
+from typing import Dict, Iterator, List, Optional, Tuple
3
4
import torch
5
6
-from vllm.sequence import (SamplerOutput, SequenceGroupMetadata, SequenceData)
7
-from vllm.worker.worker import Worker
8
-from vllm.spec_decode.util import (nvtx_range, sampler_output_to_torch,
9
- get_all_seq_ids,
6
+from vllm.sequence import SamplerOutput, SequenceData, SequenceGroupMetadata
7
+from vllm.spec_decode.interfaces import (SpeculativeProposals,
8
+ SpeculativeScorer, SpeculativeScores)
9
+from vllm.spec_decode.util import (get_all_seq_ids, nvtx_range,
10
+ sampler_output_to_torch,
11
split_batch_by_proposal_len)
11
-from vllm.spec_decode.interfaces import (SpeculativeScorer,
12
- SpeculativeProposals,
13
- SpeculativeScores)
12
+from vllm.worker.worker import Worker
13
14
SeqId = int
15
TargetSeqId = int
vllm/spec_decode/interfaces.py
+2
-2
@@ -1,6 +1,6 @@
1
-from typing import List, Tuple, Optional, Dict
2
-from dataclasses import dataclass
1
from abc import ABC, abstractmethod
2
+from dataclasses import dataclass
3
+from typing import Dict, List, Optional, Tuple
4
5
import torch
6
vllm/spec_decode/metrics.py
+5
-4
@@ -1,10 +1,11 @@
1
-import torch
1
+import time
2
from dataclasses import dataclass
3
+from typing import Callable, Optional
4
+
5
+import torch
6
+
7
from vllm.model_executor.layers.rejection_sampler import RejectionSampler
4
-from typing import Optional
8
from vllm.utils import is_pin_memory_available
6
-import time
7
-from typing import Callable
9
10
11
@dataclass
vllm/spec_decode/multi_step_worker.py
+2
-2
@@ -1,13 +1,13 @@
1
-from typing import List, Dict, Optional, Tuple
1
import copy
2
+from typing import Dict, List, Optional, Tuple
3
4
import torch
5
6
from vllm.sequence import SamplerOutput, SequenceGroupMetadata
7
-from vllm.worker.worker import Worker
7
from vllm.spec_decode.interfaces import (SpeculativeProposals,
8
SpeculativeProposer)
9
from vllm.spec_decode.util import sampler_output_to_torch
10
+from vllm.worker.worker import Worker
11
12
13
class MultiStepWorker(Worker):
vllm/spec_decode/spec_decode_worker.py
+9
-9
@@ -1,20 +1,20 @@
1
-from typing import List, Tuple, Optional, Dict
1
from functools import cached_property
2
+from typing import Dict, List, Optional, Tuple
3
4
import torch
5
6
-from vllm.spec_decode.metrics import AsyncMetricsCollector
6
+from vllm.config import CacheConfig
7
+from vllm.model_executor.layers.rejection_sampler import RejectionSampler
8
from vllm.sequence import (SamplerOutput, SequenceGroupMetadata,
9
SequenceGroupOutput, SequenceOutput)
9
-from vllm.worker.worker import Worker
10
+from vllm.spec_decode.batch_expansion import BatchExpansionTop1Scorer
11
+from vllm.spec_decode.interfaces import (SpeculativeProposals,
12
+ SpeculativeScorer, SpeculativeScores)
13
+from vllm.spec_decode.metrics import AsyncMetricsCollector
14
from vllm.spec_decode.multi_step_worker import MultiStepWorker
11
-from vllm.model_executor.layers.rejection_sampler import RejectionSampler
12
-from vllm.config import CacheConfig
13
-from vllm.spec_decode.util import (nvtx_range, get_all_seq_ids,
15
+from vllm.spec_decode.util import (get_all_seq_ids, nvtx_range,
16
split_batch_by_proposal_len)
15
-from vllm.spec_decode.interfaces import SpeculativeProposals, SpeculativeScores
16
-from vllm.spec_decode.batch_expansion import BatchExpansionTop1Scorer
17
-from vllm.spec_decode.interfaces import SpeculativeScorer
17
+from vllm.worker.worker import Worker
18
19
20
class SpecDecodeWorker:
vllm/spec_decode/util.py
+5
-3
@@ -1,8 +1,10 @@
1
-import torch
2
-from typing import List, Tuple
3
-from vllm.sequence import SequenceGroupMetadata, SamplerOutput
1
from contextlib import contextmanager
2
from itertools import chain
3
+from typing import List, Tuple
4
+
5
+import torch
6
+
7
+from vllm.sequence import SamplerOutput, SequenceGroupMetadata
8
9
SeqId = int
10
vllm/transformers_utils/configs/__init__.py
+1
-1
@@ -1,10 +1,10 @@
1
from vllm.transformers_utils.configs.chatglm import ChatGLMConfig
2
-from vllm.transformers_utils.configs.mpt import MPTConfig
2
# RWConfig is for the original tiiuae/falcon-40b(-instruct) and
3
# tiiuae/falcon-7b(-instruct) models. Newer Falcon models will use the
4
# `FalconConfig` class from the official HuggingFace transformers library.
5
from vllm.transformers_utils.configs.falcon import RWConfig
6
from vllm.transformers_utils.configs.jais import JAISConfig
7
+from vllm.transformers_utils.configs.mpt import MPTConfig
8
9
__all__ = [
10
"ChatGLMConfig",
vllm/transformers_utils/configs/mpt.py
+1
@@ -4,6 +4,7 @@
4
"""A HuggingFace-style model configuration."""
5
import warnings
6
from typing import Any, Dict, Optional, Union
7
+
8
from transformers import PretrainedConfig
9
10
attn_config_defaults: Dict = {
vllm/transformers_utils/detokenizer.py
+6
-4
@@ -1,8 +1,10 @@
1
-from typing import List, Dict, Optional
1
+from typing import Dict, List, Optional
2
+
3
from transformers import PreTrainedTokenizer
3
-from vllm.sequence import Sequence, Logprob, SequenceGroup, SamplingParams
4
-from vllm.transformers_utils.tokenizer import (detokenize_incrementally,
5
- convert_prompt_ids_to_tokens)
4
+
5
+from vllm.sequence import Logprob, SamplingParams, Sequence, SequenceGroup
6
+from vllm.transformers_utils.tokenizer import (convert_prompt_ids_to_tokens,
7
+ detokenize_incrementally)
8
from vllm.transformers_utils.tokenizer_group.base_tokenizer_group import (
9
BaseTokenizerGroup)
10
vllm/transformers_utils/tokenizer.py
+1
-1
@@ -5,8 +5,8 @@ from transformers import (AutoTokenizer, PreTrainedTokenizer,
5
6
from vllm.logger import init_logger
7
from vllm.lora.request import LoRARequest
8
-from vllm.utils import make_async
8
from vllm.transformers_utils.tokenizers import *
9
+from vllm.utils import make_async
10
11
logger = init_logger(__name__)
12
vllm/transformers_utils/tokenizer_group/__init__.py
+2
-1
@@ -1,10 +1,11 @@
1
from typing import Optional
2
+
3
from vllm.config import TokenizerPoolConfig
4
+from vllm.engine.ray_utils import ray
5
from vllm.transformers_utils.tokenizer_group.base_tokenizer_group import (
6
BaseTokenizerGroup)
7
from vllm.transformers_utils.tokenizer_group.tokenizer_group import (
8
TokenizerGroup)
7
-from vllm.engine.ray_utils import ray
9
10
if ray:
11
from vllm.transformers_utils.tokenizer_group.ray_tokenizer_group import (
vllm/transformers_utils/tokenizer_group/ray_tokenizer_group.py
+2
-2
@@ -2,16 +2,16 @@ import asyncio
2
import os
3
from typing import List, Optional
4
5
+from ray.util.scheduling_strategies import NodeAffinitySchedulingStrategy
6
from transformers import PreTrainedTokenizer
7
8
from vllm.config import TokenizerPoolConfig
8
-from vllm.lora.request import LoRARequest
9
from vllm.engine.ray_utils import ray
10
+from vllm.lora.request import LoRARequest
11
from vllm.transformers_utils.tokenizer_group.base_tokenizer_group import (
12
BaseTokenizerGroup)
13
from vllm.transformers_utils.tokenizer_group.tokenizer_group import (
14
TokenizerGroup)
14
-from ray.util.scheduling_strategies import NodeAffinitySchedulingStrategy
15
16
17
class RayTokenizerGroupPool(BaseTokenizerGroup):
vllm/transformers_utils/tokenizer_group/tokenizer_group.py
+2
-2
@@ -4,11 +4,11 @@ from transformers import PreTrainedTokenizer
4
5
from vllm.lora.request import LoRARequest
6
from vllm.transformers_utils.tokenizer import (get_lora_tokenizer,
7
- get_lora_tokenizer_async)
7
+ get_lora_tokenizer_async,
8
+ get_tokenizer)
9
from vllm.transformers_utils.tokenizer_group.base_tokenizer_group import (
10
BaseTokenizerGroup)
11
from vllm.utils import LRUCache
11
-from vllm.transformers_utils.tokenizer import get_tokenizer
12
13
14
class TokenizerGroup(BaseTokenizerGroup):
vllm/utils.py
+8
-13
@@ -1,27 +1,22 @@
1
+import asyncio
2
import enum
3
+import gc
4
import os
5
import socket
6
import subprocess
7
import uuid
6
-import gc
8
+import warnings
9
+from collections import OrderedDict
10
+from functools import lru_cache, partial
11
from platform import uname
8
-from typing import List, Tuple, Union, Generic
9
-from packaging.version import parse, Version
12
+from typing import (Any, Awaitable, Callable, Generic, Hashable, List,
13
+ Optional, Tuple, TypeVar, Union)
14
15
import psutil
16
import torch
13
-import asyncio
14
-from functools import partial, lru_cache
15
-from typing import (
16
- Awaitable,
17
- Callable,
18
- TypeVar,
19
-)
20
-from collections import OrderedDict
21
-from typing import Any, Hashable, Optional
17
+from packaging.version import Version, parse
18
19
from vllm.logger import init_logger
24
-import warnings
20
21
T = TypeVar("T")
22
logger = init_logger(__name__)
vllm/worker/cache_engine.py
+1
-1
@@ -6,7 +6,7 @@ import torch
6
from vllm.attention import get_attn_backend
7
from vllm.config import CacheConfig, ModelConfig, ParallelConfig
8
from vllm.logger import init_logger
9
-from vllm.utils import is_pin_memory_available, STR_DTYPE_TO_TORCH_DTYPE
9
+from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE, is_pin_memory_available
10
11
logger = init_logger(__name__)
12
vllm/worker/model_runner.py
+7
-8
@@ -1,29 +1,28 @@
1
import contextlib
2
import time
3
-from typing import Dict, List, Optional, Tuple, Set
3
+from typing import Dict, List, Optional, Set, Tuple
4
5
import numpy as np
6
import torch
7
import torch.nn as nn
8
9
from vllm.attention import AttentionMetadata, get_attn_backend
10
-from vllm.config import (DeviceConfig, ModelConfig, LoRAConfig, ParallelConfig,
10
+from vllm.config import (DeviceConfig, LoRAConfig, ModelConfig, ParallelConfig,
11
SchedulerConfig)
12
from vllm.logger import init_logger
13
+from vllm.lora.layers import LoRAMapping
14
+from vllm.lora.request import LoRARequest
15
+from vllm.lora.worker_manager import LRUCacheWorkerLoRAManager
16
from vllm.model_executor import SamplingMetadata
17
from vllm.model_executor.model_loader import get_model
15
-from vllm.model_executor.parallel_utils import cupy_utils
18
+from vllm.model_executor.parallel_utils import cupy_utils, custom_all_reduce
19
from vllm.model_executor.parallel_utils.communication_op import (
20
broadcast_tensor_dict)
21
from vllm.model_executor.parallel_utils.parallel_state import (
22
with_cupy_nccl_for_all_reduce)
20
-from vllm.model_executor.parallel_utils import custom_all_reduce
23
from vllm.sampling_params import SamplingParams, SamplingType
24
from vllm.sequence import SamplerOutput, SequenceData, SequenceGroupMetadata
23
-from vllm.lora.worker_manager import LRUCacheWorkerLoRAManager
24
-from vllm.lora.layers import LoRAMapping
25
-from vllm.lora.request import LoRARequest
26
-from vllm.utils import (async_tensor_h2d, CudaMemoryProfiler,
25
+from vllm.utils import (CudaMemoryProfiler, async_tensor_h2d,
26
is_pin_memory_available, make_tensor_with_pad,
27
maybe_expand_dim)
28
vllm/worker/worker.py
+4
-4
@@ -1,13 +1,14 @@
1
"""A GPU worker class."""
2
import gc
3
import os
4
-from typing import Dict, List, Tuple, Set, Optional
4
+from typing import Dict, List, Optional, Set, Tuple
5
6
import torch
7
import torch.distributed
8
9
-from vllm.config import (CacheConfig, DeviceConfig, ModelConfig,
10
- ParallelConfig, SchedulerConfig, LoRAConfig)
9
+from vllm.config import (CacheConfig, DeviceConfig, LoRAConfig, ModelConfig,
10
+ ParallelConfig, SchedulerConfig)
11
+from vllm.lora.request import LoRARequest
12
from vllm.model_executor import set_random_seed
13
from vllm.model_executor.parallel_utils import cupy_utils
14
from vllm.model_executor.parallel_utils.communication_op import (
@@ -18,7 +19,6 @@ from vllm.model_executor.parallel_utils.parallel_state import (
19
from vllm.sequence import SamplerOutput, SequenceGroupMetadata
20
from vllm.worker.cache_engine import CacheEngine
21
from vllm.worker.model_runner import ModelRunner
21
-from vllm.lora.request import LoRARequest
22
23
24
class Worker: