@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 01bfb22b41
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

[CI] Try introducing isort. (#3495)

SangBin Cho committed Mar 25, 2024 at 23:59 UTC 01bfb22b4112ee813185366ab26985d172661a61
144 files changed +506 -459
.github/workflows/ruff.yml
+5 -2
@@ -25,10 +25,13 @@ jobs:
25 - name: Install dependencies
26 run: |
27 python -m pip install --upgrade pip
28 - pip install ruff==0.1.5 codespell==2.2.6 tomli==2.0.1
28 + pip install ruff==0.1.5 codespell==2.2.6 tomli==2.0.1 isort==5.13.2
29 - name: Analysing the code with ruff
30 run: |
31 ruff .
32 - name: Spelling check with codespell
33 run: |
34 - codespell --toml pyproject.toml
\ No newline at end of file
34 + codespell --toml pyproject.toml
35 + - name: Run isort
36 + run: |
37 + isort . --check-only
benchmarks/benchmark_prefix_caching.py
+1 -2
@@ -1,8 +1,7 @@
1 import argparse
2 import time
3
4 -from vllm import LLM
5 -from vllm import SamplingParams
4 +from vllm import LLM, SamplingParams
5
6 PROMPT = "You are a helpful assistant in recognizes the content of tables in markdown format. Here is a table as fellows. You need to answer my question about the table.\n# Table\n|Opening|Opening|Sl. No.|Film|Cast|Director|Music Director|Notes|\n|----|----|----|----|----|----|----|----|\n|J A N|9|1|Agni Pushpam|Jayabharathi, Kamalahasan|Jeassy|M. K. Arjunan||\n|J A N|16|2|Priyamvada|Mohan Sharma, Lakshmi, KPAC Lalitha|K. S. Sethumadhavan|V. Dakshinamoorthy||\n|J A N|23|3|Yakshagaanam|Madhu, Sheela|Sheela|M. S. Viswanathan||\n|J A N|30|4|Paalkkadal|Sheela, Sharada|T. K. Prasad|A. T. Ummer||\n|F E B|5|5|Amma|Madhu, Srividya|M. Krishnan Nair|M. K. Arjunan||\n|F E B|13|6|Appooppan|Thikkurissi Sukumaran Nair, Kamal Haasan|P. Bhaskaran|M. S. Baburaj||\n|F E B|20|7|Srishti|Chowalloor Krishnankutty, Ravi Alummoodu|K. T. Muhammad|M. S. Baburaj||\n|F E B|20|8|Vanadevatha|Prem Nazir, Madhubala|Yusufali Kechery|G. Devarajan||\n|F E B|27|9|Samasya|Madhu, Kamalahaasan|K. Thankappan|Shyam||\n|F E B|27|10|Yudhabhoomi|K. P. Ummer, Vidhubala|Crossbelt Mani|R. K. Shekhar||\n|M A R|5|11|Seemantha Puthran|Prem Nazir, Jayabharathi|A. B. Raj|M. K. Arjunan||\n|M A R|12|12|Swapnadanam|Rani Chandra, Dr. Mohandas|K. G. George|Bhaskar Chandavarkar||\n|M A R|19|13|Thulavarsham|Prem Nazir, sreedevi, Sudheer|N. Sankaran Nair|V. Dakshinamoorthy||\n|M A R|20|14|Aruthu|Kaviyoor Ponnamma, Kamalahasan|Ravi|G. Devarajan||\n|M A R|26|15|Swimming Pool|Kamal Haasan, M. G. Soman|J. Sasikumar|M. K. Arjunan||\n\n# Question\nWhat' s the content in the (1,1) cells\n" # noqa: E501
7
benchmarks/benchmark_serving.py
+3 -6
@@ -25,15 +25,12 @@ from datetime import datetime
25 from typing import AsyncGenerator, List, Tuple
26
27 import numpy as np
28 +from backend_request_func import (ASYNC_REQUEST_FUNCS, RequestFuncInput,
29 + RequestFuncOutput)
30 from tqdm.asyncio import tqdm
31 from transformers import PreTrainedTokenizerBase
30 -from vllm.transformers_utils.tokenizer import get_tokenizer
32
32 -from backend_request_func import (
33 - ASYNC_REQUEST_FUNCS,
34 - RequestFuncInput,
35 - RequestFuncOutput,
36 -)
33 +from vllm.transformers_utils.tokenizer import get_tokenizer
34
35
36 @dataclass
benchmarks/benchmark_throughput.py
+1 -1
@@ -6,9 +6,9 @@ import time
6 from typing import List, Optional, Tuple
7
8 import torch
9 +from tqdm import tqdm
10 from transformers import (AutoModelForCausalLM, AutoTokenizer,
11 PreTrainedTokenizerBase)
11 -from tqdm import tqdm
12
13
14 def sample_requests(
benchmarks/kernels/benchmark_mixtral_moe.py
+3 -1
@@ -2,11 +2,13 @@ import json
2 import os
3 import sys
4
5 -from vllm.model_executor.layers.fused_moe import fused_moe, get_config_file_name
5 import torch
6 import torch.nn.functional as F
7 import triton
8
9 +from vllm.model_executor.layers.fused_moe import (fused_moe,
10 + get_config_file_name)
11 +
12 os.environ['CUDA_VISIBLE_DEVICES'] = '0'
13
14
benchmarks/kernels/benchmark_paged_attention.py
+2 -2
@@ -1,12 +1,12 @@
1 -from typing import Optional
1 import argparse
2 import random
3 import time
4 +from typing import Optional
5
6 import torch
7
8 -from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE, create_kv_caches_with_random
8 from vllm._C import ops
9 +from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE, create_kv_caches_with_random
10
11 NUM_BLOCKS = 1024
12 PARTITION_SIZE = 512
benchmarks/kernels/benchmark_rope.py
+4 -3
@@ -1,9 +1,10 @@
1 +import argparse
2 +from itertools import accumulate
3 from typing import Optional
4
3 -import argparse
4 -import torch
5 import nvtx
6 -from itertools import accumulate
6 +import torch
7 +
8 from vllm.model_executor.layers.rotary_embedding import get_rope
9
10
cmake/hipify.py
+1 -1
@@ -9,8 +9,8 @@
9 #
10
11 import argparse
12 -import shutil
12 import os
13 +import shutil
14
15 from torch.utils.hipify.hipify_python import hipify
16
collect_env.py
+1 -1
@@ -6,10 +6,10 @@
6 # Run it with `python collect_env.py` or `python -m torch.utils.collect_env`
7 import datetime
8 import locale
9 +import os
10 import re
11 import subprocess
12 import sys
12 -import os
13 from collections import namedtuple
14
15 try:
docs/source/conf.py
+2 -1
@@ -10,10 +10,11 @@
10 # add these directories to sys.path here. If the directory is relative to the
11 # documentation root, use os.path.abspath to make it absolute, like shown here.
12
13 +import logging
14 import os
15 import sys
16 +
17 from sphinx.ext import autodoc
16 -import logging
18
19 sys.path.insert(0, os.path.abspath(os.path.join('..', '..')))
20
examples/gradio_openai_chatbot_webserver.py
+2 -1
@@ -1,6 +1,7 @@
1 import argparse
2 -from openai import OpenAI
2 +
3 import gradio as gr
4 +from openai import OpenAI
5
6 # Argument parser setup
7 parser = argparse.ArgumentParser(
examples/llm_engine_example.py
+1 -1
@@ -1,7 +1,7 @@
1 import argparse
2 from typing import List, Tuple
3
4 -from vllm import EngineArgs, LLMEngine, SamplingParams, RequestOutput
4 +from vllm import EngineArgs, LLMEngine, RequestOutput, SamplingParams
5
6
7 def create_test_prompts() -> List[Tuple[str, SamplingParams]]:
examples/multilora_inference.py
+2 -2
@@ -5,11 +5,11 @@ for offline inference.
5 Requires HuggingFace credentials for access to Llama2.
6 """
7
8 -from typing import Optional, List, Tuple
8 +from typing import List, Optional, Tuple
9
10 from huggingface_hub import snapshot_download
11
12 -from vllm import EngineArgs, LLMEngine, SamplingParams, RequestOutput
12 +from vllm import EngineArgs, LLMEngine, RequestOutput, SamplingParams
13 from vllm.lora.request import LoRARequest
14
15
examples/offline_inference_distributed.py
+3 -1
@@ -5,11 +5,13 @@ distributively on a multi-nodes cluster.
5 Learn more about Ray Data in https://docs.ray.io/en/latest/data/data.html
6 """
7
8 -from vllm import LLM, SamplingParams
8 from typing import Dict
9 +
10 import numpy as np
11 import ray
12
13 +from vllm import LLM, SamplingParams
14 +
15 # Create a sampling params object.
16 sampling_params = SamplingParams(temperature=0.8, top_p=0.95)
17
format.sh
+42
@@ -25,6 +25,7 @@ YAPF_VERSION=$(yapf --version | awk '{print $2}')
25 RUFF_VERSION=$(ruff --version | awk '{print $2}')
26 MYPY_VERSION=$(mypy --version | awk '{print $2}')
27 CODESPELL_VERSION=$(codespell --version)
28 +ISORT_VERSION=$(isort --vn)
29
30 # # params: tool name, tool version, required version
31 tool_version_check() {
@@ -37,6 +38,7 @@ tool_version_check() {
38 tool_version_check "yapf" $YAPF_VERSION "$(grep yapf requirements-dev.txt | cut -d'=' -f3)"
39 tool_version_check "ruff" $RUFF_VERSION "$(grep "ruff==" requirements-dev.txt | cut -d'=' -f3)"
40 tool_version_check "mypy" "$MYPY_VERSION" "$(grep mypy requirements-dev.txt | cut -d'=' -f3)"
41 +tool_version_check "isort" "$ISORT_VERSION" "$(grep isort requirements-dev.txt | cut -d'=' -f3)"
42 tool_version_check "codespell" "$CODESPELL_VERSION" "$(grep codespell requirements-dev.txt | cut -d'=' -f3)"
43
44 YAPF_FLAGS=(
@@ -178,6 +180,46 @@ else
180 lint_changed
181 fi
182
183 +# check spelling of specified files
184 +isort_check() {
185 + isort "$@"
186 +}
187 +
188 +isort_check_all(){
189 + isort .
190 +}
191 +
192 +# Spelling check of files that differ from main branch.
193 +isort_check_changed() {
194 + # The `if` guard ensures that the list of filenames is not empty, which
195 + # could cause ruff to receive 0 positional arguments, making it hang
196 + # waiting for STDIN.
197 + #
198 + # `diff-filter=ACM` and $MERGEBASE is to ensure we only lint files that
199 + # exist on both branches.
200 + MERGEBASE="$(git merge-base origin/main HEAD)"
201 +
202 + if ! git diff --diff-filter=ACM --quiet --exit-code "$MERGEBASE" -- '*.py' '*.pyi' &>/dev/null; then
203 + git diff --name-only --diff-filter=ACM "$MERGEBASE" -- '*.py' '*.pyi' | xargs \
204 + isort
205 + fi
206 +}
207 +
208 +# Run Isort
209 +# This flag runs spell check of individual files. --files *must* be the first command line
210 +# arg to use this option.
211 +if [[ "$1" == '--files' ]]; then
212 + isort_check "${@:2}"
213 + # If `--all` is passed, then any further arguments are ignored and the
214 + # entire python directory is linted.
215 +elif [[ "$1" == '--all' ]]; then
216 + isort_check_all
217 +else
218 + # Check spelling only of the files that changed in last commit.
219 + isort_check_changed
220 +fi
221 +echo 'vLLM isort: Done'
222 +
223 if ! git diff --quiet &>/dev/null; then
224 echo 'Reformatted files. Please review and stage the changes.'
225 echo 'Changes not staged for commit:'
pyproject.toml
+4
@@ -51,3 +51,7 @@ exclude = "vllm/model_executor/parallel_utils/|vllm/model_executor/models/"
51 [tool.codespell]
52 ignore-words-list = "dout, te, indicies"
53 skip = "./tests/prompts"
54 +
55 +[tool.isort]
56 +use_parentheses = true
57 +skip_gitignore = true
requirements-dev.txt
+1
@@ -4,6 +4,7 @@ toml==0.10.2
4 tomli==2.0.1
5 ruff==0.1.5
6 codespell==2.2.6
7 +isort==5.13.2
8
9 # type checking
10 mypy==0.991
setup.py
+5 -5
@@ -1,16 +1,16 @@
1 import io
2 +import logging
3 import os
4 import re
4 -import logging
5 import subprocess
6 import sys
7 +from shutil import which
8 from typing import List
9
9 -from packaging.version import parse, Version
10 -from setuptools import setup, find_packages, Extension
11 -from setuptools.command.build_ext import build_ext
12 -from shutil import which
10 import torch
11 +from packaging.version import Version, parse
12 +from setuptools import Extension, find_packages, setup
13 +from setuptools.command.build_ext import build_ext
14 from torch.utils.cpp_extension import CUDA_HOME
15
16 ROOT_DIR = os.path.dirname(__file__)
tests/async_engine/test_chat_template.py
+3 -3
@@ -1,12 +1,12 @@
1 -from dataclasses import dataclass
1 import os
2 import pathlib
3 +from dataclasses import dataclass
4
5 import pytest
6
7 -from vllm.transformers_utils.tokenizer import get_tokenizer
8 -from vllm.entrypoints.openai.serving_chat import OpenAIServingChat
7 from vllm.entrypoints.openai.protocol import ChatCompletionRequest
8 +from vllm.entrypoints.openai.serving_chat import OpenAIServingChat
9 +from vllm.transformers_utils.tokenizer import get_tokenizer
10
11 chatml_jinja_path = pathlib.Path(os.path.dirname(os.path.abspath(
12 __file__))).parent.parent / "examples/template_chatml.jinja"
tests/conftest.py
+1 -1
@@ -6,8 +6,8 @@ import torch
6 from transformers import AutoModelForCausalLM
7
8 from vllm import LLM, SamplingParams
9 -from vllm.transformers_utils.tokenizer import get_tokenizer
9 from vllm.config import TokenizerPoolConfig
10 +from vllm.transformers_utils.tokenizer import get_tokenizer
11
12 _TEST_DIR = os.path.dirname(__file__)
13 _TEST_PROMPTS = [os.path.join(_TEST_DIR, "prompts", "example.txt")]
tests/core/test_block_manager.py
+5 -4
@@ -1,13 +1,14 @@
1 -import pytest
1 import time
2 from typing import List
3
4 +import pytest
5 +
6 from vllm import SamplingParams
7 from vllm.block import PhysicalTokenBlock
7 -from vllm.core.block_manager import (UncachedBlockAllocator, BlockSpaceManager,
8 - AllocStatus)
8 +from vllm.core.block_manager import (AllocStatus, BlockSpaceManager,
9 + UncachedBlockAllocator)
10 +from vllm.sequence import Logprob, Sequence, SequenceGroup, SequenceStatus
11 from vllm.utils import Device
10 -from vllm.sequence import Sequence, SequenceGroup, SequenceStatus, Logprob
12
13 from .utils import create_dummy_prompt
14
tests/core/test_scheduler.py
+3 -2
@@ -1,10 +1,11 @@
1 +import time
2 from typing import List
3 +
4 import pytest # noqa
3 -import time
5
6 from vllm.config import CacheConfig, SchedulerConfig
7 from vllm.core.scheduler import Scheduler
7 -from vllm.sequence import SequenceGroup, Logprob
8 +from vllm.sequence import Logprob, SequenceGroup
9
10 from .utils import create_dummy_prompt
11
tests/distributed/test_comm_ops.py
+3 -5
@@ -3,14 +3,12 @@
3 Run `pytest tests/distributed/test_comm_ops.py --forked`.
4 """
5 import pytest
6 -import torch
6 import ray
7 +import torch
8
9 from vllm.model_executor.parallel_utils.communication_op import (
10 - tensor_model_parallel_all_reduce,
11 - tensor_model_parallel_all_gather,
12 - broadcast_tensor_dict,
13 -)
10 + broadcast_tensor_dict, tensor_model_parallel_all_gather,
11 + tensor_model_parallel_all_reduce)
12 from vllm.test_utils import (init_test_distributed_environment,
13 multi_process_tensor_parallel)
14
tests/distributed/test_custom_all_reduce.py
+1 -1
@@ -1,6 +1,6 @@
1 +import os
2 import random
3
3 -import os
4 import pytest
5 import ray
6 import torch
tests/entrypoints/test_guided_processors.py
+3 -3
@@ -1,11 +1,11 @@
1 # This unit test should be moved to a new
2 # tests/test_guided_decoding directory.
3
4 -from transformers import AutoTokenizer
4 import torch
5 +from transformers import AutoTokenizer
6
7 -from vllm.model_executor.guided_logits_processors import (RegexLogitsProcessor,
8 - JSONLogitsProcessor)
7 +from vllm.model_executor.guided_logits_processors import (JSONLogitsProcessor,
8 + RegexLogitsProcessor)
9
10 TEST_SCHEMA = {
11 "type": "object",
tests/entrypoints/test_openai_server.py
+7 -8
@@ -1,22 +1,21 @@
1 +# imports for guided decoding tests
2 +import json
3 import os
4 +import re
5 import subprocess
6 +import sys
7 import time
8
5 -import sys
9 +import jsonschema
10 +import openai # use the official client for correctness check
11 import pytest
7 -import requests
12 # using Ray for overall ease of process management, parallel requests,
13 # and debugging.
14 import ray
11 -import openai # use the official client for correctness check
15 +import requests
16 # downloading lora to test lora requests
17 from huggingface_hub import snapshot_download
18
15 -# imports for guided decoding tests
16 -import json
17 -import jsonschema
18 -import re
19 -
19 from vllm.transformers_utils.tokenizer import get_tokenizer
20
21 MAX_SERVER_START_WAIT_S = 600 # wait for server to start for 60 seconds
tests/kernels/conftest.py
+1
@@ -1,4 +1,5 @@
1 import pytest
2 +
3 from vllm.utils import create_kv_caches_with_random
4
5
tests/kernels/test_activation.py
+1 -1
@@ -2,10 +2,10 @@ from typing import Type
2
3 import pytest
4 import torch
5 +from allclose_default import get_default_atol, get_default_rtol
6
7 from vllm.model_executor.layers.activation import (FastGELU, GeluAndMul,
8 NewGELU, SiluAndMul)
8 -from allclose_default import get_default_atol, get_default_rtol
9
10 DTYPES = [torch.half, torch.bfloat16, torch.float]
11 NUM_TOKENS = [7, 83, 2048] # Arbitrary values for testing
tests/kernels/test_attention.py
+3 -4
@@ -3,13 +3,12 @@ from typing import List, Optional, Tuple
3
4 import pytest
5 import torch
6 +from allclose_default import get_default_atol, get_default_rtol
7 from xformers import ops as xops
8 from xformers.ops.fmha.attn_bias import BlockDiagonalCausalMask
9
9 -from vllm._C import ops, cache_ops
10 -from vllm.utils import get_max_shared_memory_bytes
11 -from vllm.utils import is_hip
12 -from allclose_default import get_default_atol, get_default_rtol
10 +from vllm._C import cache_ops, ops
11 +from vllm.utils import get_max_shared_memory_bytes, is_hip
12
13 FLOAT32_BYTES = torch.finfo(torch.float).bits // 8
14 # This will change depending on the compute capability.
tests/kernels/test_cache.py
+1 -2
@@ -1,10 +1,9 @@
1 import random
2 +from typing import Tuple
3
4 import pytest
5 import torch
6
6 -from typing import Tuple
7 -
7 from vllm._C import cache_ops
8
9 COPYING_DIRECTION = [('cuda', 'cpu'), ('cuda', 'cuda'), ('cpu', 'cuda')]
tests/kernels/test_moe.py
+1 -1
@@ -7,8 +7,8 @@ import torch
7 from transformers import MixtralConfig
8 from transformers.models.mixtral.modeling_mixtral import MixtralSparseMoeBlock
9
10 -from vllm.model_executor.layers.fused_moe import fused_moe
10 from vllm.model_executor.layers.activation import SiluAndMul
11 +from vllm.model_executor.layers.fused_moe import fused_moe
12 from vllm.model_executor.models.mixtral import MixtralMoE
13
14
tests/kernels/test_pos_encoding.py
+2 -1
@@ -1,9 +1,10 @@
1 +from itertools import accumulate
2 from typing import List, Optional
3
4 import pytest
5 import torch
6 from allclose_default import get_default_atol, get_default_rtol
6 -from itertools import accumulate
7 +
8 from vllm.model_executor.layers.rotary_embedding import get_rope
9
10 IS_NEOX_STYLE = [True, False]
tests/kernels/test_prefix_prefill.py
+3 -2
@@ -1,12 +1,13 @@
1 import random
2 -import pytest
2 import time
3
4 +import pytest
5 import torch
6 -from vllm.attention.ops.prefix_prefill import context_attention_fwd
6 from xformers import ops as xops
7 from xformers.ops.fmha.attn_bias import BlockDiagonalCausalFromBottomRightMask
8
9 +from vllm.attention.ops.prefix_prefill import context_attention_fwd
10 +
11 NUM_HEADS = [64]
12 NUM_QUERIES_PER_KV = [1, 8, 64]
13 HEAD_SIZES = [128]
tests/kernels/test_rand.py
+3 -2
@@ -1,7 +1,8 @@
1 -import torch
2 -import pytest
1 import random
2
3 +import pytest
4 +import torch
5 +
6 from vllm.model_executor.layers.ops.rand import seeded_uniform
7 from vllm.model_executor.utils import set_random_seed
8
tests/kernels/test_sampler.py
+4 -4
@@ -1,15 +1,15 @@
1 import gc
2
3 -import torch
3 import pytest
4 +import torch
5 import triton
6 import triton.language as tl
7
8 from vllm.model_executor.layers.ops.sample import (
9 - _uniform_to_exponential, sample, get_num_triton_sampler_splits,
10 - MAX_TRITON_N_COLS)
11 -from vllm.model_executor.utils import set_random_seed
9 + MAX_TRITON_N_COLS, _uniform_to_exponential, get_num_triton_sampler_splits,
10 + sample)
11 from vllm.model_executor.sampling_metadata import SamplingTensors
12 +from vllm.model_executor.utils import set_random_seed
13
14 SINGLE_SPLIT_VOCAB_SIZE = 32000 # llama/mistral/mixtral vocab size
15 MULTI_SPLIT_VOCAB_SIZE = MAX_TRITON_N_COLS + 100
tests/lora/conftest.py
+4 -4
@@ -2,7 +2,7 @@ import contextlib
2 import gc
3 import tempfile
4 from collections import OrderedDict
5 -from unittest.mock import patch, MagicMock
5 +from unittest.mock import MagicMock, patch
6
7 import pytest
8 import ray
@@ -12,13 +12,13 @@ from huggingface_hub import snapshot_download
12
13 import vllm
14 from vllm.config import LoRAConfig
15 -from vllm.model_executor.layers.sampler import Sampler
16 -from vllm.model_executor.layers.logits_processor import LogitsProcessor
17 -from vllm.model_executor.model_loader import get_model
15 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
16 MergedColumnParallelLinear,
17 RowParallelLinear)
18 +from vllm.model_executor.layers.logits_processor import LogitsProcessor
19 +from vllm.model_executor.layers.sampler import Sampler
20 from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
21 +from vllm.model_executor.model_loader import get_model
22 from vllm.model_executor.parallel_utils.parallel_state import (
23 destroy_model_parallel, initialize_model_parallel)
24
tests/lora/test_layer_variation.py
+4 -2
@@ -1,12 +1,14 @@
1 +import tempfile
2 +from random import sample
3 from typing import List, Optional
4 +
5 import peft
6 import pytest
4 -from random import sample
5 -import tempfile
7 from transformers import AutoModelForCausalLM
8
9 import vllm
10 from vllm.lora.request import LoRARequest
11 +
12 from .conftest import cleanup
13
14 MODEL_PATH = "Felladrin/Llama-68M-Chat-v1"
tests/lora/test_layers.py
+14 -18
@@ -1,32 +1,28 @@
1 -import pytest
1 import random
2 from copy import deepcopy
3 from dataclasses import dataclass
5 -from typing import List, Optional, Dict, Tuple
4 +from typing import Dict, List, Optional, Tuple
5
6 +import pytest
7 import torch
8 import torch.nn.functional as F
9
10 -from vllm.lora.layers import (
11 - ColumnParallelLinearWithLoRA,
12 - MergedColumnParallelLinearWithLoRA,
13 - QKVParallelLinearWithLora,
14 - VocabParallelEmbeddingWithLoRA,
15 - RowParallelLinearWithLoRA,
16 - LogitsProcessorWithLoRA,
17 - LoRAMapping,
18 - BaseLayerWithLoRA,
19 -)
20 -from vllm.lora.models import (LoRALayerWeights, convert_mapping,
21 - PackedLoRALayerWeights)
10 from vllm.config import LoRAConfig
23 -from vllm.model_executor.layers.logits_processor import LogitsProcessor
11 +from vllm.lora.layers import (BaseLayerWithLoRA, ColumnParallelLinearWithLoRA,
12 + LogitsProcessorWithLoRA, LoRAMapping,
13 + MergedColumnParallelLinearWithLoRA,
14 + QKVParallelLinearWithLora,
15 + RowParallelLinearWithLoRA,
16 + VocabParallelEmbeddingWithLoRA)
17 +from vllm.lora.models import (LoRALayerWeights, PackedLoRALayerWeights,
18 + convert_mapping)
19 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
20 MergedColumnParallelLinear,
26 - RowParallelLinear,
27 - QKVParallelLinear)
21 + QKVParallelLinear,
22 + RowParallelLinear)
23 +from vllm.model_executor.layers.logits_processor import LogitsProcessor
24 from vllm.model_executor.layers.vocab_parallel_embedding import (
29 - VocabParallelEmbedding, ParallelLMHead)
25 + ParallelLMHead, VocabParallelEmbedding)
26 from vllm.model_executor.utils import set_random_seed
27
28 from .utils import DummyLoRAManager
tests/lora/test_llama.py
+1
@@ -3,6 +3,7 @@ import ray
3
4 import vllm
5 from vllm.lora.request import LoRARequest
6 +
7 from .conftest import cleanup
8
9 MODEL_PATH = "meta-llama/Llama-2-7b-hf"
tests/lora/test_lora_manager.py
+4 -4
@@ -8,11 +8,11 @@ from torch import nn
8
9 from vllm.config import LoRAConfig
10 from vllm.lora.layers import (ColumnParallelLinearWithLoRA,
11 - RowParallelLinearWithLoRA,
12 - MergedColumnParallelLinearWithLoRA)
11 + MergedColumnParallelLinearWithLoRA,
12 + RowParallelLinearWithLoRA)
13 from vllm.lora.lora import LoRALayerWeights, PackedLoRALayerWeights
14 -from vllm.lora.models import (LoRAModel, LoRAModelManager,
15 - LRUCacheLoRAModelManager, LoRAMapping)
14 +from vllm.lora.models import (LoRAMapping, LoRAModel, LoRAModelManager,
15 + LRUCacheLoRAModelManager)
16 from vllm.lora.request import LoRARequest
17 from vllm.lora.worker_manager import (LRUCacheWorkerLoRAManager,
18 WorkerLoRAManager)
tests/lora/test_tokenizer_group.py
+3 -1
@@ -1,8 +1,10 @@
1 import pytest
2 from transformers import AutoTokenizer, PreTrainedTokenizerBase
3 +
4 from vllm.lora.request import LoRARequest
4 -from vllm.transformers_utils.tokenizer_group import get_tokenizer_group
5 from vllm.transformers_utils.tokenizer import get_lora_tokenizer
6 +from vllm.transformers_utils.tokenizer_group import get_tokenizer_group
7 +
8 from ..conftest import get_tokenizer_pool_config
9
10
tests/lora/test_utils.py
+1 -1
@@ -2,8 +2,8 @@ from collections import OrderedDict
2
3 from torch import nn
4
5 +from vllm.lora.utils import parse_fine_tuned_lora_name, replace_submodule
6 from vllm.utils import LRUCache
6 -from vllm.lora.utils import (parse_fine_tuned_lora_name, replace_submodule)
7
8
9 def test_parse_fine_tuned_lora_name():
tests/lora/test_worker.py
+2 -2
@@ -3,10 +3,10 @@ import random
3 import tempfile
4 from unittest.mock import patch
5
6 +from vllm.config import (DeviceConfig, LoRAConfig, ModelConfig, ParallelConfig,
7 + SchedulerConfig)
8 from vllm.lora.models import LoRAMapping
9 from vllm.lora.request import LoRARequest
8 -from vllm.config import (ModelConfig, ParallelConfig, SchedulerConfig,
9 - DeviceConfig, LoRAConfig)
10 from vllm.worker.worker import Worker
11
12
tests/models/test_marlin.py
+3 -1
@@ -11,9 +11,11 @@ up to 3 times to see if we pass.
11 Run `pytest tests/models/test_marlin.py --forked`.
12 """
13
14 +from dataclasses import dataclass
15 +
16 import pytest
17 import torch
16 -from dataclasses import dataclass
18 +
19 from vllm.model_executor.layers.quantization import (
20 _QUANTIZATION_CONFIG_REGISTRY)
21
tests/samplers/test_logprobs.py
+1 -1
@@ -1,7 +1,7 @@
1 import pytest
2 import torch
3 -from tests.conftest import VllmRunner
3
4 +from tests.conftest import VllmRunner
5 from vllm import SamplingParams
6
7 MODELS = ["facebook/opt-125m"]
tests/samplers/test_rejection_sampler.py
+2 -3
@@ -1,13 +1,12 @@
1 """Tests for rejection sampling."""
2 -import pytest
2 from typing import List, Tuple
3
4 +import pytest
5 import torch
6 import torch.nn.functional as F
7
8 -from vllm.model_executor.utils import set_random_seed
9 -
8 from vllm.model_executor.layers.rejection_sampler import RejectionSampler
9 +from vllm.model_executor.utils import set_random_seed
10
11 CUDA_DEVICES = [
12 f"cuda:{i}" for i in range(1 if torch.cuda.device_count() == 1 else 2)
tests/samplers/test_sampler.py
+1 -2
@@ -1,11 +1,10 @@
1 import random
2 -from typing import Tuple, List
2 +from typing import List, Optional, Tuple
3 from unittest.mock import patch
4
5 import pytest
6 import torch
7 from transformers import GenerationConfig, GenerationMixin
8 -from typing import Optional
8
9 from vllm.model_executor.layers.sampler import Sampler
10 from vllm.model_executor.utils import set_random_seed
tests/samplers/test_seeded_generate.py
+1 -1
@@ -8,8 +8,8 @@ from itertools import combinations
8
9 import pytest
10
11 -from vllm.model_executor.utils import set_random_seed
11 from vllm import SamplingParams
12 +from vllm.model_executor.utils import set_random_seed
13
14 MODEL = "facebook/opt-125m"
15 RANDOM_SEEDS = list(range(5))
tests/spec_decode/test_batch_expansion.py
+2 -2
@@ -1,9 +1,9 @@
1 -import torch
1 import pytest
2 +import torch
3
4 from vllm.spec_decode.batch_expansion import BatchExpansionTop1Scorer
5
6 -from .utils import mock_worker, create_seq_group_metadata_from_prompts
6 +from .utils import create_seq_group_metadata_from_prompts, mock_worker
7
8
9 @pytest.mark.parametrize('num_target_seq_ids', [100])
tests/spec_decode/test_metrics.py
+3 -3
@@ -1,9 +1,9 @@
1 -import torch
1 import math
3 -import pytest
4 -
2 from unittest.mock import MagicMock
3
4 +import pytest
5 +import torch
6 +
7 from vllm.spec_decode.metrics import AsyncMetricsCollector
8
9
tests/spec_decode/test_multi_step_worker.py
+10 -9
@@ -1,18 +1,19 @@
1 -import torch
1 import random
3 -import pytest
2 from unittest.mock import MagicMock
3
6 -from vllm.spec_decode.multi_step_worker import (MultiStepWorker,
7 - DraftModelTop1Proposer)
8 -from vllm.worker.worker import Worker
4 +import pytest
5 +import torch
6 +
7 from vllm.model_executor.utils import set_random_seed
8 from vllm.sequence import SamplerOutput
9 +from vllm.spec_decode.multi_step_worker import (DraftModelTop1Proposer,
10 + MultiStepWorker)
11 +from vllm.worker.worker import Worker
12
12 -from .utils import (create_execute_model_data, create_worker,
13 - create_seq_group_metadata_from_prompts, zero_kv_cache,
14 - patch_execute_model_with_seeds,
15 - assert_logprobs_dict_allclose, create_batch)
13 +from .utils import (assert_logprobs_dict_allclose, create_batch,
14 + create_execute_model_data,
15 + create_seq_group_metadata_from_prompts, create_worker,
16 + patch_execute_model_with_seeds, zero_kv_cache)
17
18
19 @pytest.mark.parametrize('num_steps', list(range(1, 17)))
tests/spec_decode/test_spec_decode_worker.py
+11 -9
@@ -1,18 +1,20 @@
1 -import torch
1 import random
3 -import pytest
2 from unittest.mock import MagicMock
3
4 +import pytest
5 +import torch
6 +
7 +from vllm.model_executor.layers.rejection_sampler import RejectionSampler
8 +from vllm.model_executor.utils import set_random_seed
9 +from vllm.spec_decode.interfaces import SpeculativeProposals
10 +from vllm.spec_decode.metrics import (AsyncMetricsCollector,
11 + SpecDecodeWorkerMetrics)
12 from vllm.spec_decode.multi_step_worker import MultiStepWorker
13 from vllm.spec_decode.spec_decode_worker import (SpecDecodeWorker,
14 split_num_cache_blocks_evenly)
9 -from vllm.spec_decode.interfaces import SpeculativeProposals
10 -from vllm.model_executor.utils import set_random_seed
11 -from vllm.model_executor.layers.rejection_sampler import RejectionSampler
12 -from .utils import (mock_worker, create_batch, ExecuteModelData,
13 - create_sampler_output_list)
14 -from vllm.spec_decode.metrics import (SpecDecodeWorkerMetrics,
15 - AsyncMetricsCollector)
15 +
16 +from .utils import (ExecuteModelData, create_batch, create_sampler_output_list,
17 + mock_worker)
18
19
20 @pytest.mark.parametrize('k', [1, 2, 6])
tests/spec_decode/test_utils.py
+4 -4
@@ -1,9 +1,9 @@
1 -from vllm.spec_decode.util import get_all_seq_ids
2 -from vllm.sequence import SequenceGroupMetadata
3 -from vllm.spec_decode.util import split_batch_by_proposal_len
1 +from unittest.mock import MagicMock
2
3 import pytest
6 -from unittest.mock import MagicMock
4 +
5 +from vllm.sequence import SequenceGroupMetadata
6 +from vllm.spec_decode.util import get_all_seq_ids, split_batch_by_proposal_len
7
8
9 def test_get_all_seq_ids():
tests/spec_decode/utils.py
+11 -9
@@ -1,17 +1,19 @@
1 -import torch
2 -from typing import List, Optional, Dict, Iterable, Union
1 +from dataclasses import dataclass, fields
2 +from itertools import count
3 +from typing import Dict, Iterable, List, Optional, Union
4 from unittest.mock import MagicMock
5
5 -from vllm.worker.worker import Worker
6 -from vllm.utils import get_distributed_init_method, get_ip, get_open_port
6 +import torch
7 +
8 from vllm.engine.arg_utils import EngineArgs
8 -from vllm.sequence import (Logprob, SequenceGroupMetadata, SequenceData,
9 - SamplerOutput, SequenceGroupOutput, SequenceOutput)
9 +from vllm.model_executor.utils import set_random_seed
10 from vllm.sampling_params import SamplingParams
11 +from vllm.sequence import (Logprob, SamplerOutput, SequenceData,
12 + SequenceGroupMetadata, SequenceGroupOutput,
13 + SequenceOutput)
14 +from vllm.utils import get_distributed_init_method, get_ip, get_open_port
15 from vllm.worker.cache_engine import CacheEngine
12 -from vllm.model_executor.utils import set_random_seed
13 -from itertools import count
14 -from dataclasses import dataclass, fields
16 +from vllm.worker.worker import Worker
17
18
19 @dataclass
tests/test_cache_block_hashing.py
+1 -1
@@ -7,8 +7,8 @@ from typing import List, Optional
7 import pytest
8
9 from vllm.lora.request import LoRARequest
10 -from vllm.transformers_utils.tokenizer_group import TokenizerGroup
10 from vllm.sequence import Sequence
11 +from vllm.transformers_utils.tokenizer_group import TokenizerGroup
12
13 # Make two prefixes with different first blocks.
14 prefix_start = [("You are an expert"), ("You are a")]
tests/test_sequence.py
+1 -1
@@ -1,6 +1,6 @@
1 import pytest
2
3 -from vllm.sequence import SequenceGroupOutput, SamplerOutput, SequenceOutput
3 +from vllm.sequence import SamplerOutput, SequenceGroupOutput, SequenceOutput
4
5
6 @pytest.fixture
tests/tokenization/test_cached_tokenizer.py
+3 -1
@@ -1,7 +1,9 @@
1 from copy import deepcopy
2 -from vllm.transformers_utils.tokenizer import get_cached_tokenizer
2 +
3 from transformers import AutoTokenizer
4
5 +from vllm.transformers_utils.tokenizer import get_cached_tokenizer
6 +
7
8 def test_cached_tokenizer():
9 reference_tokenizer = AutoTokenizer.from_pretrained("gpt2")
tests/tokenization/test_detokenize.py
+5 -5
@@ -1,12 +1,12 @@
1 -import pytest
1 +from typing import Dict, List
2
3 +import pytest
4 from transformers import AutoTokenizer
4 -from typing import List, Dict
5
6 -from vllm.sequence import Sequence, Logprob, SamplingParams, SequenceGroup
7 -from vllm.transformers_utils.tokenizer_group import get_tokenizer_group
8 -from vllm.transformers_utils.tokenizer import detokenize_incrementally
6 +from vllm.sequence import Logprob, SamplingParams, Sequence, SequenceGroup
7 from vllm.transformers_utils.detokenizer import Detokenizer
8 +from vllm.transformers_utils.tokenizer import detokenize_incrementally
9 +from vllm.transformers_utils.tokenizer_group import get_tokenizer_group
10
11 TRUTH = [
12 "Hello here, this is a simple test",
tests/tokenization/test_tokenizer_group.py
+4 -2
@@ -1,14 +1,16 @@
1 -import os
2 -import pytest
1 import asyncio
2 +import os
3 from unittest.mock import patch
4
5 +import pytest
6 from transformers import AutoTokenizer, PreTrainedTokenizerBase
7 +
8 from vllm.transformers_utils.tokenizer_group import get_tokenizer_group
9 from vllm.transformers_utils.tokenizer_group.ray_tokenizer_group import (
10 RayTokenizerGroupPool)
11 from vllm.transformers_utils.tokenizer_group.tokenizer_group import (
12 TokenizerGroup)
13 +
14 from ..conftest import get_tokenizer_pool_config
15
16
tests/worker/test_swap.py
+1 -1
@@ -1,8 +1,8 @@
1 import torch
2
3 from vllm.engine.arg_utils import EngineArgs
4 -from vllm.worker.worker import Worker
4 from vllm.utils import get_distributed_init_method, get_ip, get_open_port
5 +from vllm.worker.worker import Worker
6
7
8 def test_swap() -> None:
vllm/attention/__init__.py
+2 -1
@@ -1,4 +1,5 @@
1 -from vllm.attention.backends.abstract import AttentionBackend, AttentionMetadata
1 +from vllm.attention.backends.abstract import (AttentionBackend,
2 + AttentionMetadata)
3 from vllm.attention.layer import Attention
4 from vllm.attention.selector import get_attn_backend
5
vllm/attention/backends/flash_attn.py
+3 -2
@@ -7,12 +7,13 @@ flashinfer for all the attention operations.
7 from dataclasses import dataclass
8 from typing import Dict, List, Optional, Tuple, Type
9
10 -from flash_attn import flash_attn_varlen_func
10 import torch
11 +from flash_attn import flash_attn_varlen_func
12
13 from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
14 AttentionMetadata)
15 -from vllm.attention.ops.paged_attn import PagedAttention, PagedAttentionMetadata
15 +from vllm.attention.ops.paged_attn import (PagedAttention,
16 + PagedAttentionMetadata)
17
18
19 class FlashAttentionBackend(AttentionBackend):
vllm/attention/backends/xformers.py
+2 -1
@@ -11,7 +11,8 @@ from xformers.ops.fmha.attn_bias import (AttentionBias,
11
12 from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
13 AttentionMetadata)
14 -from vllm.attention.ops.paged_attn import PagedAttention, PagedAttentionMetadata
14 +from vllm.attention.ops.paged_attn import (PagedAttention,
15 + PagedAttentionMetadata)
16 from vllm.logger import init_logger
17 from vllm.utils import is_hip
18
vllm/attention/ops/paged_attn.py
+1 -2
@@ -3,8 +3,7 @@ from typing import Dict, List, Optional, Tuple
3
4 import torch
5
6 -from vllm._C import cache_ops
7 -from vllm._C import ops
6 +from vllm._C import cache_ops, ops
7 from vllm.attention.ops.prefix_prefill import context_attention_fwd
8
9 # Should be the same as PARTITION_SIZE in `paged_attention_v2_launcher`.
vllm/attention/selector.py
+4 -2
@@ -13,11 +13,13 @@ logger = init_logger(__name__)
13 def get_attn_backend(dtype: torch.dtype) -> AttentionBackend:
14 if _can_use_flash_attn(dtype):
15 logger.info("Using FlashAttention backend.")
16 - from vllm.attention.backends.flash_attn import FlashAttentionBackend # noqa: F401
16 + from vllm.attention.backends.flash_attn import ( # noqa: F401
17 + FlashAttentionBackend)
18 return FlashAttentionBackend
19 else:
20 logger.info("Using XFormers backend.")
20 - from vllm.attention.backends.xformers import XFormersBackend # noqa: F401
21 + from vllm.attention.backends.xformers import ( # noqa: F401
22 + XFormersBackend)
23 return XFormersBackend
24
25
vllm/config.py
+7 -6
@@ -1,15 +1,15 @@
1 -from typing import TYPE_CHECKING, Optional, Union, ClassVar
2 -from dataclasses import dataclass
1 +import json
2 import os
4 -from packaging.version import Version
3 +from dataclasses import dataclass
4 +from typing import TYPE_CHECKING, ClassVar, Optional, Union
5
6 -import json
6 import torch
7 +from packaging.version import Version
8 from transformers import PretrainedConfig
9
10 from vllm.logger import init_logger
11 from vllm.transformers_utils.config import get_config
12 -from vllm.utils import get_cpu_memory, is_hip, is_neuron, get_nvcc_cuda_version
12 +from vllm.utils import get_cpu_memory, get_nvcc_cuda_version, is_hip, is_neuron
13
14 if TYPE_CHECKING:
15 from ray.util.placement_group import PlacementGroup
@@ -103,7 +103,8 @@ class ModelConfig:
103 if os.environ.get("VLLM_USE_MODELSCOPE", "False").lower() == "true":
104 # download model from ModelScope hub,
105 # lazy import so that modelscope is not required for normal use.
106 - from modelscope.hub.snapshot_download import snapshot_download # pylint: disable=C
106 + # pylint: disable=C.
107 + from modelscope.hub.snapshot_download import snapshot_download
108
109 if not os.path.exists(model):
110 model_path = snapshot_download(model_id=model,
vllm/core/block_manager.py
+3 -3
@@ -1,15 +1,15 @@
1 """A block manager that manages token blocks."""
2 import enum
3 +from abc import ABC, abstractmethod
4 from itertools import count, takewhile
5 from os.path import commonprefix
6 from typing import Dict, List, Optional, Set, Tuple
6 -from abc import ABC, abstractmethod
7
8 from vllm.block import BlockTable, PhysicalTokenBlock
9 +from vllm.core.evictor import EvictionPolicy, Evictor, make_evictor
10 +from vllm.logger import init_logger
11 from vllm.sequence import Sequence, SequenceGroup, SequenceStatus
12 from vllm.utils import Device
11 -from vllm.core.evictor import Evictor, EvictionPolicy, make_evictor
12 -from vllm.logger import init_logger
13
14 logger = init_logger(__name__)
15
vllm/core/evictor.py
+1 -1
@@ -1,6 +1,6 @@
1 import enum
2 -from typing import OrderedDict
2 from abc import ABC, abstractmethod, abstractproperty
3 +from typing import OrderedDict
4
5 from vllm.block import PhysicalTokenBlock
6
vllm/core/scheduler.py
+3 -3
@@ -1,13 +1,13 @@
1 -from collections import deque
1 import enum
2 import time
4 -from typing import Deque, Dict, Iterable, List, Optional, Tuple, Union, Set
3 +from collections import deque
4 +from typing import Deque, Dict, Iterable, List, Optional, Set, Tuple, Union
5
6 from vllm.config import CacheConfig, LoRAConfig, SchedulerConfig
7 from vllm.core.block_manager import AllocStatus, BlockSpaceManager
8 from vllm.core.policy import PolicyFactory
9 -from vllm.lora.request import LoRARequest
9 from vllm.logger import init_logger
10 +from vllm.lora.request import LoRARequest
11 from vllm.sequence import (Sequence, SequenceData, SequenceGroup,
12 SequenceGroupMetadata, SequenceStatus)
13
vllm/engine/arg_utils.py
+2 -3
@@ -3,9 +3,8 @@ import dataclasses
3 from dataclasses import dataclass
4 from typing import Optional, Tuple
5
6 -from vllm.config import (CacheConfig, DeviceConfig, ModelConfig,
7 - ParallelConfig, SchedulerConfig, LoRAConfig,
8 - TokenizerPoolConfig)
6 +from vllm.config import (CacheConfig, DeviceConfig, LoRAConfig, ModelConfig,
7 + ParallelConfig, SchedulerConfig, TokenizerPoolConfig)
8
9
10 @dataclass
vllm/engine/async_llm_engine.py
+3 -3
@@ -2,17 +2,17 @@ import asyncio
2 import os
3 import time
4 from functools import partial
5 -from typing import (Callable, Dict, Iterable, List, Optional, Set, Tuple, Type,
6 - Union, AsyncIterator)
5 +from typing import (AsyncIterator, Callable, Dict, Iterable, List, Optional,
6 + Set, Tuple, Type, Union)
7
8 from transformers import PreTrainedTokenizer
9
10 -from vllm.lora.request import LoRARequest
10 from vllm.config import ModelConfig
11 from vllm.engine.arg_utils import AsyncEngineArgs
12 from vllm.engine.llm_engine import LLMEngine
13 from vllm.engine.ray_utils import initialize_ray_cluster, ray
14 from vllm.logger import init_logger
15 +from vllm.lora.request import LoRARequest
16 from vllm.outputs import RequestOutput
17 from vllm.sampling_params import SamplingParams
18
vllm/engine/llm_engine.py
+5 -5
@@ -4,22 +4,22 @@ from typing import Iterable, List, Optional, Tuple, Type, Union
4 from transformers import PreTrainedTokenizer
5
6 import vllm
7 -from vllm.lora.request import LoRARequest
8 -from vllm.config import (CacheConfig, DeviceConfig, ModelConfig,
9 - ParallelConfig, SchedulerConfig, LoRAConfig)
7 +from vllm.config import (CacheConfig, DeviceConfig, LoRAConfig, ModelConfig,
8 + ParallelConfig, SchedulerConfig)
9 from vllm.core.scheduler import Scheduler, SchedulerOutputs
10 from vllm.engine.arg_utils import EngineArgs
12 -from vllm.executor.executor_base import ExecutorBase
11 from vllm.engine.metrics import StatLogger, Stats
12 from vllm.engine.ray_utils import initialize_ray_cluster
13 +from vllm.executor.executor_base import ExecutorBase
14 from vllm.logger import init_logger
15 +from vllm.lora.request import LoRARequest
16 from vllm.outputs import RequestOutput
17 from vllm.sampling_params import SamplingParams
18 from vllm.sequence import (SamplerOutput, Sequence, SequenceGroup,
19 SequenceGroupOutput, SequenceOutput, SequenceStatus)
20 +from vllm.transformers_utils.detokenizer import Detokenizer
21 from vllm.transformers_utils.tokenizer_group import (BaseTokenizerGroup,
22 get_tokenizer_group)
22 -from vllm.transformers_utils.detokenizer import Detokenizer
23 from vllm.utils import Counter
24
25 logger = init_logger(__name__)
vllm/engine/metrics.py
+7 -6
@@ -1,11 +1,12 @@
1 -from vllm.logger import init_logger
2 -from prometheus_client import (Counter, Gauge, Histogram, Info, REGISTRY,
3 - disable_created_metrics)
4 -
1 import time
6 -import numpy as np
7 -from typing import Dict, List
2 from dataclasses import dataclass
3 +from typing import Dict, List
4 +
5 +import numpy as np
6 +from prometheus_client import (REGISTRY, Counter, Gauge, Histogram, Info,
7 + disable_created_metrics)
8 +
9 +from vllm.logger import init_logger
10
11 logger = init_logger(__name__)
12
vllm/engine/ray_utils.py
+2 -3
@@ -1,10 +1,9 @@
1 import pickle
2 -
3 -from typing import Optional, List, Tuple
2 +from typing import List, Optional, Tuple
3
4 from vllm.config import ParallelConfig
5 from vllm.logger import init_logger
7 -from vllm.utils import is_hip, set_cuda_visible_devices, get_ip
6 +from vllm.utils import get_ip, is_hip, set_cuda_visible_devices
7
8 logger = init_logger(__name__)
9
vllm/entrypoints/api_server.py
+1 -1
@@ -11,9 +11,9 @@ import json
11 import ssl
12 from typing import AsyncGenerator
13
14 +import uvicorn
15 from fastapi import FastAPI, Request
16 from fastapi.responses import JSONResponse, Response, StreamingResponse
16 -import uvicorn
17
18 from vllm.engine.arg_utils import AsyncEngineArgs
19 from vllm.engine.async_llm_engine import AsyncLLMEngine
vllm/entrypoints/llm.py
+1 -1
@@ -3,9 +3,9 @@ from typing import List, Optional, Union
3 from tqdm import tqdm
4 from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast
5
6 -from vllm.lora.request import LoRARequest
6 from vllm.engine.arg_utils import EngineArgs
7 from vllm.engine.llm_engine import LLMEngine
8 +from vllm.lora.request import LoRARequest
9 from vllm.outputs import RequestOutput
10 from vllm.sampling_params import SamplingParams
11 from vllm.utils import Counter
vllm/entrypoints/openai/api_server.py
+8 -9
@@ -1,28 +1,27 @@
1 import asyncio
2 -from contextlib import asynccontextmanager
3 -import os
2 import importlib
3 import inspect
4 +import os
5 +from contextlib import asynccontextmanager
6 +from http import HTTPStatus
7
7 -from prometheus_client import make_asgi_app
8 import fastapi
9 import uvicorn
10 -from http import HTTPStatus
10 from fastapi import Request
11 from fastapi.exceptions import RequestValidationError
12 from fastapi.middleware.cors import CORSMiddleware
14 -from fastapi.responses import JSONResponse, StreamingResponse, Response
13 +from fastapi.responses import JSONResponse, Response, StreamingResponse
14 +from prometheus_client import make_asgi_app
15
16 import vllm
17 from vllm.engine.arg_utils import AsyncEngineArgs
18 from vllm.engine.async_llm_engine import AsyncLLMEngine
19 -from vllm.entrypoints.openai.protocol import (CompletionRequest,
20 - ChatCompletionRequest,
21 - ErrorResponse)
22 -from vllm.logger import init_logger
19 from vllm.entrypoints.openai.cli_args import make_arg_parser
20 +from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
21 + CompletionRequest, ErrorResponse)
22 from vllm.entrypoints.openai.serving_chat import OpenAIServingChat
23 from vllm.entrypoints.openai.serving_completion import OpenAIServingCompletion
24 +from vllm.logger import init_logger
25
26 TIMEOUT_KEEP_ALIVE = 5 # seconds
27
vllm/entrypoints/openai/protocol.py
+2 -3
@@ -3,12 +3,11 @@
3 import time
4 from typing import Dict, List, Literal, Optional, Union
5
6 +import torch
7 from pydantic import BaseModel, Field, model_validator
8
8 -from vllm.utils import random_uuid
9 from vllm.sampling_params import SamplingParams
10 -
11 -import torch
10 +from vllm.utils import random_uuid
11
12
13 class ErrorResponse(BaseModel):
vllm/entrypoints/openai/serving_chat.py
+8 -6
@@ -1,19 +1,21 @@
1 -import time
1 import codecs
2 +import time
3 +from typing import AsyncGenerator, AsyncIterator, List, Optional, Union
4 +
5 from fastapi import Request
4 -from typing import AsyncGenerator, AsyncIterator, Optional, List, Union
5 -from vllm.logger import init_logger
6 -from vllm.utils import random_uuid
6 +
7 from vllm.engine.async_llm_engine import AsyncLLMEngine
8 from vllm.entrypoints.openai.protocol import (
9 ChatCompletionRequest, ChatCompletionResponse,
10 ChatCompletionResponseChoice, ChatCompletionResponseStreamChoice,
11 ChatCompletionStreamResponse, ChatMessage, DeltaMessage, ErrorResponse,
12 UsageInfo)
13 -from vllm.outputs import RequestOutput
14 -from vllm.entrypoints.openai.serving_engine import OpenAIServing, LoRA
13 +from vllm.entrypoints.openai.serving_engine import LoRA, OpenAIServing
14 +from vllm.logger import init_logger
15 from vllm.model_executor.guided_decoding import (
16 get_guided_decoding_logits_processor)
17 +from vllm.outputs import RequestOutput
18 +from vllm.utils import random_uuid
19
20 logger = init_logger(__name__)
21
vllm/entrypoints/openai/serving_completion.py
+14 -15
@@ -1,24 +1,23 @@
1 import asyncio
2 import time
3 +from typing import (AsyncGenerator, AsyncIterator, Callable, Dict, List,
4 + Optional, Tuple)
5 +
6 from fastapi import Request
4 -from typing import (AsyncGenerator, AsyncIterator, Callable, List, Optional,
5 - Dict, Tuple)
6 -from vllm.logger import init_logger
7 -from vllm.utils import random_uuid
7 +
8 from vllm.engine.async_llm_engine import AsyncLLMEngine
9 -from vllm.entrypoints.openai.protocol import (
10 - CompletionRequest,
11 - CompletionResponse,
12 - CompletionResponseChoice,
13 - CompletionResponseStreamChoice,
14 - CompletionStreamResponse,
15 - LogProbs,
16 - UsageInfo,
17 -)
18 -from vllm.outputs import RequestOutput
19 -from vllm.entrypoints.openai.serving_engine import OpenAIServing, LoRA
9 +from vllm.entrypoints.openai.protocol import (CompletionRequest,
10 + CompletionResponse,
11 + CompletionResponseChoice,
12 + CompletionResponseStreamChoice,
13 + CompletionStreamResponse,
14 + LogProbs, UsageInfo)
15 +from vllm.entrypoints.openai.serving_engine import LoRA, OpenAIServing
16 +from vllm.logger import init_logger
17 from vllm.model_executor.guided_decoding import (
18 get_guided_decoding_logits_processor)
19 +from vllm.outputs import RequestOutput
20 +from vllm.utils import random_uuid
21
22 logger = init_logger(__name__)
23
vllm/entrypoints/openai/serving_engine.py
+6 -6
@@ -3,16 +3,16 @@ import json
3 from dataclasses import dataclass
4 from http import HTTPStatus
5 from typing import Dict, List, Optional, Union
6 -from vllm.logger import init_logger
7 -from vllm.transformers_utils.tokenizer import get_tokenizer
6 +
7 from vllm.engine.async_llm_engine import AsyncLLMEngine
9 -from vllm.entrypoints.openai.protocol import (CompletionRequest,
10 - ChatCompletionRequest,
11 - ErrorResponse, LogProbs,
12 - ModelCard, ModelList,
8 +from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
9 + CompletionRequest, ErrorResponse,
10 + LogProbs, ModelCard, ModelList,
11 ModelPermission)
12 +from vllm.logger import init_logger
13 from vllm.lora.request import LoRARequest
14 from vllm.sequence import Logprob
15 +from vllm.transformers_utils.tokenizer import get_tokenizer
16
17 logger = init_logger(__name__)
18
vllm/executor/executor_base.py
+2 -2
@@ -1,8 +1,8 @@
1 from abc import ABC, abstractmethod
2 from typing import Dict, List, Optional
3
4 -from vllm.config import (CacheConfig, DeviceConfig, ModelConfig,
5 - ParallelConfig, SchedulerConfig, LoRAConfig)
4 +from vllm.config import (CacheConfig, DeviceConfig, LoRAConfig, ModelConfig,
5 + ParallelConfig, SchedulerConfig)
6 from vllm.lora.request import LoRARequest
7 from vllm.sequence import SamplerOutput, SequenceGroupMetadata
8
vllm/executor/gpu_executor.py
+4 -4
@@ -1,13 +1,13 @@
1 from typing import Dict, List, Optional
2
3 -from vllm.lora.request import LoRARequest
4 -from vllm.config import (CacheConfig, DeviceConfig, ModelConfig,
5 - ParallelConfig, SchedulerConfig, LoRAConfig)
3 +from vllm.config import (CacheConfig, DeviceConfig, LoRAConfig, ModelConfig,
4 + ParallelConfig, SchedulerConfig)
5 from vllm.executor.executor_base import ExecutorAsyncBase, ExecutorBase
6 from vllm.executor.utils import check_block_size_valid
7 from vllm.logger import init_logger
8 +from vllm.lora.request import LoRARequest
9 from vllm.sequence import SamplerOutput, SequenceGroupMetadata
10 -from vllm.utils import (get_ip, get_open_port, get_distributed_init_method,
10 +from vllm.utils import (get_distributed_init_method, get_ip, get_open_port,
11 make_async)
12
13 logger = init_logger(__name__)
vllm/executor/neuron_executor.py
+3 -3
@@ -1,10 +1,10 @@
1 from typing import Dict, List, Optional
2
3 -from vllm.lora.request import LoRARequest
4 -from vllm.config import (CacheConfig, DeviceConfig, ModelConfig,
5 - ParallelConfig, SchedulerConfig, LoRAConfig)
3 +from vllm.config import (CacheConfig, DeviceConfig, LoRAConfig, ModelConfig,
4 + ParallelConfig, SchedulerConfig)
5 from vllm.executor.executor_base import ExecutorBase
6 from vllm.logger import init_logger
7 +from vllm.lora.request import LoRARequest
8 from vllm.sequence import SamplerOutput, SequenceGroupMetadata
9
10 logger = init_logger(__name__)
vllm/executor/ray_gpu_executor.py
+6 -6
@@ -1,20 +1,20 @@
1 import asyncio
2 import copy
3 -from collections import defaultdict
3 import os
4 import pickle
5 +from collections import defaultdict
6 from typing import TYPE_CHECKING, Any, Dict, List, Optional
7
8 -from vllm.config import (CacheConfig, DeviceConfig, ModelConfig,
9 - ParallelConfig, SchedulerConfig, LoRAConfig)
8 +from vllm.config import (CacheConfig, DeviceConfig, LoRAConfig, ModelConfig,
9 + ParallelConfig, SchedulerConfig)
10 from vllm.engine.ray_utils import RayWorkerVllm, ray
11 from vllm.executor.executor_base import ExecutorAsyncBase, ExecutorBase
12 from vllm.executor.utils import check_block_size_valid
13 from vllm.logger import init_logger
14 from vllm.lora.request import LoRARequest
15 from vllm.sequence import SamplerOutput, SequenceGroupMetadata
16 -from vllm.utils import (set_cuda_visible_devices, get_ip, get_open_port,
17 - get_distributed_init_method, make_async)
16 +from vllm.utils import (get_distributed_init_method, get_ip, get_open_port,
17 + make_async, set_cuda_visible_devices)
18
19 if ray is not None:
20 from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy
@@ -343,7 +343,7 @@ class RayGPUExecutor(ExecutorBase):
343 raise ValueError(f"Ray version {required_version} or greater is "
344 f"required, but found {current_version}")
345
346 - from ray.dag import MultiOutputNode, InputNode
346 + from ray.dag import InputNode, MultiOutputNode
347 assert self.parallel_config.worker_use_ray
348
349 # Right now, compiled DAG requires at least 1 arg. We send
vllm/logger.py
+1 -1
@@ -2,8 +2,8 @@
2 # https://github.com/skypilot-org/skypilot/blob/86dc0f6283a335e4aa37b3c10716f90999f48ab6/sky/sky_logging.py
3 """Logging configuration for vLLM."""
4 import logging
5 -import sys
5 import os
6 +import sys
7
8 VLLM_CONFIGURE_LOGGING = int(os.getenv("VLLM_CONFIGURE_LOGGING", "1"))
9
vllm/lora/layers.py
+6 -8
@@ -10,18 +10,16 @@ from transformers import PretrainedConfig
10
11 from vllm.config import LoRAConfig
12 from vllm.lora.punica import add_lora, add_lora_slice, bgmv
13 -from vllm.model_executor.parallel_utils.communication_op import (
14 - tensor_model_parallel_all_gather,
15 - tensor_model_parallel_all_reduce,
16 - tensor_model_parallel_gather,
17 -)
13 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
19 - RowParallelLinear,
14 + MergedColumnParallelLinear,
15 QKVParallelLinear,
21 - MergedColumnParallelLinear)
16 + RowParallelLinear)
17 from vllm.model_executor.layers.logits_processor import LogitsProcessor
18 from vllm.model_executor.layers.vocab_parallel_embedding import (
24 - VocabParallelEmbedding, ParallelLMHead)
19 + ParallelLMHead, VocabParallelEmbedding)
20 +from vllm.model_executor.parallel_utils.communication_op import (
21 + tensor_model_parallel_all_gather, tensor_model_parallel_all_reduce,
22 + tensor_model_parallel_gather)
23 from vllm.model_executor.parallel_utils.parallel_state import (
24 get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
25 from vllm.model_executor.parallel_utils.utils import (
vllm/lora/lora.py
+1
@@ -1,6 +1,7 @@
1 from typing import List, Optional
2
3 import torch
4 +
5 from vllm.utils import is_pin_memory_available
6
7
vllm/lora/models.py
+2 -3
@@ -4,19 +4,18 @@ import logging
4 import math
5 import os
6 import re
7 -from typing import (Callable, Dict, Hashable, List, Optional, Tuple, Type)
7 +from typing import Callable, Dict, Hashable, List, Optional, Tuple, Type
8
9 import safetensors.torch
10 import torch
11 from torch import nn
12
13 from vllm.config import LoRAConfig
14 -from vllm.utils import LRUCache, is_pin_memory_available
15 -
14 from vllm.lora.layers import (BaseLayerWithLoRA, LoRAMapping, from_layer,
15 from_layer_logits_processor)
16 from vllm.lora.lora import LoRALayerWeights, PackedLoRALayerWeights
17 from vllm.lora.utils import parse_fine_tuned_lora_name, replace_submodule
18 +from vllm.utils import LRUCache, is_pin_memory_available
19
20 logger = logging.getLogger(__name__)
21
vllm/lora/worker_manager.py
+2 -2
@@ -4,11 +4,11 @@ from typing import Any, Dict, List, Optional, Set, Type
4
5 import torch
6
7 +from vllm.config import LoRAConfig
8 +from vllm.lora.layers import LoRAMapping
9 from vllm.lora.models import (LoRAModel, LoRAModelManager,
10 LRUCacheLoRAModelManager, create_lora_manager)
11 from vllm.lora.request import LoRARequest
10 -from vllm.lora.layers import LoRAMapping
11 -from vllm.config import LoRAConfig
12
13 logger = logging.getLogger(__name__)
14
vllm/model_executor/guided_decoding.py
+6 -6
@@ -5,16 +5,16 @@ from enum import Enum
5 from functools import lru_cache
6 from json import dumps as json_dumps
7 from re import escape as regex_escape
8 -from typing import Union, Tuple
8 +from typing import Tuple, Union
9
10 from pydantic import BaseModel
11 from transformers import PreTrainedTokenizerBase
12
13 -from vllm.entrypoints.openai.protocol import (CompletionRequest,
14 - ChatCompletionRequest)
15 -from vllm.model_executor.guided_logits_processors import (JSONLogitsProcessor,
16 - RegexLogitsProcessor,
17 - CFGLogitsProcessor)
13 +from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
14 + CompletionRequest)
15 +from vllm.model_executor.guided_logits_processors import (CFGLogitsProcessor,
16 + JSONLogitsProcessor,
17 + RegexLogitsProcessor)
18
19
20 class GuidedDecodingMode(Enum):
vllm/model_executor/guided_logits_processors.py
+3 -3
@@ -16,13 +16,13 @@
16 import json
17 import math
18 from collections import defaultdict
19 -from typing import Union, DefaultDict, Dict, List, Optional, Callable
19 +from typing import Callable, DefaultDict, Dict, List, Optional, Union
20
21 import torch
22 +from outlines.fsm.fsm import CFGFSM, RegexFSM
23 +from outlines.fsm.json_schema import build_regex_from_schema
24 from pydantic import BaseModel
25 from transformers import PreTrainedTokenizerBase
24 -from outlines.fsm.fsm import RegexFSM, CFGFSM
25 -from outlines.fsm.json_schema import build_regex_from_schema
26
27
28 class BaseLogitsProcessor:
vllm/model_executor/layers/fused_moe/__init__.py
+1 -3
@@ -1,7 +1,5 @@
1 from vllm.model_executor.layers.fused_moe.fused_moe import (
2 - fused_moe,
3 - get_config_file_name,
4 -)
2 + fused_moe, get_config_file_name)
3
4 __all__ = [
5 "fused_moe",
vllm/model_executor/layers/linear.py
+3 -3
@@ -5,14 +5,14 @@ import torch
5 import torch.nn.functional as F
6 from torch.nn.parameter import Parameter
7
8 +from vllm.logger import init_logger
9 +from vllm.model_executor.parallel_utils.communication_op import (
10 + tensor_model_parallel_all_gather, tensor_model_parallel_all_reduce)
11 from vllm.model_executor.parallel_utils.parallel_state import (
12 get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
10 -from vllm.model_executor.parallel_utils.communication_op import (
11 - tensor_model_parallel_all_reduce, tensor_model_parallel_all_gather)
13 from vllm.model_executor.parallel_utils.utils import (
14 divide, split_tensor_along_last_dim)
15 from vllm.model_executor.utils import set_weight_attrs
15 -from vllm.logger import init_logger
16
17 logger = init_logger(__name__)
18
vllm/model_executor/layers/ops/rand.py
+2 -2
@@ -1,9 +1,9 @@
1 +from typing import Optional, Union
2 +
3 import torch
4 import triton
5 import triton.language as tl
6
5 -from typing import Optional, Union
6 -
7
8 def seeded_uniform(
9 *size,
vllm/model_executor/layers/ops/sample.py
+1 -1
@@ -1,5 +1,5 @@
1 import math
2 -from typing import Tuple, Optional
2 +from typing import Optional, Tuple
3
4 import torch
5 import triton
vllm/model_executor/layers/quantization/__init__.py
+2 -2
@@ -1,11 +1,11 @@
1 from typing import Type
2
3 +from vllm.model_executor.layers.quantization.awq import AWQConfig
4 from vllm.model_executor.layers.quantization.base_config import (
5 QuantizationConfig)
5 -from vllm.model_executor.layers.quantization.awq import AWQConfig
6 from vllm.model_executor.layers.quantization.gptq import GPTQConfig
7 -from vllm.model_executor.layers.quantization.squeezellm import SqueezeLLMConfig
7 from vllm.model_executor.layers.quantization.marlin import MarlinConfig
8 +from vllm.model_executor.layers.quantization.squeezellm import SqueezeLLMConfig
9
10 _QUANTIZATION_CONFIG_REGISTRY = {
11 "awq": AWQConfig,
vllm/model_executor/layers/quantization/gptq.py
+1 -1
@@ -1,7 +1,7 @@
1 import enum
2 from enum import Enum
3 -from typing import Any, Dict, List, Optional
3 from fractions import Fraction
4 +from typing import Any, Dict, List, Optional
5
6 import torch
7 from torch.nn.parameter import Parameter
vllm/model_executor/layers/quantization/marlin.py
+2 -1
@@ -4,7 +4,8 @@ import torch
4 from torch.nn.parameter import Parameter
5
6 from vllm._C import ops
7 -from vllm.model_executor.layers.linear import LinearMethodBase, set_weight_attrs
7 +from vllm.model_executor.layers.linear import (LinearMethodBase,
8 + set_weight_attrs)
9 from vllm.model_executor.layers.quantization.base_config import (
10 QuantizationConfig)
11
vllm/model_executor/layers/rejection_sampler.py
+2 -2
@@ -1,9 +1,9 @@
1 -from typing import Tuple, Optional
1 from functools import cached_property
2 +from typing import Optional, Tuple
3
4 import torch
5 -import torch.nn as nn
5 import torch.jit
6 +import torch.nn as nn
7
8
9 class RejectionSampler(nn.Module):
vllm/model_executor/layers/vocab_parallel_embedding.py
+3 -5
@@ -4,13 +4,11 @@ import torch
4 import torch.nn.functional as F
5 from torch.nn.parameter import Parameter
6
7 -from vllm.model_executor.parallel_utils.parallel_state import (
8 - get_tensor_model_parallel_rank,
9 - get_tensor_model_parallel_world_size,
10 -)
11 -from vllm.model_executor.parallel_utils.utils import divide
7 from vllm.model_executor.parallel_utils.communication_op import (
8 tensor_model_parallel_all_reduce)
9 +from vllm.model_executor.parallel_utils.parallel_state import (
10 + get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
11 +from vllm.model_executor.parallel_utils.utils import divide
12 from vllm.model_executor.utils import set_weight_attrs
13
14 DEFAULT_VOCAB_PADDING_SIZE = 64
vllm/model_executor/models/baichuan.py
+2 -2
@@ -32,11 +32,11 @@ from vllm.model_executor.layers.linear import (LinearMethodBase,
32 MergedColumnParallelLinear,
33 QKVParallelLinear,
34 RowParallelLinear)
35 -from vllm.model_executor.layers.rotary_embedding import get_rope
35 from vllm.model_executor.layers.logits_processor import LogitsProcessor
36 +from vllm.model_executor.layers.rotary_embedding import get_rope
37 from vllm.model_executor.layers.sampler import Sampler
38 from vllm.model_executor.layers.vocab_parallel_embedding import (
39 - VocabParallelEmbedding, ParallelLMHead)
39 + ParallelLMHead, VocabParallelEmbedding)
40 from vllm.model_executor.parallel_utils.parallel_state import (
41 get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
42 from vllm.model_executor.sampling_metadata import SamplingMetadata
vllm/model_executor/models/chatglm.py
+2 -2
@@ -15,11 +15,11 @@ from vllm.model_executor.layers.linear import (LinearMethodBase,
15 MergedColumnParallelLinear,
16 QKVParallelLinear,
17 RowParallelLinear)
18 -from vllm.model_executor.layers.rotary_embedding import get_rope
18 from vllm.model_executor.layers.logits_processor import LogitsProcessor
19 +from vllm.model_executor.layers.rotary_embedding import get_rope
20 from vllm.model_executor.layers.sampler import Sampler
21 from vllm.model_executor.layers.vocab_parallel_embedding import (
22 - VocabParallelEmbedding, ParallelLMHead)
22 + ParallelLMHead, VocabParallelEmbedding)
23 from vllm.model_executor.parallel_utils.parallel_state import (
24 get_tensor_model_parallel_world_size)
25 from vllm.model_executor.sampling_metadata import SamplingMetadata
vllm/model_executor/models/deepseek.py
+3 -3
@@ -33,14 +33,14 @@ from vllm.model_executor.layers.fused_moe import fused_moe
33 from vllm.model_executor.layers.layernorm import RMSNorm
34 from vllm.model_executor.layers.linear import (LinearMethodBase,
35 MergedColumnParallelLinear,
36 - ReplicatedLinear,
36 QKVParallelLinear,
37 + ReplicatedLinear,
38 RowParallelLinear)
39 -from vllm.model_executor.layers.rotary_embedding import get_rope
39 from vllm.model_executor.layers.logits_processor import LogitsProcessor
40 +from vllm.model_executor.layers.rotary_embedding import get_rope
41 from vllm.model_executor.layers.sampler import Sampler
42 from vllm.model_executor.layers.vocab_parallel_embedding import (
43 - VocabParallelEmbedding, ParallelLMHead)
43 + ParallelLMHead, VocabParallelEmbedding)
44 from vllm.model_executor.parallel_utils.communication_op import (
45 tensor_model_parallel_all_reduce)
46 from vllm.model_executor.parallel_utils.parallel_state import (
vllm/model_executor/models/falcon.py
+1 -1
@@ -32,8 +32,8 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
32 LinearMethodBase,
33 QKVParallelLinear,
34 RowParallelLinear)
35 -from vllm.model_executor.layers.rotary_embedding import get_rope
35 from vllm.model_executor.layers.logits_processor import LogitsProcessor
36 +from vllm.model_executor.layers.rotary_embedding import get_rope
37 from vllm.model_executor.layers.sampler import Sampler
38 from vllm.model_executor.layers.vocab_parallel_embedding import (
39 VocabParallelEmbedding)
vllm/model_executor/models/gemma.py
+1 -1
@@ -28,8 +28,8 @@ from vllm.model_executor.layers.linear import (LinearMethodBase,
28 MergedColumnParallelLinear,
29 QKVParallelLinear,
30 RowParallelLinear)
31 -from vllm.model_executor.layers.rotary_embedding import get_rope
31 from vllm.model_executor.layers.logits_processor import LogitsProcessor
32 +from vllm.model_executor.layers.rotary_embedding import get_rope
33 from vllm.model_executor.layers.sampler import Sampler
34 from vllm.model_executor.layers.vocab_parallel_embedding import (
35 VocabParallelEmbedding)
vllm/model_executor/models/gpt_j.py
+2 -2
@@ -28,11 +28,11 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
28 LinearMethodBase,
29 QKVParallelLinear,
30 RowParallelLinear)
31 -from vllm.model_executor.layers.rotary_embedding import get_rope
31 from vllm.model_executor.layers.logits_processor import LogitsProcessor
32 +from vllm.model_executor.layers.rotary_embedding import get_rope
33 from vllm.model_executor.layers.sampler import Sampler
34 from vllm.model_executor.layers.vocab_parallel_embedding import (
35 - VocabParallelEmbedding, ParallelLMHead)
35 + ParallelLMHead, VocabParallelEmbedding)
36 from vllm.model_executor.parallel_utils.parallel_state import (
37 get_tensor_model_parallel_world_size)
38 from vllm.model_executor.sampling_metadata import SamplingMetadata
vllm/model_executor/models/gpt_neox.py
+2 -2
@@ -28,11 +28,11 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
28 LinearMethodBase,
29 QKVParallelLinear,
30 RowParallelLinear)
31 -from vllm.model_executor.layers.rotary_embedding import get_rope
31 from vllm.model_executor.layers.logits_processor import LogitsProcessor
32 +from vllm.model_executor.layers.rotary_embedding import get_rope
33 from vllm.model_executor.layers.sampler import Sampler
34 from vllm.model_executor.layers.vocab_parallel_embedding import (
35 - VocabParallelEmbedding, ParallelLMHead)
35 + ParallelLMHead, VocabParallelEmbedding)
36 from vllm.model_executor.parallel_utils.parallel_state import (
37 get_tensor_model_parallel_world_size)
38 from vllm.model_executor.sampling_metadata import SamplingMetadata
vllm/model_executor/models/internlm2.py
+2 -2
@@ -12,11 +12,11 @@ from vllm.model_executor.layers.linear import (LinearMethodBase,
12 MergedColumnParallelLinear,
13 QKVParallelLinear,
14 RowParallelLinear)
15 -from vllm.model_executor.layers.rotary_embedding import get_rope
15 from vllm.model_executor.layers.logits_processor import LogitsProcessor
16 +from vllm.model_executor.layers.rotary_embedding import get_rope
17 from vllm.model_executor.layers.sampler import Sampler
18 from vllm.model_executor.layers.vocab_parallel_embedding import (
19 - VocabParallelEmbedding, ParallelLMHead)
19 + ParallelLMHead, VocabParallelEmbedding)
20 from vllm.model_executor.parallel_utils.parallel_state import (
21 get_tensor_model_parallel_world_size)
22 from vllm.model_executor.sampling_metadata import SamplingMetadata
vllm/model_executor/models/jais.py
+10 -16
@@ -24,29 +24,23 @@ from typing import List, Optional
24
25 import torch
26 from torch import nn
27 -from vllm.transformers_utils.configs import JAISConfig
27
28 from vllm.attention import Attention, AttentionMetadata
30 -from vllm.model_executor.layers.linear import (
31 - ColumnParallelLinear,
32 - LinearMethodBase,
33 - QKVParallelLinear,
34 - RowParallelLinear,
35 -)
29 +from vllm.model_executor.layers.linear import (ColumnParallelLinear,
30 + LinearMethodBase,
31 + QKVParallelLinear,
32 + RowParallelLinear)
33 from vllm.model_executor.layers.logits_processor import LogitsProcessor
34 from vllm.model_executor.layers.sampler import Sampler
35 from vllm.model_executor.layers.vocab_parallel_embedding import (
39 - VocabParallelEmbedding, )
36 + VocabParallelEmbedding)
37 from vllm.model_executor.parallel_utils.parallel_state import (
41 - get_tensor_model_parallel_world_size,
42 - get_tensor_model_parallel_rank,
43 -)
44 -from vllm.model_executor.weight_utils import (
45 - default_weight_loader,
46 - hf_model_weights_iterator,
47 -)
48 -from vllm.sequence import SamplerOutput
38 + get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
39 from vllm.model_executor.sampling_metadata import SamplingMetadata
40 +from vllm.model_executor.weight_utils import (default_weight_loader,
41 + hf_model_weights_iterator)
42 +from vllm.sequence import SamplerOutput
43 +from vllm.transformers_utils.configs import JAISConfig
44
45
46 class SwiGLUActivation(nn.Module):
vllm/model_executor/models/llama.py
+2 -2
@@ -35,11 +35,11 @@ from vllm.model_executor.layers.linear import (LinearMethodBase,
35 MergedColumnParallelLinear,
36 QKVParallelLinear,
37 RowParallelLinear)
38 -from vllm.model_executor.layers.rotary_embedding import get_rope
38 from vllm.model_executor.layers.logits_processor import LogitsProcessor
39 +from vllm.model_executor.layers.rotary_embedding import get_rope
40 from vllm.model_executor.layers.sampler import Sampler
41 from vllm.model_executor.layers.vocab_parallel_embedding import (
42 - VocabParallelEmbedding, ParallelLMHead, DEFAULT_VOCAB_PADDING_SIZE)
42 + DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
43 from vllm.model_executor.parallel_utils.parallel_state import (
44 get_tensor_model_parallel_world_size)
45 from vllm.model_executor.sampling_metadata import SamplingMetadata
vllm/model_executor/models/mixtral.py
+2 -2
@@ -35,11 +35,11 @@ from vllm.model_executor.layers.linear import (LinearMethodBase,
35 QKVParallelLinear,
36 ReplicatedLinear,
37 RowParallelLinear)
38 -from vllm.model_executor.layers.rotary_embedding import get_rope
38 from vllm.model_executor.layers.logits_processor import LogitsProcessor
39 +from vllm.model_executor.layers.rotary_embedding import get_rope
40 from vllm.model_executor.layers.sampler import Sampler
41 from vllm.model_executor.layers.vocab_parallel_embedding import (
42 - VocabParallelEmbedding, ParallelLMHead, DEFAULT_VOCAB_PADDING_SIZE)
42 + DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
43 from vllm.model_executor.parallel_utils.communication_op import (
44 tensor_model_parallel_all_reduce)
45 from vllm.model_executor.parallel_utils.parallel_state import (
vllm/model_executor/models/mixtral_quant.py
+3 -5
@@ -24,24 +24,22 @@
24 from typing import List, Optional
25
26 import numpy as np
27 -
27 import torch
28 import torch.nn.functional as F
30 -
29 from torch import nn
30 from transformers import MixtralConfig
31
32 from vllm.attention import Attention, AttentionMetadata
33 from vllm.model_executor.layers.layernorm import RMSNorm
34 from vllm.model_executor.layers.linear import (LinearMethodBase,
37 - ReplicatedLinear,
35 QKVParallelLinear,
36 + ReplicatedLinear,
37 RowParallelLinear)
40 -from vllm.model_executor.layers.rotary_embedding import get_rope
38 from vllm.model_executor.layers.logits_processor import LogitsProcessor
39 +from vllm.model_executor.layers.rotary_embedding import get_rope
40 from vllm.model_executor.layers.sampler import Sampler
41 from vllm.model_executor.layers.vocab_parallel_embedding import (
44 - VocabParallelEmbedding, ParallelLMHead)
42 + ParallelLMHead, VocabParallelEmbedding)
43 from vllm.model_executor.parallel_utils.communication_op import (
44 tensor_model_parallel_all_reduce)
45 from vllm.model_executor.parallel_utils.parallel_state import (
vllm/model_executor/models/olmo.py
+10 -15
@@ -40,32 +40,27 @@ from typing import List, Optional, Tuple
40
41 import torch
42 import torch.nn.functional as F
43 +# this model must need this dependency
44 +from hf_olmo import OLMoConfig
45 from torch import nn
46
47 from vllm.attention import Attention, AttentionMetadata
46 -from vllm.model_executor.layers.linear import (
47 - ColumnParallelLinear,
48 - LinearMethodBase,
49 - QKVParallelLinear,
50 - RowParallelLinear,
51 -)
52 -from vllm.model_executor.layers.rotary_embedding import get_rope
48 +from vllm.model_executor.layers.linear import (ColumnParallelLinear,
49 + LinearMethodBase,
50 + QKVParallelLinear,
51 + RowParallelLinear)
52 from vllm.model_executor.layers.logits_processor import LogitsProcessor
53 +from vllm.model_executor.layers.rotary_embedding import get_rope
54 from vllm.model_executor.layers.sampler import Sampler
55 from vllm.model_executor.layers.vocab_parallel_embedding import (
56 VocabParallelEmbedding)
57 from vllm.model_executor.parallel_utils.parallel_state import (
58 - get_tensor_model_parallel_world_size, )
58 + get_tensor_model_parallel_world_size)
59 from vllm.model_executor.sampling_metadata import SamplingMetadata
60 -from vllm.model_executor.weight_utils import (
61 - default_weight_loader,
62 - hf_model_weights_iterator,
63 -)
60 +from vllm.model_executor.weight_utils import (default_weight_loader,
61 + hf_model_weights_iterator)
62 from vllm.sequence import SamplerOutput
63
66 -# this model must need this dependency
67 -from hf_olmo import OLMoConfig
68 -
64
65 class SwiGLU(nn.Module):
66
vllm/model_executor/models/orion.py
+2 -2
@@ -16,11 +16,11 @@ from vllm.model_executor.layers.linear import (LinearMethodBase,
16 MergedColumnParallelLinear,
17 QKVParallelLinear,
18 RowParallelLinear)
19 -from vllm.model_executor.layers.rotary_embedding import get_rope
19 from vllm.model_executor.layers.logits_processor import LogitsProcessor
20 +from vllm.model_executor.layers.rotary_embedding import get_rope
21 from vllm.model_executor.layers.sampler import Sampler
22 from vllm.model_executor.layers.vocab_parallel_embedding import (
23 - VocabParallelEmbedding, ParallelLMHead)
23 + ParallelLMHead, VocabParallelEmbedding)
24 from vllm.model_executor.parallel_utils.parallel_state import (
25 get_tensor_model_parallel_world_size)
26 from vllm.model_executor.sampling_metadata import SamplingMetadata
vllm/model_executor/models/phi.py
+2 -2
@@ -47,11 +47,11 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
47 LinearMethodBase,
48 QKVParallelLinear,
49 RowParallelLinear)
50 -from vllm.model_executor.layers.rotary_embedding import get_rope
50 from vllm.model_executor.layers.logits_processor import LogitsProcessor
51 +from vllm.model_executor.layers.rotary_embedding import get_rope
52 from vllm.model_executor.layers.sampler import Sampler
53 from vllm.model_executor.layers.vocab_parallel_embedding import (
54 - VocabParallelEmbedding, ParallelLMHead)
54 + ParallelLMHead, VocabParallelEmbedding)
55 from vllm.model_executor.parallel_utils.parallel_state import (
56 get_tensor_model_parallel_world_size)
57 from vllm.model_executor.sampling_metadata import SamplingMetadata
vllm/model_executor/models/qwen.py
+2 -2
@@ -17,11 +17,11 @@ from vllm.model_executor.layers.linear import (LinearMethodBase,
17 MergedColumnParallelLinear,
18 QKVParallelLinear,
19 RowParallelLinear)
20 -from vllm.model_executor.layers.rotary_embedding import get_rope
20 from vllm.model_executor.layers.logits_processor import LogitsProcessor
21 +from vllm.model_executor.layers.rotary_embedding import get_rope
22 from vllm.model_executor.layers.sampler import Sampler
23 from vllm.model_executor.layers.vocab_parallel_embedding import (
24 - VocabParallelEmbedding, ParallelLMHead)
24 + ParallelLMHead, VocabParallelEmbedding)
25 from vllm.model_executor.parallel_utils.parallel_state import (
26 get_tensor_model_parallel_world_size)
27 from vllm.model_executor.sampling_metadata import SamplingMetadata
vllm/model_executor/models/qwen2.py
+3 -3
@@ -29,24 +29,24 @@ from torch import nn
29 from transformers import Qwen2Config
30
31 from vllm.attention import Attention, AttentionMetadata
32 +from vllm.config import LoRAConfig
33 from vllm.model_executor.layers.activation import SiluAndMul
34 from vllm.model_executor.layers.layernorm import RMSNorm
35 from vllm.model_executor.layers.linear import (LinearMethodBase,
36 MergedColumnParallelLinear,
37 QKVParallelLinear,
38 RowParallelLinear)
38 -from vllm.model_executor.layers.rotary_embedding import get_rope
39 from vllm.model_executor.layers.logits_processor import LogitsProcessor
40 +from vllm.model_executor.layers.rotary_embedding import get_rope
41 from vllm.model_executor.layers.sampler import Sampler
42 from vllm.model_executor.layers.vocab_parallel_embedding import (
42 - VocabParallelEmbedding, ParallelLMHead)
43 + ParallelLMHead, VocabParallelEmbedding)
44 from vllm.model_executor.parallel_utils.parallel_state import (
45 get_tensor_model_parallel_world_size)
46 from vllm.model_executor.sampling_metadata import SamplingMetadata
47 from vllm.model_executor.weight_utils import (default_weight_loader,
48 hf_model_weights_iterator)
49 from vllm.sequence import SamplerOutput
49 -from vllm.config import LoRAConfig
50
51
52 class Qwen2MLP(nn.Module):
vllm/model_executor/models/stablelm.py
+2 -2
@@ -31,11 +31,11 @@ from vllm.model_executor.layers.linear import (LinearMethodBase,
31 MergedColumnParallelLinear,
32 QKVParallelLinear,
33 RowParallelLinear)
34 -from vllm.model_executor.layers.rotary_embedding import get_rope
34 from vllm.model_executor.layers.logits_processor import LogitsProcessor
35 +from vllm.model_executor.layers.rotary_embedding import get_rope
36 from vllm.model_executor.layers.sampler import Sampler
37 from vllm.model_executor.layers.vocab_parallel_embedding import (
38 - VocabParallelEmbedding, ParallelLMHead)
38 + ParallelLMHead, VocabParallelEmbedding)
39 from vllm.model_executor.parallel_utils.parallel_state import (
40 get_tensor_model_parallel_world_size)
41 from vllm.model_executor.sampling_metadata import SamplingMetadata
vllm/model_executor/models/starcoder2.py
+3 -3
@@ -25,19 +25,19 @@ from torch import nn
25 from transformers import Starcoder2Config
26
27 from vllm.attention import Attention, AttentionMetadata
28 -from vllm.model_executor.sampling_metadata import SamplingMetadata
28 from vllm.model_executor.layers.activation import get_act_fn
30 -from vllm.model_executor.layers.rotary_embedding import get_rope
29 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
30 LinearMethodBase,
31 QKVParallelLinear,
32 RowParallelLinear)
33 from vllm.model_executor.layers.logits_processor import LogitsProcessor
34 +from vllm.model_executor.layers.rotary_embedding import get_rope
35 from vllm.model_executor.layers.sampler import Sampler
36 from vllm.model_executor.layers.vocab_parallel_embedding import (
38 - VocabParallelEmbedding, ParallelLMHead, DEFAULT_VOCAB_PADDING_SIZE)
37 + DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
38 from vllm.model_executor.parallel_utils.parallel_state import (
39 get_tensor_model_parallel_world_size)
40 +from vllm.model_executor.sampling_metadata import SamplingMetadata
41 from vllm.model_executor.weight_utils import (default_weight_loader,
42 hf_model_weights_iterator)
43 from vllm.sequence import SamplerOutput
vllm/model_executor/neuron_model_loader.py
+2 -2
@@ -110,8 +110,8 @@ def _get_model_architecture(config: PretrainedConfig) -> Type[nn.Module]:
110 def get_neuron_model(model_config: ModelConfig,
111 parallel_config: ParallelConfig,
112 scheduler_config: SchedulerConfig) -> nn.Module:
113 - from transformers_neuronx.config import (NeuronConfig,
114 - ContinuousBatchingConfig)
113 + from transformers_neuronx.config import (ContinuousBatchingConfig,
114 + NeuronConfig)
115
116 # Create a model instance.
117 model = NeuronCasualLM(model_config.hf_config)
vllm/model_executor/parallel_utils/communication_op.py
+3 -6
@@ -5,14 +5,11 @@ import torch
5 from torch.distributed import ProcessGroup
6
7 from vllm.model_executor.parallel_utils import cupy_utils
8 -from vllm.model_executor.parallel_utils.parallel_state import (
9 - get_tensor_model_parallel_rank,
10 - get_tensor_model_parallel_world_size,
11 - get_tensor_model_parallel_group,
12 - is_cupy_nccl_enabled_for_all_reduce,
13 -)
8 from vllm.model_executor.parallel_utils.custom_all_reduce import (
9 custom_all_reduce)
10 +from vllm.model_executor.parallel_utils.parallel_state import (
11 + get_tensor_model_parallel_group, get_tensor_model_parallel_rank,
12 + get_tensor_model_parallel_world_size, is_cupy_nccl_enabled_for_all_reduce)
13
14
15 def tensor_model_parallel_all_reduce(input_: torch.Tensor) -> torch.Tensor:
vllm/model_executor/parallel_utils/custom_all_reduce.py
+3 -2
@@ -6,11 +6,12 @@ import torch.distributed as dist
6
7 from vllm.logger import init_logger
8 from vllm.model_executor.parallel_utils.parallel_state import (
9 - get_tensor_model_parallel_world_size, get_tensor_model_parallel_rank)
9 + get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
10
11 try:
12 - from vllm._C import custom_ar
12 import pynvml
13 +
14 + from vllm._C import custom_ar
15 except ImportError:
16 # For AMD GPUs
17 custom_ar = None
vllm/model_executor/sampling_metadata.py
+2 -3
@@ -1,11 +1,10 @@
1 +import random
2 from dataclasses import dataclass
3 from typing import Dict, List, Optional, Tuple
4
5 import torch
5 -import random
6
7 -from vllm.model_executor.layers.ops.sample import (
8 - get_num_triton_sampler_splits)
7 +from vllm.model_executor.layers.ops.sample import get_num_triton_sampler_splits
8 from vllm.sampling_params import SamplingParams, SamplingType
9 from vllm.sequence import SequenceData
10 from vllm.utils import is_pin_memory_available
vllm/model_executor/weight_utils.py
+6 -6
@@ -1,23 +1,23 @@
1 """Utilities for downloading and initializing model weights."""
2 -import filelock
2 +import fnmatch
3 import glob
4 import hashlib
5 -import fnmatch
5 import json
6 import os
7 from collections import defaultdict
8 from typing import Any, Iterator, List, Optional, Tuple
9
11 -from huggingface_hub import snapshot_download, HfFileSystem
10 +import filelock
11 import numpy as np
13 -from safetensors.torch import load_file, save_file, safe_open
12 import torch
13 +from huggingface_hub import HfFileSystem, snapshot_download
14 +from safetensors.torch import load_file, safe_open, save_file
15 from tqdm.auto import tqdm
16
17 from vllm.config import ModelConfig
18 from vllm.logger import init_logger
19 -from vllm.model_executor.layers.quantization import (get_quantization_config,
20 - QuantizationConfig)
19 +from vllm.model_executor.layers.quantization import (QuantizationConfig,
20 + get_quantization_config)
21
22 logger = init_logger(__name__)
23
vllm/outputs.py
+3 -3
@@ -1,9 +1,9 @@
1 -from typing import List, Optional
1 import time
2 +from typing import List, Optional
3
4 -from vllm.sequence import (PromptLogprobs, SampleLogprobs, SequenceGroup,
5 - SequenceStatus, RequestMetrics)
4 from vllm.lora.request import LoRARequest
5 +from vllm.sequence import (PromptLogprobs, RequestMetrics, SampleLogprobs,
6 + SequenceGroup, SequenceStatus)
7
8
9 class CompletionOutput:
vllm/sequence.py
+3 -2
@@ -2,14 +2,15 @@
2 import copy
3 import enum
4 from dataclasses import dataclass
5 -from typing import Dict, List, Optional, Union, TYPE_CHECKING
5 +from typing import TYPE_CHECKING, Dict, List, Optional, Union
6
7 from vllm.block import LogicalTokenBlock
8 -from vllm.sampling_params import SamplingParams
8 from vllm.lora.request import LoRARequest
9 +from vllm.sampling_params import SamplingParams
10
11 if TYPE_CHECKING:
12 import torch
13 +
14 from vllm.spec_decode.metrics import SpecDecodeWorkerMetrics
15
16
vllm/spec_decode/batch_expansion.py
+7 -8
@@ -1,16 +1,15 @@
1 -from typing import Iterator, List, Tuple, Optional, Dict
1 from itertools import chain, count
2 +from typing import Dict, Iterator, List, Optional, Tuple
3
4 import torch
5
6 -from vllm.sequence import (SamplerOutput, SequenceGroupMetadata, SequenceData)
7 -from vllm.worker.worker import Worker
8 -from vllm.spec_decode.util import (nvtx_range, sampler_output_to_torch,
9 - get_all_seq_ids,
6 +from vllm.sequence import SamplerOutput, SequenceData, SequenceGroupMetadata
7 +from vllm.spec_decode.interfaces import (SpeculativeProposals,
8 + SpeculativeScorer, SpeculativeScores)
9 +from vllm.spec_decode.util import (get_all_seq_ids, nvtx_range,
10 + sampler_output_to_torch,
11 split_batch_by_proposal_len)
11 -from vllm.spec_decode.interfaces import (SpeculativeScorer,
12 - SpeculativeProposals,
13 - SpeculativeScores)
12 +from vllm.worker.worker import Worker
13
14 SeqId = int
15 TargetSeqId = int
vllm/spec_decode/interfaces.py
+2 -2
@@ -1,6 +1,6 @@
1 -from typing import List, Tuple, Optional, Dict
2 -from dataclasses import dataclass
1 from abc import ABC, abstractmethod
2 +from dataclasses import dataclass
3 +from typing import Dict, List, Optional, Tuple
4
5 import torch
6
vllm/spec_decode/metrics.py
+5 -4
@@ -1,10 +1,11 @@
1 -import torch
1 +import time
2 from dataclasses import dataclass
3 +from typing import Callable, Optional
4 +
5 +import torch
6 +
7 from vllm.model_executor.layers.rejection_sampler import RejectionSampler
4 -from typing import Optional
8 from vllm.utils import is_pin_memory_available
6 -import time
7 -from typing import Callable
9
10
11 @dataclass
vllm/spec_decode/multi_step_worker.py
+2 -2
@@ -1,13 +1,13 @@
1 -from typing import List, Dict, Optional, Tuple
1 import copy
2 +from typing import Dict, List, Optional, Tuple
3
4 import torch
5
6 from vllm.sequence import SamplerOutput, SequenceGroupMetadata
7 -from vllm.worker.worker import Worker
7 from vllm.spec_decode.interfaces import (SpeculativeProposals,
8 SpeculativeProposer)
9 from vllm.spec_decode.util import sampler_output_to_torch
10 +from vllm.worker.worker import Worker
11
12
13 class MultiStepWorker(Worker):
vllm/spec_decode/spec_decode_worker.py
+9 -9
@@ -1,20 +1,20 @@
1 -from typing import List, Tuple, Optional, Dict
1 from functools import cached_property
2 +from typing import Dict, List, Optional, Tuple
3
4 import torch
5
6 -from vllm.spec_decode.metrics import AsyncMetricsCollector
6 +from vllm.config import CacheConfig
7 +from vllm.model_executor.layers.rejection_sampler import RejectionSampler
8 from vllm.sequence import (SamplerOutput, SequenceGroupMetadata,
9 SequenceGroupOutput, SequenceOutput)
9 -from vllm.worker.worker import Worker
10 +from vllm.spec_decode.batch_expansion import BatchExpansionTop1Scorer
11 +from vllm.spec_decode.interfaces import (SpeculativeProposals,
12 + SpeculativeScorer, SpeculativeScores)
13 +from vllm.spec_decode.metrics import AsyncMetricsCollector
14 from vllm.spec_decode.multi_step_worker import MultiStepWorker
11 -from vllm.model_executor.layers.rejection_sampler import RejectionSampler
12 -from vllm.config import CacheConfig
13 -from vllm.spec_decode.util import (nvtx_range, get_all_seq_ids,
15 +from vllm.spec_decode.util import (get_all_seq_ids, nvtx_range,
16 split_batch_by_proposal_len)
15 -from vllm.spec_decode.interfaces import SpeculativeProposals, SpeculativeScores
16 -from vllm.spec_decode.batch_expansion import BatchExpansionTop1Scorer
17 -from vllm.spec_decode.interfaces import SpeculativeScorer
17 +from vllm.worker.worker import Worker
18
19
20 class SpecDecodeWorker:
vllm/spec_decode/util.py
+5 -3
@@ -1,8 +1,10 @@
1 -import torch
2 -from typing import List, Tuple
3 -from vllm.sequence import SequenceGroupMetadata, SamplerOutput
1 from contextlib import contextmanager
2 from itertools import chain
3 +from typing import List, Tuple
4 +
5 +import torch
6 +
7 +from vllm.sequence import SamplerOutput, SequenceGroupMetadata
8
9 SeqId = int
10
vllm/transformers_utils/configs/__init__.py
+1 -1
@@ -1,10 +1,10 @@
1 from vllm.transformers_utils.configs.chatglm import ChatGLMConfig
2 -from vllm.transformers_utils.configs.mpt import MPTConfig
2 # RWConfig is for the original tiiuae/falcon-40b(-instruct) and
3 # tiiuae/falcon-7b(-instruct) models. Newer Falcon models will use the
4 # `FalconConfig` class from the official HuggingFace transformers library.
5 from vllm.transformers_utils.configs.falcon import RWConfig
6 from vllm.transformers_utils.configs.jais import JAISConfig
7 +from vllm.transformers_utils.configs.mpt import MPTConfig
8
9 __all__ = [
10 "ChatGLMConfig",
vllm/transformers_utils/configs/mpt.py
+1
@@ -4,6 +4,7 @@
4 """A HuggingFace-style model configuration."""
5 import warnings
6 from typing import Any, Dict, Optional, Union
7 +
8 from transformers import PretrainedConfig
9
10 attn_config_defaults: Dict = {
vllm/transformers_utils/detokenizer.py
+6 -4
@@ -1,8 +1,10 @@
1 -from typing import List, Dict, Optional
1 +from typing import Dict, List, Optional
2 +
3 from transformers import PreTrainedTokenizer
3 -from vllm.sequence import Sequence, Logprob, SequenceGroup, SamplingParams
4 -from vllm.transformers_utils.tokenizer import (detokenize_incrementally,
5 - convert_prompt_ids_to_tokens)
4 +
5 +from vllm.sequence import Logprob, SamplingParams, Sequence, SequenceGroup
6 +from vllm.transformers_utils.tokenizer import (convert_prompt_ids_to_tokens,
7 + detokenize_incrementally)
8 from vllm.transformers_utils.tokenizer_group.base_tokenizer_group import (
9 BaseTokenizerGroup)
10
vllm/transformers_utils/tokenizer.py
+1 -1
@@ -5,8 +5,8 @@ from transformers import (AutoTokenizer, PreTrainedTokenizer,
5
6 from vllm.logger import init_logger
7 from vllm.lora.request import LoRARequest
8 -from vllm.utils import make_async
8 from vllm.transformers_utils.tokenizers import *
9 +from vllm.utils import make_async
10
11 logger = init_logger(__name__)
12
vllm/transformers_utils/tokenizer_group/__init__.py
+2 -1
@@ -1,10 +1,11 @@
1 from typing import Optional
2 +
3 from vllm.config import TokenizerPoolConfig
4 +from vllm.engine.ray_utils import ray
5 from vllm.transformers_utils.tokenizer_group.base_tokenizer_group import (
6 BaseTokenizerGroup)
7 from vllm.transformers_utils.tokenizer_group.tokenizer_group import (
8 TokenizerGroup)
7 -from vllm.engine.ray_utils import ray
9
10 if ray:
11 from vllm.transformers_utils.tokenizer_group.ray_tokenizer_group import (
vllm/transformers_utils/tokenizer_group/ray_tokenizer_group.py
+2 -2
@@ -2,16 +2,16 @@ import asyncio
2 import os
3 from typing import List, Optional
4
5 +from ray.util.scheduling_strategies import NodeAffinitySchedulingStrategy
6 from transformers import PreTrainedTokenizer
7
8 from vllm.config import TokenizerPoolConfig
8 -from vllm.lora.request import LoRARequest
9 from vllm.engine.ray_utils import ray
10 +from vllm.lora.request import LoRARequest
11 from vllm.transformers_utils.tokenizer_group.base_tokenizer_group import (
12 BaseTokenizerGroup)
13 from vllm.transformers_utils.tokenizer_group.tokenizer_group import (
14 TokenizerGroup)
14 -from ray.util.scheduling_strategies import NodeAffinitySchedulingStrategy
15
16
17 class RayTokenizerGroupPool(BaseTokenizerGroup):
vllm/transformers_utils/tokenizer_group/tokenizer_group.py
+2 -2
@@ -4,11 +4,11 @@ from transformers import PreTrainedTokenizer
4
5 from vllm.lora.request import LoRARequest
6 from vllm.transformers_utils.tokenizer import (get_lora_tokenizer,
7 - get_lora_tokenizer_async)
7 + get_lora_tokenizer_async,
8 + get_tokenizer)
9 from vllm.transformers_utils.tokenizer_group.base_tokenizer_group import (
10 BaseTokenizerGroup)
11 from vllm.utils import LRUCache
11 -from vllm.transformers_utils.tokenizer import get_tokenizer
12
13
14 class TokenizerGroup(BaseTokenizerGroup):
vllm/utils.py
+8 -13
@@ -1,27 +1,22 @@
1 +import asyncio
2 import enum
3 +import gc
4 import os
5 import socket
6 import subprocess
7 import uuid
6 -import gc
8 +import warnings
9 +from collections import OrderedDict
10 +from functools import lru_cache, partial
11 from platform import uname
8 -from typing import List, Tuple, Union, Generic
9 -from packaging.version import parse, Version
12 +from typing import (Any, Awaitable, Callable, Generic, Hashable, List,
13 + Optional, Tuple, TypeVar, Union)
14
15 import psutil
16 import torch
13 -import asyncio
14 -from functools import partial, lru_cache
15 -from typing import (
16 - Awaitable,
17 - Callable,
18 - TypeVar,
19 -)
20 -from collections import OrderedDict
21 -from typing import Any, Hashable, Optional
17 +from packaging.version import Version, parse
18
19 from vllm.logger import init_logger
24 -import warnings
20
21 T = TypeVar("T")
22 logger = init_logger(__name__)
vllm/worker/cache_engine.py
+1 -1
@@ -6,7 +6,7 @@ import torch
6 from vllm.attention import get_attn_backend
7 from vllm.config import CacheConfig, ModelConfig, ParallelConfig
8 from vllm.logger import init_logger
9 -from vllm.utils import is_pin_memory_available, STR_DTYPE_TO_TORCH_DTYPE
9 +from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE, is_pin_memory_available
10
11 logger = init_logger(__name__)
12
vllm/worker/model_runner.py
+7 -8
@@ -1,29 +1,28 @@
1 import contextlib
2 import time
3 -from typing import Dict, List, Optional, Tuple, Set
3 +from typing import Dict, List, Optional, Set, Tuple
4
5 import numpy as np
6 import torch
7 import torch.nn as nn
8
9 from vllm.attention import AttentionMetadata, get_attn_backend
10 -from vllm.config import (DeviceConfig, ModelConfig, LoRAConfig, ParallelConfig,
10 +from vllm.config import (DeviceConfig, LoRAConfig, ModelConfig, ParallelConfig,
11 SchedulerConfig)
12 from vllm.logger import init_logger
13 +from vllm.lora.layers import LoRAMapping
14 +from vllm.lora.request import LoRARequest
15 +from vllm.lora.worker_manager import LRUCacheWorkerLoRAManager
16 from vllm.model_executor import SamplingMetadata
17 from vllm.model_executor.model_loader import get_model
15 -from vllm.model_executor.parallel_utils import cupy_utils
18 +from vllm.model_executor.parallel_utils import cupy_utils, custom_all_reduce
19 from vllm.model_executor.parallel_utils.communication_op import (
20 broadcast_tensor_dict)
21 from vllm.model_executor.parallel_utils.parallel_state import (
22 with_cupy_nccl_for_all_reduce)
20 -from vllm.model_executor.parallel_utils import custom_all_reduce
23 from vllm.sampling_params import SamplingParams, SamplingType
24 from vllm.sequence import SamplerOutput, SequenceData, SequenceGroupMetadata
23 -from vllm.lora.worker_manager import LRUCacheWorkerLoRAManager
24 -from vllm.lora.layers import LoRAMapping
25 -from vllm.lora.request import LoRARequest
26 -from vllm.utils import (async_tensor_h2d, CudaMemoryProfiler,
25 +from vllm.utils import (CudaMemoryProfiler, async_tensor_h2d,
26 is_pin_memory_available, make_tensor_with_pad,
27 maybe_expand_dim)
28
vllm/worker/worker.py
+4 -4
@@ -1,13 +1,14 @@
1 """A GPU worker class."""
2 import gc
3 import os
4 -from typing import Dict, List, Tuple, Set, Optional
4 +from typing import Dict, List, Optional, Set, Tuple
5
6 import torch
7 import torch.distributed
8
9 -from vllm.config import (CacheConfig, DeviceConfig, ModelConfig,
10 - ParallelConfig, SchedulerConfig, LoRAConfig)
9 +from vllm.config import (CacheConfig, DeviceConfig, LoRAConfig, ModelConfig,
10 + ParallelConfig, SchedulerConfig)
11 +from vllm.lora.request import LoRARequest
12 from vllm.model_executor import set_random_seed
13 from vllm.model_executor.parallel_utils import cupy_utils
14 from vllm.model_executor.parallel_utils.communication_op import (
@@ -18,7 +19,6 @@ from vllm.model_executor.parallel_utils.parallel_state import (
19 from vllm.sequence import SamplerOutput, SequenceGroupMetadata
20 from vllm.worker.cache_engine import CacheEngine
21 from vllm.worker.model_runner import ModelRunner
21 -from vllm.lora.request import LoRARequest
22
23
24 class Worker: