A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 15:45 UTC
[CI/Build] Update Ruff version (#8469)
Signed-off-by: Aaron Pham <contact@aarnphm.xyz> Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com>
Aaron Pham committed
Sep 18, 2024 at 07:00 UTC
9d104b5beb7bbb51c64b680e007f39169489ea86
27 files changed
+50
-77
.github/workflows/ruff.yml
+2
-2
@@ -25,10 +25,10 @@ jobs:
25
- name: Install dependencies
26
run: |
27
python -m pip install --upgrade pip
28
- pip install ruff==0.1.5 codespell==2.3.0 tomli==2.0.1 isort==5.13.2
28
+ pip install -r requirements-lint.txt
29
- name: Analysing the code with ruff
30
run: |
31
- ruff .
31
+ ruff check .
32
- name: Spelling check with codespell
33
run: |
34
codespell --toml pyproject.toml
benchmarks/kernels/graph_machete_bench.py
+1
-3
@@ -45,8 +45,7 @@ if __name__ == "__main__":
45
rows = int(math.ceil(len(results) / 2))
46
fig, axs = plt.subplots(rows, 2, figsize=(12, 5 * rows))
47
axs = axs.flatten()
48
- axs_idx = 0
49
- for shape, data in results.items():
48
+ for axs_idx, (shape, data) in enumerate(results.items()):
49
plt.sca(axs[axs_idx])
50
df = pd.DataFrame(data)
51
sns.lineplot(data=df,
@@ -59,6 +58,5 @@ if __name__ == "__main__":
58
palette="Dark2")
59
plt.title(f"Shape: {shape}")
60
plt.ylabel("time (median, s)")
62
- axs_idx += 1
61
plt.tight_layout()
62
plt.savefig("graph_machete_bench.pdf")
format.sh
+2
-2
@@ -159,7 +159,7 @@ echo 'vLLM codespell: Done'
159
160
# Lint specified files
161
lint() {
162
- ruff "$@"
162
+ ruff check "$@"
163
}
164
165
# Lint files that differ from main branch. Ignores dirs that are not slated
@@ -175,7 +175,7 @@ lint_changed() {
175
176
if ! git diff --diff-filter=ACM --quiet --exit-code "$MERGEBASE" -- '*.py' '*.pyi' &>/dev/null; then
177
git diff --name-only --diff-filter=ACM "$MERGEBASE" -- '*.py' '*.pyi' | xargs \
178
- ruff
178
+ ruff check
179
fi
180
181
}
pyproject.toml
+2
@@ -42,6 +42,8 @@ ignore = [
42
"E731",
43
# Loop control variable not used within loop body
44
"B007",
45
+ # f-string format
46
+ "UP032",
47
]
48
49
[tool.mypy]
requirements-lint.txt
+1
-1
@@ -2,7 +2,7 @@
2
yapf==0.32.0
3
toml==0.10.2
4
tomli==2.0.1
5
-ruff==0.1.5
5
+ruff==0.6.5
6
codespell==2.3.0
7
isort==5.13.2
8
clang-format==18.1.5
tests/conftest.py
+1
-4
@@ -158,10 +158,7 @@ def should_do_global_cleanup_after_test(request) -> bool:
158
to initialize torch.
159
"""
160
161
- if request.node.get_closest_marker("skip_global_cleanup"):
162
- return False
163
-
164
- return True
161
+ return not request.node.get_closest_marker("skip_global_cleanup")
162
163
164
@pytest.fixture(autouse=True)
tests/lora/conftest.py
+1
-4
@@ -65,10 +65,7 @@ def should_do_global_cleanup_after_test(request) -> bool:
65
to initialize torch.
66
"""
67
68
- if request.node.get_closest_marker("skip_global_cleanup"):
69
- return False
70
-
71
- return True
68
+ return not request.node.get_closest_marker("skip_global_cleanup")
69
70
71
@pytest.fixture(autouse=True)
tests/multimodal/test_base.py
+1
-1
@@ -5,7 +5,7 @@ from vllm.multimodal.base import MultiModalInputs, NestedTensors
5
6
def assert_nested_tensors_equal(expected: NestedTensors,
7
actual: NestedTensors):
8
- assert type(expected) == type(actual)
8
+ assert type(expected) == type(actual) # noqa: E721
9
if isinstance(expected, torch.Tensor):
10
assert torch.equal(expected, actual)
11
else:
tests/test_cache_block_hashing.py
+1
-4
@@ -66,8 +66,7 @@ def test_auto_prefix_caching(model: str, block_size: int, max_num_seqs: int,
66
67
hashes.append([])
68
prompts = [prefix + prompt for prompt in sample_prompts]
69
- seq_id = 0
70
- for prompt in prompts:
69
+ for seq_id, prompt in enumerate(prompts):
70
hashes[-1].append([])
71
prompt_token_ids = tokenizer.encode(prompt)
72
seq = Sequence(seq_id,
@@ -83,8 +82,6 @@ def test_auto_prefix_caching(model: str, block_size: int, max_num_seqs: int,
82
for idx in range(num_blocks):
83
hashes[-1][-1].append(seq.hash_of_block(idx))
84
86
- seq_id += 1
87
-
85
# Check that hashes made with two prefixes with different first blocks are
86
# different everywhere.
87
for hash0, hash1 in zip(flatten_2d(hashes[0]), flatten_2d(hashes[1])):
tests/test_logger.py
+2
-2
@@ -111,7 +111,7 @@ def test_an_error_is_raised_when_custom_logging_config_file_does_not_exist():
111
configuration occurs."""
112
with pytest.raises(RuntimeError) as ex_info:
113
_configure_vllm_root_logger()
114
- assert ex_info.type == RuntimeError
114
+ assert ex_info.type == RuntimeError # noqa: E721
115
assert "File does not exist" in str(ex_info)
116
117
@@ -152,7 +152,7 @@ def test_an_error_is_raised_when_custom_logging_config_is_unexpected_json(
152
logging_config_file.name):
153
with pytest.raises(ValueError) as ex_info:
154
_configure_vllm_root_logger()
155
- assert ex_info.type == ValueError
155
+ assert ex_info.type == ValueError # noqa: E721
156
assert "Invalid logging config. Expected Dict, got" in str(ex_info)
157
158
tests/worker/test_encoder_decoder_model_runner.py
+1
-3
@@ -453,8 +453,7 @@ def test_prepare_decode(batch_size):
453
# each sequence) in the decode phase
454
455
expected_selected_token_indices = []
456
- selected_token_start_idx = 0
457
- for seq_len in seq_lens:
456
+ for selected_token_start_idx, seq_len in enumerate(seq_lens):
457
# Compute the index offset of the final token in each
458
# sequence's decoded outputs; since a single token is
459
# decoded per iteration per sequence, then the length
@@ -463,7 +462,6 @@ def test_prepare_decode(batch_size):
462
# generated tokens is 0 (i.e. the expected sampling index
463
# for a given sequence is just `selected_token_start_idx`)
464
expected_selected_token_indices.append(selected_token_start_idx)
466
- selected_token_start_idx += 1
465
466
sampling_metadata = model_input.sampling_metadata
467
actual = sampling_metadata.selected_token_indices
tests/worker/test_model_runner.py
+1
-3
@@ -241,10 +241,8 @@ def test_prepare_decode_cuda_graph(batch_size):
241
242
# Verify Sampling
243
expected_selected_token_indices = []
244
- selected_token_start_idx = 0
245
- for _ in context_lens:
244
+ for selected_token_start_idx, _ in enumerate(context_lens):
245
expected_selected_token_indices.append(selected_token_start_idx)
247
- selected_token_start_idx += 1
246
sampling_metadata = SamplingMetadata.prepare(
247
seq_group_metadata_list,
248
seq_lens,
vllm/adapter_commons/utils.py
+1
-1
@@ -42,7 +42,7 @@ def list_adapters(registered_adapters: Dict[int, Any]) -> Dict[int, Any]:
42
43
def get_adapter(adapter_id: int,
44
registered_adapters: Dict[int, Any]) -> Optional[Any]:
45
- return registered_adapters.get(adapter_id, None)
45
+ return registered_adapters.get(adapter_id)
46
47
48
## worker functions
vllm/attention/backends/utils.py
+2
-4
@@ -33,10 +33,8 @@ def is_block_tables_empty(block_tables: Union[None, Dict]):
33
"""
34
if block_tables is None:
35
return True
36
- if isinstance(block_tables, dict) and all(
37
- value is None for value in block_tables.values()):
38
- return True
39
- return False
36
+ return (isinstance(block_tables, dict)
37
+ and all(value is None for value in block_tables.values()))
38
39
40
def compute_slot_mapping_start_idx(is_prompt: bool, query_len: int,
vllm/core/block/prefix_caching_block.py
+1
-3
@@ -417,9 +417,7 @@ class PrefixCachingBlockAllocator(BlockAllocator):
417
418
def is_block_cached(self, block: Block) -> bool:
419
assert block.content_hash is not None
420
- if block.content_hash in self._cached_blocks:
421
- return True
422
- return False
420
+ return block.content_hash in self._cached_blocks
421
422
def promote_to_immutable_block(self, block: Block) -> BlockId:
423
"""Once a mutable block is full, it can be promoted to an immutable
vllm/core/block_manager_v2.py
+1
-3
@@ -399,9 +399,7 @@ class BlockSpaceManagerV2(BlockSpaceManager):
399
"""
400
alloc_status = self._can_swap(seq_group, Device.CPU,
401
SequenceStatus.RUNNING)
402
- if alloc_status == AllocStatus.OK:
403
- return True
404
- return False
402
+ return alloc_status == AllocStatus.OK
403
404
def swap_out(self, seq_group: SequenceGroup) -> List[Tuple[int, int]]:
405
"""Returns the block id mapping (from GPU to CPU) generated by
vllm/engine/async_llm_engine.py
+3
-3
@@ -826,7 +826,7 @@ class AsyncLLMEngine:
826
request_id: The unique id of the request.
827
lora_request: LoRA request to use for generation, if any.
828
trace_headers: OpenTelemetry trace headers.
829
- prompt_adapter_request: Prompt Adapter request to use
829
+ prompt_adapter_request: Prompt Adapter request to use
830
for generation, if any.
831
832
Yields:
@@ -1042,7 +1042,7 @@ class AsyncLLMEngine:
1042
async def start_profile(self) -> None:
1043
# using type instead of isinstance to check to avoid capturing
1044
# inherited classes
1045
- if type(self.engine.model_executor) == GPUExecutorAsync:
1045
+ if type(self.engine.model_executor) == GPUExecutorAsync: # noqa: E721
1046
self.engine.model_executor.start_profile()
1047
else:
1048
self.engine.model_executor._run_workers("start_profile")
@@ -1050,7 +1050,7 @@ class AsyncLLMEngine:
1050
async def stop_profile(self) -> None:
1051
# using type instead of isinstance to check to avoid capturing
1052
# inherited classes
1053
- if type(self.engine.model_executor) == GPUExecutorAsync:
1053
+ if type(self.engine.model_executor) == GPUExecutorAsync: # noqa: E721
1054
self.engine.model_executor.stop_profile()
1055
else:
1056
self.engine.model_executor._run_workers("stop_profile")
vllm/engine/llm_engine.py
+3
-3
@@ -144,7 +144,7 @@ class LLMEngine:
144
decoding.
145
executor_class: The model executor class for managing distributed
146
execution.
147
- prompt_adapter_config (Optional): The configuration related to serving
147
+ prompt_adapter_config (Optional): The configuration related to serving
148
prompt adapters.
149
log_stats: Whether to log statistics.
150
usage_context: Specified entry point, used for usage info collection.
@@ -1605,7 +1605,7 @@ class LLMEngine:
1605
def start_profile(self) -> None:
1606
# using type instead of isinstance to check to avoid capturing
1607
# inherited classes (MultiprocessingGPUExecutor)
1608
- if type(self.model_executor) == GPUExecutor:
1608
+ if type(self.model_executor) == GPUExecutor: # noqa: E721
1609
self.model_executor.start_profile()
1610
else:
1611
self.model_executor._run_workers("start_profile")
@@ -1613,7 +1613,7 @@ class LLMEngine:
1613
def stop_profile(self) -> None:
1614
# using type instead of isinstance to check to avoid capturing
1615
# inherited classes (MultiprocessingGPUExecutor)
1616
- if type(self.model_executor) == GPUExecutor:
1616
+ if type(self.model_executor) == GPUExecutor: # noqa: E721
1617
self.model_executor.stop_profile()
1618
else:
1619
self.model_executor._run_workers("stop_profile")
vllm/model_executor/guided_decoding/outlines_logits_processors.py
+2
-2
@@ -67,9 +67,9 @@ class BaseLogitsProcessor:
67
instruction = self._guide.get_next_instruction(
68
state=self._fsm_state[seq_id])
69
70
- if type(instruction) == Generate:
70
+ if type(instruction) == Generate: # noqa: E721
71
allowed_tokens = instruction.tokens
72
- elif type(instruction) == Write:
72
+ elif type(instruction) == Write: # noqa: E721
73
# TODO: support fast forward tokens
74
allowed_tokens = [instruction.tokens[0]]
75
else:
vllm/model_executor/layers/quantization/awq_marlin.py
+3
-3
@@ -110,9 +110,9 @@ class AWQMarlinConfig(QuantizationConfig):
110
def is_awq_marlin_compatible(cls, quant_config: Dict[str, Any]):
111
# Extract data from quant config.
112
quant_method = quant_config.get("quant_method", "").lower()
113
- num_bits = quant_config.get("bits", None)
114
- group_size = quant_config.get("group_size", None)
115
- has_zp = quant_config.get("zero_point", None)
113
+ num_bits = quant_config.get("bits")
114
+ group_size = quant_config.get("group_size")
115
+ has_zp = quant_config.get("zero_point")
116
117
if quant_method != "awq":
118
return False
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py
+7
-7
@@ -1,4 +1,4 @@
1
-from typing import Any, Dict, List, Optional
1
+from typing import Any, Dict, List, Optional, cast
2
3
import torch
4
from pydantic import BaseModel
@@ -79,8 +79,8 @@ class CompressedTensorsConfig(QuantizationConfig):
79
@classmethod
80
def from_config(cls, config: Dict[str, Any]) -> "CompressedTensorsConfig":
81
target_scheme_map: Dict[str, Any] = dict()
82
- ignore: List[str] = config.get("ignore", None)
83
- quant_format: str = config.get("format", None)
82
+ ignore = cast(List[str], config.get("ignore"))
83
+ quant_format = cast(str, config.get("format"))
84
85
# The quant_config has multiple config_groups, each containing
86
# an input_activations key with details about how the activations are
@@ -200,7 +200,7 @@ class CompressedTensorsConfig(QuantizationConfig):
200
is_per_tensor_or_channel_weight = (weight_quant.strategy in [
201
QuantizationStrategy.TENSOR, QuantizationStrategy.CHANNEL
202
])
203
- if not (is_symmetric_weight and is_static_weight
203
+ if not (is_symmetric_weight and is_static_weight # noqa: SIM103
204
and is_per_tensor_or_channel_weight):
205
return False
206
@@ -333,7 +333,7 @@ class CompressedTensorsLinearMethod(LinearMethodBase):
333
output_size: int, params_dtype: torch.dtype,
334
**extra_weight_attrs):
335
"""
336
- Use the CompressedTensorsScheme associated with each layer to create
336
+ Use the CompressedTensorsScheme associated with each layer to create
337
the necessary parameters for the layer. See LinearMethodBase for param
338
details
339
"""
@@ -352,8 +352,8 @@ class CompressedTensorsLinearMethod(LinearMethodBase):
352
x: torch.Tensor,
353
bias: Optional[torch.Tensor] = None):
354
"""
355
- Use the output of create_weights and the CompressedTensorsScheme
356
- associated with the layer to apply the forward pass with the
355
+ Use the output of create_weights and the CompressedTensorsScheme
356
+ associated with the layer to apply the forward pass with the
357
layer input. See LinearMethodBase for param details
358
359
"""
vllm/model_executor/layers/quantization/gptq_marlin.py
+4
-4
@@ -132,10 +132,10 @@ class GPTQMarlinConfig(QuantizationConfig):
132
def is_gptq_marlin_compatible(cls, quant_config: Dict[str, Any]):
133
# Extract data from quant config.
134
quant_method = quant_config.get("quant_method", "").lower()
135
- num_bits = quant_config.get("bits", None)
136
- group_size = quant_config.get("group_size", None)
137
- sym = quant_config.get("sym", None)
138
- desc_act = quant_config.get("desc_act", None)
135
+ num_bits = quant_config.get("bits")
136
+ group_size = quant_config.get("group_size")
137
+ sym = quant_config.get("sym")
138
+ desc_act = quant_config.get("desc_act")
139
140
if quant_method != "gptq":
141
return False
vllm/model_executor/model_loader/tensorizer.py
+1
-3
@@ -408,9 +408,7 @@ def is_vllm_tensorized(tensorizer_config: "TensorizerConfig") -> bool:
408
"inferred as vLLM models, so setting vllm_tensorized=True is "
409
"only necessary for models serialized prior to this change.")
410
return True
411
- if (".vllm_tensorized_marker" in deserializer):
412
- return True
413
- return False
411
+ return ".vllm_tensorized_marker" in deserializer
412
413
414
def serialize_vllm_model(
vllm/model_executor/models/minicpmv.py
+1
-1
@@ -884,7 +884,7 @@ class MiniCPMV(MiniCPMVBaseModel):
884
version = str(config.version).split(".")
885
version = tuple([int(x) for x in version])
886
# Dispatch class based on version
887
- instance_class = _SUPPORT_VERSION.get(version, None)
887
+ instance_class = _SUPPORT_VERSION.get(version)
888
if instance_class is None:
889
raise ValueError(
890
"Currently, MiniCPMV only supports versions 2.0, 2.5, and 2.6")
vllm/spec_decode/draft_model_runner.py
+1
-4
@@ -183,10 +183,7 @@ class TP1DraftModelRunner(ModelRunner):
183
return False
184
185
# TODO: Add soft-tuning prompt adapter support
186
- if self.prompt_adapter_config:
187
- return False
188
-
189
- return True
186
+ return not self.prompt_adapter_config
187
188
@torch.inference_mode()
189
def execute_model(
vllm/spec_decode/metrics.py
+2
-5
@@ -104,13 +104,10 @@ class AsyncMetricsCollector:
104
if self._rank != 0:
105
return False
106
107
- if (now - self._last_metrics_collect_time <
108
- self._rejsample_metrics_collect_interval_s):
109
- return False
110
- return True
107
+ return now - self._last_metrics_collect_time >= self._rejsample_metrics_collect_interval_s # noqa: E501
108
109
def _copy_rejsample_metrics_async(self) -> torch.cuda.Event:
113
- """Copy rejection/typical-acceptance sampling metrics
110
+ """Copy rejection/typical-acceptance sampling metrics
111
(number of accepted tokens, etc) to CPU asynchronously.
112
113
Returns a CUDA event recording when the copy is complete.
vllm/triton_utils/libentry.py
+2
-2
@@ -35,8 +35,8 @@ class LibEntry(triton.KernelInterface):
35
dns_key = [
36
arg.dtype if hasattr(
37
arg, "data_ptr") else type(arg) if not isinstance(arg, int)
38
- else "i32" if -(2**31) <= arg and arg <= 2**31 -
39
- 1 else "u64" if 2**63 <= arg and arg <= 2**64 - 1 else "i64"
38
+ else "i32" if arg >= -(2**31) and arg <= 2**31 -
39
+ 1 else "u64" if arg >= 2**63 and arg <= 2**64 - 1 else "i64"
40
for arg in dns_args
41
]
42
# const args passed by position