@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 9d104b5beb
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 15:45 UTC

[CI/Build] Update Ruff version (#8469)

Signed-off-by: Aaron Pham <contact@aarnphm.xyz> Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com>

Aaron Pham committed Sep 18, 2024 at 07:00 UTC 9d104b5beb7bbb51c64b680e007f39169489ea86
27 files changed +50 -77
.github/workflows/ruff.yml
+2 -2
@@ -25,10 +25,10 @@ jobs:
25 - name: Install dependencies
26 run: |
27 python -m pip install --upgrade pip
28 - pip install ruff==0.1.5 codespell==2.3.0 tomli==2.0.1 isort==5.13.2
28 + pip install -r requirements-lint.txt
29 - name: Analysing the code with ruff
30 run: |
31 - ruff .
31 + ruff check .
32 - name: Spelling check with codespell
33 run: |
34 codespell --toml pyproject.toml
benchmarks/kernels/graph_machete_bench.py
+1 -3
@@ -45,8 +45,7 @@ if __name__ == "__main__":
45 rows = int(math.ceil(len(results) / 2))
46 fig, axs = plt.subplots(rows, 2, figsize=(12, 5 * rows))
47 axs = axs.flatten()
48 - axs_idx = 0
49 - for shape, data in results.items():
48 + for axs_idx, (shape, data) in enumerate(results.items()):
49 plt.sca(axs[axs_idx])
50 df = pd.DataFrame(data)
51 sns.lineplot(data=df,
@@ -59,6 +58,5 @@ if __name__ == "__main__":
58 palette="Dark2")
59 plt.title(f"Shape: {shape}")
60 plt.ylabel("time (median, s)")
62 - axs_idx += 1
61 plt.tight_layout()
62 plt.savefig("graph_machete_bench.pdf")
format.sh
+2 -2
@@ -159,7 +159,7 @@ echo 'vLLM codespell: Done'
159
160 # Lint specified files
161 lint() {
162 - ruff "$@"
162 + ruff check "$@"
163 }
164
165 # Lint files that differ from main branch. Ignores dirs that are not slated
@@ -175,7 +175,7 @@ lint_changed() {
175
176 if ! git diff --diff-filter=ACM --quiet --exit-code "$MERGEBASE" -- '*.py' '*.pyi' &>/dev/null; then
177 git diff --name-only --diff-filter=ACM "$MERGEBASE" -- '*.py' '*.pyi' | xargs \
178 - ruff
178 + ruff check
179 fi
180
181 }
pyproject.toml
+2
@@ -42,6 +42,8 @@ ignore = [
42 "E731",
43 # Loop control variable not used within loop body
44 "B007",
45 + # f-string format
46 + "UP032",
47 ]
48
49 [tool.mypy]
requirements-lint.txt
+1 -1
@@ -2,7 +2,7 @@
2 yapf==0.32.0
3 toml==0.10.2
4 tomli==2.0.1
5 -ruff==0.1.5
5 +ruff==0.6.5
6 codespell==2.3.0
7 isort==5.13.2
8 clang-format==18.1.5
tests/conftest.py
+1 -4
@@ -158,10 +158,7 @@ def should_do_global_cleanup_after_test(request) -> bool:
158 to initialize torch.
159 """
160
161 - if request.node.get_closest_marker("skip_global_cleanup"):
162 - return False
163 -
164 - return True
161 + return not request.node.get_closest_marker("skip_global_cleanup")
162
163
164 @pytest.fixture(autouse=True)
tests/lora/conftest.py
+1 -4
@@ -65,10 +65,7 @@ def should_do_global_cleanup_after_test(request) -> bool:
65 to initialize torch.
66 """
67
68 - if request.node.get_closest_marker("skip_global_cleanup"):
69 - return False
70 -
71 - return True
68 + return not request.node.get_closest_marker("skip_global_cleanup")
69
70
71 @pytest.fixture(autouse=True)
tests/multimodal/test_base.py
+1 -1
@@ -5,7 +5,7 @@ from vllm.multimodal.base import MultiModalInputs, NestedTensors
5
6 def assert_nested_tensors_equal(expected: NestedTensors,
7 actual: NestedTensors):
8 - assert type(expected) == type(actual)
8 + assert type(expected) == type(actual) # noqa: E721
9 if isinstance(expected, torch.Tensor):
10 assert torch.equal(expected, actual)
11 else:
tests/test_cache_block_hashing.py
+1 -4
@@ -66,8 +66,7 @@ def test_auto_prefix_caching(model: str, block_size: int, max_num_seqs: int,
66
67 hashes.append([])
68 prompts = [prefix + prompt for prompt in sample_prompts]
69 - seq_id = 0
70 - for prompt in prompts:
69 + for seq_id, prompt in enumerate(prompts):
70 hashes[-1].append([])
71 prompt_token_ids = tokenizer.encode(prompt)
72 seq = Sequence(seq_id,
@@ -83,8 +82,6 @@ def test_auto_prefix_caching(model: str, block_size: int, max_num_seqs: int,
82 for idx in range(num_blocks):
83 hashes[-1][-1].append(seq.hash_of_block(idx))
84
86 - seq_id += 1
87 -
85 # Check that hashes made with two prefixes with different first blocks are
86 # different everywhere.
87 for hash0, hash1 in zip(flatten_2d(hashes[0]), flatten_2d(hashes[1])):
tests/test_logger.py
+2 -2
@@ -111,7 +111,7 @@ def test_an_error_is_raised_when_custom_logging_config_file_does_not_exist():
111 configuration occurs."""
112 with pytest.raises(RuntimeError) as ex_info:
113 _configure_vllm_root_logger()
114 - assert ex_info.type == RuntimeError
114 + assert ex_info.type == RuntimeError # noqa: E721
115 assert "File does not exist" in str(ex_info)
116
117
@@ -152,7 +152,7 @@ def test_an_error_is_raised_when_custom_logging_config_is_unexpected_json(
152 logging_config_file.name):
153 with pytest.raises(ValueError) as ex_info:
154 _configure_vllm_root_logger()
155 - assert ex_info.type == ValueError
155 + assert ex_info.type == ValueError # noqa: E721
156 assert "Invalid logging config. Expected Dict, got" in str(ex_info)
157
158
tests/worker/test_encoder_decoder_model_runner.py
+1 -3
@@ -453,8 +453,7 @@ def test_prepare_decode(batch_size):
453 # each sequence) in the decode phase
454
455 expected_selected_token_indices = []
456 - selected_token_start_idx = 0
457 - for seq_len in seq_lens:
456 + for selected_token_start_idx, seq_len in enumerate(seq_lens):
457 # Compute the index offset of the final token in each
458 # sequence's decoded outputs; since a single token is
459 # decoded per iteration per sequence, then the length
@@ -463,7 +462,6 @@ def test_prepare_decode(batch_size):
462 # generated tokens is 0 (i.e. the expected sampling index
463 # for a given sequence is just `selected_token_start_idx`)
464 expected_selected_token_indices.append(selected_token_start_idx)
466 - selected_token_start_idx += 1
465
466 sampling_metadata = model_input.sampling_metadata
467 actual = sampling_metadata.selected_token_indices
tests/worker/test_model_runner.py
+1 -3
@@ -241,10 +241,8 @@ def test_prepare_decode_cuda_graph(batch_size):
241
242 # Verify Sampling
243 expected_selected_token_indices = []
244 - selected_token_start_idx = 0
245 - for _ in context_lens:
244 + for selected_token_start_idx, _ in enumerate(context_lens):
245 expected_selected_token_indices.append(selected_token_start_idx)
247 - selected_token_start_idx += 1
246 sampling_metadata = SamplingMetadata.prepare(
247 seq_group_metadata_list,
248 seq_lens,
vllm/adapter_commons/utils.py
+1 -1
@@ -42,7 +42,7 @@ def list_adapters(registered_adapters: Dict[int, Any]) -> Dict[int, Any]:
42
43 def get_adapter(adapter_id: int,
44 registered_adapters: Dict[int, Any]) -> Optional[Any]:
45 - return registered_adapters.get(adapter_id, None)
45 + return registered_adapters.get(adapter_id)
46
47
48 ## worker functions
vllm/attention/backends/utils.py
+2 -4
@@ -33,10 +33,8 @@ def is_block_tables_empty(block_tables: Union[None, Dict]):
33 """
34 if block_tables is None:
35 return True
36 - if isinstance(block_tables, dict) and all(
37 - value is None for value in block_tables.values()):
38 - return True
39 - return False
36 + return (isinstance(block_tables, dict)
37 + and all(value is None for value in block_tables.values()))
38
39
40 def compute_slot_mapping_start_idx(is_prompt: bool, query_len: int,
vllm/core/block/prefix_caching_block.py
+1 -3
@@ -417,9 +417,7 @@ class PrefixCachingBlockAllocator(BlockAllocator):
417
418 def is_block_cached(self, block: Block) -> bool:
419 assert block.content_hash is not None
420 - if block.content_hash in self._cached_blocks:
421 - return True
422 - return False
420 + return block.content_hash in self._cached_blocks
421
422 def promote_to_immutable_block(self, block: Block) -> BlockId:
423 """Once a mutable block is full, it can be promoted to an immutable
vllm/core/block_manager_v2.py
+1 -3
@@ -399,9 +399,7 @@ class BlockSpaceManagerV2(BlockSpaceManager):
399 """
400 alloc_status = self._can_swap(seq_group, Device.CPU,
401 SequenceStatus.RUNNING)
402 - if alloc_status == AllocStatus.OK:
403 - return True
404 - return False
402 + return alloc_status == AllocStatus.OK
403
404 def swap_out(self, seq_group: SequenceGroup) -> List[Tuple[int, int]]:
405 """Returns the block id mapping (from GPU to CPU) generated by
vllm/engine/async_llm_engine.py
+3 -3
@@ -826,7 +826,7 @@ class AsyncLLMEngine:
826 request_id: The unique id of the request.
827 lora_request: LoRA request to use for generation, if any.
828 trace_headers: OpenTelemetry trace headers.
829 - prompt_adapter_request: Prompt Adapter request to use
829 + prompt_adapter_request: Prompt Adapter request to use
830 for generation, if any.
831
832 Yields:
@@ -1042,7 +1042,7 @@ class AsyncLLMEngine:
1042 async def start_profile(self) -> None:
1043 # using type instead of isinstance to check to avoid capturing
1044 # inherited classes
1045 - if type(self.engine.model_executor) == GPUExecutorAsync:
1045 + if type(self.engine.model_executor) == GPUExecutorAsync: # noqa: E721
1046 self.engine.model_executor.start_profile()
1047 else:
1048 self.engine.model_executor._run_workers("start_profile")
@@ -1050,7 +1050,7 @@ class AsyncLLMEngine:
1050 async def stop_profile(self) -> None:
1051 # using type instead of isinstance to check to avoid capturing
1052 # inherited classes
1053 - if type(self.engine.model_executor) == GPUExecutorAsync:
1053 + if type(self.engine.model_executor) == GPUExecutorAsync: # noqa: E721
1054 self.engine.model_executor.stop_profile()
1055 else:
1056 self.engine.model_executor._run_workers("stop_profile")
vllm/engine/llm_engine.py
+3 -3
@@ -144,7 +144,7 @@ class LLMEngine:
144 decoding.
145 executor_class: The model executor class for managing distributed
146 execution.
147 - prompt_adapter_config (Optional): The configuration related to serving
147 + prompt_adapter_config (Optional): The configuration related to serving
148 prompt adapters.
149 log_stats: Whether to log statistics.
150 usage_context: Specified entry point, used for usage info collection.
@@ -1605,7 +1605,7 @@ class LLMEngine:
1605 def start_profile(self) -> None:
1606 # using type instead of isinstance to check to avoid capturing
1607 # inherited classes (MultiprocessingGPUExecutor)
1608 - if type(self.model_executor) == GPUExecutor:
1608 + if type(self.model_executor) == GPUExecutor: # noqa: E721
1609 self.model_executor.start_profile()
1610 else:
1611 self.model_executor._run_workers("start_profile")
@@ -1613,7 +1613,7 @@ class LLMEngine:
1613 def stop_profile(self) -> None:
1614 # using type instead of isinstance to check to avoid capturing
1615 # inherited classes (MultiprocessingGPUExecutor)
1616 - if type(self.model_executor) == GPUExecutor:
1616 + if type(self.model_executor) == GPUExecutor: # noqa: E721
1617 self.model_executor.stop_profile()
1618 else:
1619 self.model_executor._run_workers("stop_profile")
vllm/model_executor/guided_decoding/outlines_logits_processors.py
+2 -2
@@ -67,9 +67,9 @@ class BaseLogitsProcessor:
67 instruction = self._guide.get_next_instruction(
68 state=self._fsm_state[seq_id])
69
70 - if type(instruction) == Generate:
70 + if type(instruction) == Generate: # noqa: E721
71 allowed_tokens = instruction.tokens
72 - elif type(instruction) == Write:
72 + elif type(instruction) == Write: # noqa: E721
73 # TODO: support fast forward tokens
74 allowed_tokens = [instruction.tokens[0]]
75 else:
vllm/model_executor/layers/quantization/awq_marlin.py
+3 -3
@@ -110,9 +110,9 @@ class AWQMarlinConfig(QuantizationConfig):
110 def is_awq_marlin_compatible(cls, quant_config: Dict[str, Any]):
111 # Extract data from quant config.
112 quant_method = quant_config.get("quant_method", "").lower()
113 - num_bits = quant_config.get("bits", None)
114 - group_size = quant_config.get("group_size", None)
115 - has_zp = quant_config.get("zero_point", None)
113 + num_bits = quant_config.get("bits")
114 + group_size = quant_config.get("group_size")
115 + has_zp = quant_config.get("zero_point")
116
117 if quant_method != "awq":
118 return False
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py
+7 -7
@@ -1,4 +1,4 @@
1 -from typing import Any, Dict, List, Optional
1 +from typing import Any, Dict, List, Optional, cast
2
3 import torch
4 from pydantic import BaseModel
@@ -79,8 +79,8 @@ class CompressedTensorsConfig(QuantizationConfig):
79 @classmethod
80 def from_config(cls, config: Dict[str, Any]) -> "CompressedTensorsConfig":
81 target_scheme_map: Dict[str, Any] = dict()
82 - ignore: List[str] = config.get("ignore", None)
83 - quant_format: str = config.get("format", None)
82 + ignore = cast(List[str], config.get("ignore"))
83 + quant_format = cast(str, config.get("format"))
84
85 # The quant_config has multiple config_groups, each containing
86 # an input_activations key with details about how the activations are
@@ -200,7 +200,7 @@ class CompressedTensorsConfig(QuantizationConfig):
200 is_per_tensor_or_channel_weight = (weight_quant.strategy in [
201 QuantizationStrategy.TENSOR, QuantizationStrategy.CHANNEL
202 ])
203 - if not (is_symmetric_weight and is_static_weight
203 + if not (is_symmetric_weight and is_static_weight # noqa: SIM103
204 and is_per_tensor_or_channel_weight):
205 return False
206
@@ -333,7 +333,7 @@ class CompressedTensorsLinearMethod(LinearMethodBase):
333 output_size: int, params_dtype: torch.dtype,
334 **extra_weight_attrs):
335 """
336 - Use the CompressedTensorsScheme associated with each layer to create
336 + Use the CompressedTensorsScheme associated with each layer to create
337 the necessary parameters for the layer. See LinearMethodBase for param
338 details
339 """
@@ -352,8 +352,8 @@ class CompressedTensorsLinearMethod(LinearMethodBase):
352 x: torch.Tensor,
353 bias: Optional[torch.Tensor] = None):
354 """
355 - Use the output of create_weights and the CompressedTensorsScheme
356 - associated with the layer to apply the forward pass with the
355 + Use the output of create_weights and the CompressedTensorsScheme
356 + associated with the layer to apply the forward pass with the
357 layer input. See LinearMethodBase for param details
358
359 """
vllm/model_executor/layers/quantization/gptq_marlin.py
+4 -4
@@ -132,10 +132,10 @@ class GPTQMarlinConfig(QuantizationConfig):
132 def is_gptq_marlin_compatible(cls, quant_config: Dict[str, Any]):
133 # Extract data from quant config.
134 quant_method = quant_config.get("quant_method", "").lower()
135 - num_bits = quant_config.get("bits", None)
136 - group_size = quant_config.get("group_size", None)
137 - sym = quant_config.get("sym", None)
138 - desc_act = quant_config.get("desc_act", None)
135 + num_bits = quant_config.get("bits")
136 + group_size = quant_config.get("group_size")
137 + sym = quant_config.get("sym")
138 + desc_act = quant_config.get("desc_act")
139
140 if quant_method != "gptq":
141 return False
vllm/model_executor/model_loader/tensorizer.py
+1 -3
@@ -408,9 +408,7 @@ def is_vllm_tensorized(tensorizer_config: "TensorizerConfig") -> bool:
408 "inferred as vLLM models, so setting vllm_tensorized=True is "
409 "only necessary for models serialized prior to this change.")
410 return True
411 - if (".vllm_tensorized_marker" in deserializer):
412 - return True
413 - return False
411 + return ".vllm_tensorized_marker" in deserializer
412
413
414 def serialize_vllm_model(
vllm/model_executor/models/minicpmv.py
+1 -1
@@ -884,7 +884,7 @@ class MiniCPMV(MiniCPMVBaseModel):
884 version = str(config.version).split(".")
885 version = tuple([int(x) for x in version])
886 # Dispatch class based on version
887 - instance_class = _SUPPORT_VERSION.get(version, None)
887 + instance_class = _SUPPORT_VERSION.get(version)
888 if instance_class is None:
889 raise ValueError(
890 "Currently, MiniCPMV only supports versions 2.0, 2.5, and 2.6")
vllm/spec_decode/draft_model_runner.py
+1 -4
@@ -183,10 +183,7 @@ class TP1DraftModelRunner(ModelRunner):
183 return False
184
185 # TODO: Add soft-tuning prompt adapter support
186 - if self.prompt_adapter_config:
187 - return False
188 -
189 - return True
186 + return not self.prompt_adapter_config
187
188 @torch.inference_mode()
189 def execute_model(
vllm/spec_decode/metrics.py
+2 -5
@@ -104,13 +104,10 @@ class AsyncMetricsCollector:
104 if self._rank != 0:
105 return False
106
107 - if (now - self._last_metrics_collect_time <
108 - self._rejsample_metrics_collect_interval_s):
109 - return False
110 - return True
107 + return now - self._last_metrics_collect_time >= self._rejsample_metrics_collect_interval_s # noqa: E501
108
109 def _copy_rejsample_metrics_async(self) -> torch.cuda.Event:
113 - """Copy rejection/typical-acceptance sampling metrics
110 + """Copy rejection/typical-acceptance sampling metrics
111 (number of accepted tokens, etc) to CPU asynchronously.
112
113 Returns a CUDA event recording when the copy is complete.
vllm/triton_utils/libentry.py
+2 -2
@@ -35,8 +35,8 @@ class LibEntry(triton.KernelInterface):
35 dns_key = [
36 arg.dtype if hasattr(
37 arg, "data_ptr") else type(arg) if not isinstance(arg, int)
38 - else "i32" if -(2**31) <= arg and arg <= 2**31 -
39 - 1 else "u64" if 2**63 <= arg and arg <= 2**64 - 1 else "i64"
38 + else "i32" if arg >= -(2**31) and arg <= 2**31 -
39 + 1 else "u64" if arg >= 2**63 and arg <= 2**64 - 1 else "i64"
40 for arg in dns_args
41 ]
42 # const args passed by position