@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 64ea24d0b3
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 02:00 UTC

[ci/lint] Add back default arg for pre-commit (#12279)

Signed-off-by: kevin <kevin@anyscale.com>

Kevin H. Luu committed Jan 21, 2025 at 17:15 UTC 64ea24d0b3ca794faabc5f1570a921a7b44118b7
6 files changed +26 -29
.github/workflows/pre-commit.yml
+1 -1
@@ -16,4 +16,4 @@ jobs:
16 - run: echo "::add-matcher::.github/workflows/matchers/actionlint.json"
17 - uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd # v3.0.1
18 with:
19 - extra_args: --hook-stage manual
19 + extra_args: --all-files --hook-stage manual
tests/models/decoder_only/language/test_gguf.py
+7 -10
@@ -74,11 +74,7 @@ DOLPHIN_CONFIG = GGUFTestConfig(
74 )
75
76 MODELS = [
77 - LLAMA_CONFIG,
78 - QWEN2_CONFIG,
79 - PHI3_CONFIG,
80 - GPT2_CONFIG,
81 - STABLELM_CONFIG,
77 + LLAMA_CONFIG, QWEN2_CONFIG, PHI3_CONFIG, GPT2_CONFIG, STABLELM_CONFIG,
78 DOLPHIN_CONFIG
79 # STARCODER_CONFIG, # broken
80 ]
@@ -114,11 +110,12 @@ def test_models(
110 messages, tokenize=False, add_generation_prompt=True)
111
112 # Run unquantized model.
117 - with vllm_runner(model_name=model.original_model,
118 - enforce_eager=True, # faster tests
119 - dtype=dtype,
120 - max_model_len=MAX_MODEL_LEN,
121 - tensor_parallel_size=tp_size) as original_model:
113 + with vllm_runner(
114 + model_name=model.original_model,
115 + enforce_eager=True, # faster tests
116 + dtype=dtype,
117 + max_model_len=MAX_MODEL_LEN,
118 + tensor_parallel_size=tp_size) as original_model:
119 original_outputs = original_model.generate_greedy_logprobs(
120 example_prompts[:-1], max_tokens, num_logprobs)
121
vllm/model_executor/models/paligemma.py
+1 -1
@@ -147,7 +147,7 @@ class PaliGemmaForConditionalGeneration(nn.Module, SupportsMultiModal,
147 "up_proj",
148 ],
149 }
150 -
150 +
151 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
152 super().__init__()
153 config = vllm_config.model_config.hf_config
vllm/model_executor/models/siglip.py
+3 -5
@@ -348,12 +348,10 @@ class SiglipMLP(nn.Module):
348 if quant_config and quant_config.get_name() == "bitsandbytes":
349 quantizable = True
350 else:
351 - # For other quantization, we require the hidden size to be a
351 + # For other quantization, we require the hidden size to be a
352 # multiple of 64
353 - quantizable = (
354 - config.hidden_size % 64 == 0
355 - and config.intermediate_size % 64 == 0
356 - )
353 + quantizable = (config.hidden_size % 64 == 0
354 + and config.intermediate_size % 64 == 0)
355 self.fc1 = ColumnParallelLinear(
356 config.hidden_size,
357 config.intermediate_size,
vllm/platforms/__init__.py
+1 -1
@@ -101,7 +101,7 @@ def cpu_platform_plugin() -> Optional[str]:
101 try:
102 from importlib.metadata import version
103 is_cpu = "cpu" in version("vllm")
104 - if is_cpu == False:
104 + if not is_cpu:
105 import platform
106 is_cpu = platform.machine().lower().startswith("arm")
107
vllm/v1/stats/common.py
+13 -11
@@ -10,10 +10,11 @@ from msgspec import field as msgspec_field
10 from vllm.sampling_params import SamplingParams
11
12
13 -class RequestStatsUpdate(msgspec.Struct,
14 - array_like=True,
15 - omit_defaults=True,
16 - gc=False):
13 +class RequestStatsUpdate(
14 + msgspec.Struct, # type: ignore
15 + array_like=True,
16 + omit_defaults=True,
17 + gc=False):
18 """
19 An update to the request stats.
20
@@ -341,8 +342,8 @@ class RequestStats:
342 self.queued_ts_s = ts
343 elif update.type == RequestStatsUpdate.Type.PREFILLING:
344 self.prefill_start_ts_s_lst.append(ts)
344 - self.num_cached_tokens = update.num_cached_tokens
345 - self.num_computed_tokens = update.num_computed_tokens
345 + self.num_cached_tokens = update.num_cached_tokens or 0
346 + self.num_computed_tokens = update.num_computed_tokens or 0
347 elif update.type == RequestStatsUpdate.Type.PREEMPTED:
348 self._reset_for_preemption(ts)
349 elif update.type == RequestStatsUpdate.Type.DECODING:
@@ -350,7 +351,7 @@ class RequestStats:
351 elif update.type == RequestStatsUpdate.Type.DETOKENIZED:
352 self._record_detokenized_output(
353 ts,
353 - update.num_new_tokens,
354 + update.num_new_tokens or 0,
355 )
356 elif update.type == RequestStatsUpdate.Type.FINISHED:
357 self.finished_ts_s = ts
@@ -425,10 +426,11 @@ class EngineCoreProcessStats:
426 output_queue_size: Optional[int] = None
427
428
428 -class EngineCoreStatsSnapshot(msgspec.Struct,
429 - array_like=True,
430 - omit_defaults=True,
431 - gc=False):
429 +class EngineCoreStatsSnapshot(
430 + msgspec.Struct, # type: ignore
431 + array_like=True,
432 + omit_defaults=True,
433 + gc=False):
434 """
435 A snapshot of the EngineCore's current stats over a period of time.
436 """