A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 02:00 UTC
[ci/lint] Add back default arg for pre-commit (#12279)
Signed-off-by: kevin <kevin@anyscale.com>
Kevin H. Luu committed
Jan 21, 2025 at 17:15 UTC
64ea24d0b3ca794faabc5f1570a921a7b44118b7
6 files changed
+26
-29
.github/workflows/pre-commit.yml
+1
-1
@@ -16,4 +16,4 @@ jobs:
16
- run: echo "::add-matcher::.github/workflows/matchers/actionlint.json"
17
- uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd # v3.0.1
18
with:
19
- extra_args: --hook-stage manual
19
+ extra_args: --all-files --hook-stage manual
tests/models/decoder_only/language/test_gguf.py
+7
-10
@@ -74,11 +74,7 @@ DOLPHIN_CONFIG = GGUFTestConfig(
74
)
75
76
MODELS = [
77
- LLAMA_CONFIG,
78
- QWEN2_CONFIG,
79
- PHI3_CONFIG,
80
- GPT2_CONFIG,
81
- STABLELM_CONFIG,
77
+ LLAMA_CONFIG, QWEN2_CONFIG, PHI3_CONFIG, GPT2_CONFIG, STABLELM_CONFIG,
78
DOLPHIN_CONFIG
79
# STARCODER_CONFIG, # broken
80
]
@@ -114,11 +110,12 @@ def test_models(
110
messages, tokenize=False, add_generation_prompt=True)
111
112
# Run unquantized model.
117
- with vllm_runner(model_name=model.original_model,
118
- enforce_eager=True, # faster tests
119
- dtype=dtype,
120
- max_model_len=MAX_MODEL_LEN,
121
- tensor_parallel_size=tp_size) as original_model:
113
+ with vllm_runner(
114
+ model_name=model.original_model,
115
+ enforce_eager=True, # faster tests
116
+ dtype=dtype,
117
+ max_model_len=MAX_MODEL_LEN,
118
+ tensor_parallel_size=tp_size) as original_model:
119
original_outputs = original_model.generate_greedy_logprobs(
120
example_prompts[:-1], max_tokens, num_logprobs)
121
vllm/model_executor/models/paligemma.py
+1
-1
@@ -147,7 +147,7 @@ class PaliGemmaForConditionalGeneration(nn.Module, SupportsMultiModal,
147
"up_proj",
148
],
149
}
150
-
150
+
151
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
152
super().__init__()
153
config = vllm_config.model_config.hf_config
vllm/model_executor/models/siglip.py
+3
-5
@@ -348,12 +348,10 @@ class SiglipMLP(nn.Module):
348
if quant_config and quant_config.get_name() == "bitsandbytes":
349
quantizable = True
350
else:
351
- # For other quantization, we require the hidden size to be a
351
+ # For other quantization, we require the hidden size to be a
352
# multiple of 64
353
- quantizable = (
354
- config.hidden_size % 64 == 0
355
- and config.intermediate_size % 64 == 0
356
- )
353
+ quantizable = (config.hidden_size % 64 == 0
354
+ and config.intermediate_size % 64 == 0)
355
self.fc1 = ColumnParallelLinear(
356
config.hidden_size,
357
config.intermediate_size,
vllm/platforms/__init__.py
+1
-1
@@ -101,7 +101,7 @@ def cpu_platform_plugin() -> Optional[str]:
101
try:
102
from importlib.metadata import version
103
is_cpu = "cpu" in version("vllm")
104
- if is_cpu == False:
104
+ if not is_cpu:
105
import platform
106
is_cpu = platform.machine().lower().startswith("arm")
107
vllm/v1/stats/common.py
+13
-11
@@ -10,10 +10,11 @@ from msgspec import field as msgspec_field
10
from vllm.sampling_params import SamplingParams
11
12
13
-class RequestStatsUpdate(msgspec.Struct,
14
- array_like=True,
15
- omit_defaults=True,
16
- gc=False):
13
+class RequestStatsUpdate(
14
+ msgspec.Struct, # type: ignore
15
+ array_like=True,
16
+ omit_defaults=True,
17
+ gc=False):
18
"""
19
An update to the request stats.
20
@@ -341,8 +342,8 @@ class RequestStats:
342
self.queued_ts_s = ts
343
elif update.type == RequestStatsUpdate.Type.PREFILLING:
344
self.prefill_start_ts_s_lst.append(ts)
344
- self.num_cached_tokens = update.num_cached_tokens
345
- self.num_computed_tokens = update.num_computed_tokens
345
+ self.num_cached_tokens = update.num_cached_tokens or 0
346
+ self.num_computed_tokens = update.num_computed_tokens or 0
347
elif update.type == RequestStatsUpdate.Type.PREEMPTED:
348
self._reset_for_preemption(ts)
349
elif update.type == RequestStatsUpdate.Type.DECODING:
@@ -350,7 +351,7 @@ class RequestStats:
351
elif update.type == RequestStatsUpdate.Type.DETOKENIZED:
352
self._record_detokenized_output(
353
ts,
353
- update.num_new_tokens,
354
+ update.num_new_tokens or 0,
355
)
356
elif update.type == RequestStatsUpdate.Type.FINISHED:
357
self.finished_ts_s = ts
@@ -425,10 +426,11 @@ class EngineCoreProcessStats:
426
output_queue_size: Optional[int] = None
427
428
428
-class EngineCoreStatsSnapshot(msgspec.Struct,
429
- array_like=True,
430
- omit_defaults=True,
431
- gc=False):
429
+class EngineCoreStatsSnapshot(
430
+ msgspec.Struct, # type: ignore
431
+ array_like=True,
432
+ omit_defaults=True,
433
+ gc=False):
434
"""
435
A snapshot of the EngineCore's current stats over a period of time.
436
"""