A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[mypy] Fix incorrect type hint for EAGLE3 support (#23617)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Aug 26, 2025 at 14:50 UTC
7d67a9d9f93f86b74066c64c373405aa088e4a16
3 files changed
+8
-8
vllm/model_executor/models/llama.py
+3
-3
@@ -353,7 +353,7 @@ class LlamaModel(nn.Module):
353
else:
354
self.norm = PPMissingLayer()
355
356
- self.aux_hidden_state_layers: tuple[int] = tuple()
356
+ self.aux_hidden_state_layers = tuple[int, ...]()
357
358
self.make_empty_intermediate_tensors = (
359
make_empty_intermediate_tensors_factory(
@@ -553,10 +553,10 @@ class LlamaForCausalLM(nn.Module, SupportsLoRA, SupportsPP, SupportsEagle3):
553
self.make_empty_intermediate_tensors = (
554
self.model.make_empty_intermediate_tensors)
555
556
- def set_aux_hidden_state_layers(self, layers: tuple[int]) -> None:
556
+ def set_aux_hidden_state_layers(self, layers: tuple[int, ...]) -> None:
557
self.model.aux_hidden_state_layers = layers
558
559
- def get_eagle3_aux_hidden_state_layers(self) -> tuple[int]:
559
+ def get_eagle3_aux_hidden_state_layers(self) -> tuple[int, ...]:
560
num_layers = len(self.model.layers)
561
return (2, num_layers // 2, num_layers - 3)
562
vllm/model_executor/models/qwen2.py
+3
-3
@@ -333,7 +333,7 @@ class Qwen2Model(nn.Module):
333
else:
334
self.norm = PPMissingLayer()
335
336
- self.aux_hidden_state_layers: tuple[int] = tuple()
336
+ self.aux_hidden_state_layers = tuple[int, ...]()
337
338
def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
339
return self.embed_tokens(input_ids)
@@ -488,10 +488,10 @@ class Qwen2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP, SupportsEagle3):
488
def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
489
return self.model.get_input_embeddings(input_ids)
490
491
- def set_aux_hidden_state_layers(self, layers: tuple[int]) -> None:
491
+ def set_aux_hidden_state_layers(self, layers: tuple[int, ...]) -> None:
492
self.model.aux_hidden_state_layers = layers
493
494
- def get_eagle3_aux_hidden_state_layers(self) -> tuple[int]:
494
+ def get_eagle3_aux_hidden_state_layers(self) -> tuple[int, ...]:
495
num_layers = len(self.model.layers)
496
return (2, num_layers // 2, num_layers - 3)
497
vllm/model_executor/models/qwen3.py
+2
-2
@@ -304,10 +304,10 @@ class Qwen3ForCausalLM(nn.Module, SupportsLoRA, SupportsPP, SupportsEagle3):
304
self.make_empty_intermediate_tensors = (
305
self.model.make_empty_intermediate_tensors)
306
307
- def set_aux_hidden_state_layers(self, layers: tuple[int]) -> None:
307
+ def set_aux_hidden_state_layers(self, layers: tuple[int, ...]) -> None:
308
self.model.aux_hidden_state_layers = layers
309
310
- def get_eagle3_aux_hidden_state_layers(self) -> tuple[int]:
310
+ def get_eagle3_aux_hidden_state_layers(self) -> tuple[int, ...]:
311
num_layers = len(self.model.layers)
312
return (2, num_layers // 2, num_layers - 3)
313