A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 08:15 UTC
[bugfix] fix broken tests of mlp speculator (#10177)
Signed-off-by: youkaichao <youkaichao@gmail.com>
youkaichao committed
Nov 9, 2024 at 00:04 UTC
bd46357ad90fdb4263a3155c358d37d32dab127c
1 file changed
+3
-2
vllm/model_executor/models/mlp_speculator.py
+3
-2
@@ -4,13 +4,13 @@ from typing import Iterable, List, Tuple
4
import torch
5
import torch.nn as nn
6
7
+from vllm.config import VllmConfig
8
from vllm.model_executor import SamplingMetadata
9
from vllm.model_executor.layers.logits_processor import LogitsProcessor
10
from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
11
from vllm.model_executor.layers.vocab_parallel_embedding import (
12
ParallelLMHead, VocabParallelEmbedding)
13
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
13
-from vllm.transformers_utils.configs import MLPSpeculatorConfig
14
15
SQRT2 = 2**0.5
16
@@ -65,8 +65,9 @@ class MLPSpeculator(nn.Module):
65
https://huggingface.co/ibm-fms and https://huggingface.co/ibm-granite
66
"""
67
68
- def __init__(self, config: MLPSpeculatorConfig, **kwargs) -> None:
68
+ def __init__(self, vllm_config: VllmConfig, prefix: str = "") -> None:
69
super().__init__()
70
+ config = vllm_config.model_config.hf_config
71
self.n_predict = config.n_predict
72
self.vocab_size = config.vocab_size
73
self.emb_dim = config.emb_dim