@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / bd46357ad9
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 08:15 UTC

[bugfix] fix broken tests of mlp speculator (#10177)

Signed-off-by: youkaichao <youkaichao@gmail.com>

youkaichao committed Nov 9, 2024 at 00:04 UTC bd46357ad90fdb4263a3155c358d37d32dab127c
1 file changed +3 -2
vllm/model_executor/models/mlp_speculator.py
+3 -2
@@ -4,13 +4,13 @@ from typing import Iterable, List, Tuple
4 import torch
5 import torch.nn as nn
6
7 +from vllm.config import VllmConfig
8 from vllm.model_executor import SamplingMetadata
9 from vllm.model_executor.layers.logits_processor import LogitsProcessor
10 from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
11 from vllm.model_executor.layers.vocab_parallel_embedding import (
12 ParallelLMHead, VocabParallelEmbedding)
13 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
13 -from vllm.transformers_utils.configs import MLPSpeculatorConfig
14
15 SQRT2 = 2**0.5
16
@@ -65,8 +65,9 @@ class MLPSpeculator(nn.Module):
65 https://huggingface.co/ibm-fms and https://huggingface.co/ibm-granite
66 """
67
68 - def __init__(self, config: MLPSpeculatorConfig, **kwargs) -> None:
68 + def __init__(self, vllm_config: VllmConfig, prefix: str = "") -> None:
69 super().__init__()
70 + config = vllm_config.model_config.hf_config
71 self.n_predict = config.n_predict
72 self.vocab_size = config.vocab_size
73 self.emb_dim = config.emb_dim