A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
Upgrade transformers version to 4.36.0 (#2046)
Woosuk Kwon committed
Dec 11, 2023 at 18:39 UTC
cb3f30c600169210f9715f084e34adf2afc4f7d7
3 files changed
+5
-5
requirements-rocm.txt
+1
-1
@@ -10,7 +10,7 @@ numpy
10
tokenizers>=0.15.0
11
huggingface_hub<0.18,>=0.16.4
12
einops # Required for phi-1_5
13
-transformers >= 4.34.0 # Required for Mistral.
13
+transformers >= 4.36.0 # Required for Mixtral.
14
fastapi
15
uvicorn[standard]
16
pydantic == 1.10.13 # Required for OpenAI server.
requirements.txt
+1
-1
@@ -7,7 +7,7 @@ sentencepiece # Required for LLaMA tokenizer.
7
numpy
8
einops # Required for phi-1_5
9
torch >= 2.1.1
10
-transformers >= 4.34.0 # Required for Mistral.
10
+transformers >= 4.36.0 # Required for Mixtral.
11
xformers >= 0.0.23 # Required for CUDA 12.1.
12
fastapi
13
uvicorn[standard]
vllm/model_executor/models/mixtral.py
+3
-3
@@ -29,7 +29,7 @@ import torch
29
import torch.nn.functional as F
30
31
from torch import nn
32
-from transformers import MistralConfig
32
+from transformers import MixtralConfig
33
34
try:
35
import megablocks.ops as ops
@@ -395,7 +395,7 @@ class MixtralDecoderLayer(nn.Module):
395
396
def __init__(
397
self,
398
- config: MistralConfig,
398
+ config: MixtralConfig,
399
) -> None:
400
super().__init__()
401
self.hidden_size = config.hidden_size
@@ -443,7 +443,7 @@ class MixtralForCausalLM(nn.Module):
443
444
def __init__(
445
self,
446
- config: MistralConfig,
446
+ config: MixtralConfig,
447
linear_method: Optional[LinearMethodBase] = None,
448
) -> None:
449
super().__init__()