A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 10:45 UTC
[Bugfix] Fix Fuyu tensor parallel inference (#8986)
Isotr0py committed
Oct 1, 2024 at 17:51 UTC
bc4eb65b5492b4f84a1b714bfc14bcff73d401f1
3 files changed
+15
-12
tests/distributed/test_pipeline_parallel.py
+3
-1
@@ -37,7 +37,9 @@ VLLM_MULTI_NODE = os.getenv("VLLM_MULTI_NODE", "0") == "1"
37
(1, 2, 1, 1, 1, "OpenGVLab/InternVL2-1B", "mp"),
38
(1, 2, 1, 1, 1, "OpenGVLab/InternVL2-2B", "mp"),
39
(1, 2, 1, 0, 1, "OpenGVLab/InternVL2-4B", "mp"),
40
- (1, 2, 0, 1, 0, "Qwen/Qwen2-VL-2B-Instruct", "mp")
40
+ (1, 2, 0, 1, 0, "Qwen/Qwen2-VL-2B-Instruct", "mp"),
41
+ # TP only models
42
+ (2, 1, 1, 0, 0, "adept/fuyu-8b", "mp"),
43
],
44
)
45
@fork_new_process_for_each_test
vllm/model_executor/models/fuyu.py
+2
-1
@@ -237,8 +237,9 @@ class FuyuForCausalLM(nn.Module, SupportsMultiModal):
237
self.image_feature_size,
238
config.hidden_size,
239
quant_config=quant_config,
240
+ gather_output=True,
241
)
241
- self.language_model = PersimmonForCausalLM(config,
242
+ self.language_model = PersimmonForCausalLM(config.text_config,
243
cache_config=cache_config,
244
quant_config=quant_config)
245
vllm/model_executor/models/persimmon.py
+10
-10
@@ -25,11 +25,11 @@ from typing import Iterable, List, Optional, Tuple
25
import torch
26
from torch import nn
27
from transformers import PersimmonConfig
28
-from transformers.activations import ReLUSquaredActivation
28
29
from vllm.attention import Attention, AttentionMetadata
30
from vllm.config import CacheConfig
31
from vllm.distributed import get_tensor_model_parallel_world_size
32
+from vllm.model_executor.layers.activation import get_act_fn
33
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
34
QKVParallelLinear,
35
RowParallelLinear)
@@ -57,7 +57,7 @@ class PersimmonMLP(nn.Module):
57
self.dense_4h_to_h = RowParallelLinear(config.intermediate_size,
58
config.hidden_size,
59
quant_config=quant_config)
60
- self.act = ReLUSquaredActivation()
60
+ self.act = get_act_fn(config.hidden_act, quant_config)
61
62
def forward(self, hidden_states) -> torch.Tensor:
63
hidden_states, _ = self.dense_h_to_4h(hidden_states)
@@ -96,7 +96,7 @@ class PersimmonAttention(nn.Module):
96
quant_config=quant_config,
97
)
98
self.dense = RowParallelLinear(
99
- self.num_heads * self.head_dim,
99
+ self.total_num_heads * self.head_dim,
100
self.hidden_size,
101
bias=True,
102
quant_config=quant_config,
@@ -213,10 +213,10 @@ class PersimmonModel(nn.Module):
213
cache_config: Optional[CacheConfig] = None,
214
quant_config: Optional[QuantizationConfig] = None):
215
super().__init__()
216
- self.vocab_size = config.text_config.vocab_size
216
+ self.vocab_size = config.vocab_size
217
218
- self.embed_tokens = VocabParallelEmbedding(
219
- config.text_config.vocab_size, config.hidden_size)
218
+ self.embed_tokens = VocabParallelEmbedding(config.vocab_size,
219
+ config.hidden_size)
220
self.layers = nn.ModuleList([
221
PersimmonDecoderLayer(config,
222
cache_config=cache_config,
@@ -252,19 +252,19 @@ class PersimmonModel(nn.Module):
252
class PersimmonForCausalLM(nn.Module):
253
254
def __init__(self,
255
- config,
255
+ config: PersimmonConfig,
256
cache_config: Optional[CacheConfig] = None,
257
quant_config: Optional[QuantizationConfig] = None):
258
super().__init__()
259
self.config = config
260
- self.vocab_size = config.text_config.vocab_size
260
+ self.vocab_size = config.vocab_size
261
self.model = PersimmonModel(config,
262
cache_config=cache_config,
263
quant_config=quant_config)
264
- self.lm_head = ParallelLMHead(config.text_config.vocab_size,
264
+ self.lm_head = ParallelLMHead(config.vocab_size,
265
config.hidden_size,
266
bias=False)
267
- self.logits_processor = LogitsProcessor(config.text_config.vocab_size)
267
+ self.logits_processor = LogitsProcessor(config.vocab_size)
268
self.sampler = Sampler()
269
270
def forward(