@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / bc4eb65b54
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 10:45 UTC

[Bugfix] Fix Fuyu tensor parallel inference (#8986)

Isotr0py committed Oct 1, 2024 at 17:51 UTC bc4eb65b5492b4f84a1b714bfc14bcff73d401f1
3 files changed +15 -12
tests/distributed/test_pipeline_parallel.py
+3 -1
@@ -37,7 +37,9 @@ VLLM_MULTI_NODE = os.getenv("VLLM_MULTI_NODE", "0") == "1"
37 (1, 2, 1, 1, 1, "OpenGVLab/InternVL2-1B", "mp"),
38 (1, 2, 1, 1, 1, "OpenGVLab/InternVL2-2B", "mp"),
39 (1, 2, 1, 0, 1, "OpenGVLab/InternVL2-4B", "mp"),
40 - (1, 2, 0, 1, 0, "Qwen/Qwen2-VL-2B-Instruct", "mp")
40 + (1, 2, 0, 1, 0, "Qwen/Qwen2-VL-2B-Instruct", "mp"),
41 + # TP only models
42 + (2, 1, 1, 0, 0, "adept/fuyu-8b", "mp"),
43 ],
44 )
45 @fork_new_process_for_each_test
vllm/model_executor/models/fuyu.py
+2 -1
@@ -237,8 +237,9 @@ class FuyuForCausalLM(nn.Module, SupportsMultiModal):
237 self.image_feature_size,
238 config.hidden_size,
239 quant_config=quant_config,
240 + gather_output=True,
241 )
241 - self.language_model = PersimmonForCausalLM(config,
242 + self.language_model = PersimmonForCausalLM(config.text_config,
243 cache_config=cache_config,
244 quant_config=quant_config)
245
vllm/model_executor/models/persimmon.py
+10 -10
@@ -25,11 +25,11 @@ from typing import Iterable, List, Optional, Tuple
25 import torch
26 from torch import nn
27 from transformers import PersimmonConfig
28 -from transformers.activations import ReLUSquaredActivation
28
29 from vllm.attention import Attention, AttentionMetadata
30 from vllm.config import CacheConfig
31 from vllm.distributed import get_tensor_model_parallel_world_size
32 +from vllm.model_executor.layers.activation import get_act_fn
33 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
34 QKVParallelLinear,
35 RowParallelLinear)
@@ -57,7 +57,7 @@ class PersimmonMLP(nn.Module):
57 self.dense_4h_to_h = RowParallelLinear(config.intermediate_size,
58 config.hidden_size,
59 quant_config=quant_config)
60 - self.act = ReLUSquaredActivation()
60 + self.act = get_act_fn(config.hidden_act, quant_config)
61
62 def forward(self, hidden_states) -> torch.Tensor:
63 hidden_states, _ = self.dense_h_to_4h(hidden_states)
@@ -96,7 +96,7 @@ class PersimmonAttention(nn.Module):
96 quant_config=quant_config,
97 )
98 self.dense = RowParallelLinear(
99 - self.num_heads * self.head_dim,
99 + self.total_num_heads * self.head_dim,
100 self.hidden_size,
101 bias=True,
102 quant_config=quant_config,
@@ -213,10 +213,10 @@ class PersimmonModel(nn.Module):
213 cache_config: Optional[CacheConfig] = None,
214 quant_config: Optional[QuantizationConfig] = None):
215 super().__init__()
216 - self.vocab_size = config.text_config.vocab_size
216 + self.vocab_size = config.vocab_size
217
218 - self.embed_tokens = VocabParallelEmbedding(
219 - config.text_config.vocab_size, config.hidden_size)
218 + self.embed_tokens = VocabParallelEmbedding(config.vocab_size,
219 + config.hidden_size)
220 self.layers = nn.ModuleList([
221 PersimmonDecoderLayer(config,
222 cache_config=cache_config,
@@ -252,19 +252,19 @@ class PersimmonModel(nn.Module):
252 class PersimmonForCausalLM(nn.Module):
253
254 def __init__(self,
255 - config,
255 + config: PersimmonConfig,
256 cache_config: Optional[CacheConfig] = None,
257 quant_config: Optional[QuantizationConfig] = None):
258 super().__init__()
259 self.config = config
260 - self.vocab_size = config.text_config.vocab_size
260 + self.vocab_size = config.vocab_size
261 self.model = PersimmonModel(config,
262 cache_config=cache_config,
263 quant_config=quant_config)
264 - self.lm_head = ParallelLMHead(config.text_config.vocab_size,
264 + self.lm_head = ParallelLMHead(config.vocab_size,
265 config.hidden_size,
266 bias=False)
267 - self.logits_processor = LogitsProcessor(config.text_config.vocab_size)
267 + self.logits_processor = LogitsProcessor(config.vocab_size)
268 self.sampler = Sampler()
269
270 def forward(