@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / ad6f78053e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[torch.compile] expanding support and fix allgather compilation (#9637)

Signed-off-by: youkaichao <youkaichao@gmail.com> Co-authored-by: youkaichao <youkaichao@gmail.com>

Yongzao committed Oct 24, 2024 at 16:32 UTC ad6f78053ed33b2386713b574976523858a879b5
6 files changed +16 -1
vllm/distributed/parallel_state.py
+6 -1
@@ -392,8 +392,12 @@ class GroupCoordinator:
392 # Convert negative dim to positive.
393 dim += input_.dim()
394 input_size = input_.size()
395 + # NOTE: we have to use concat-style all-gather here,
396 + # stack-style all-gather has compatibility issues with
397 + # torch.compile . see https://github.com/pytorch/pytorch/issues/138795
398 + output_size = (input_size[0] * world_size, ) + input_size[1:]
399 # Allocate output tensor.
396 - output_tensor = torch.empty((world_size, ) + input_size,
400 + output_tensor = torch.empty(output_size,
401 dtype=input_.dtype,
402 device=input_.device)
403 # All-gather.
@@ -401,6 +405,7 @@ class GroupCoordinator:
405 input_,
406 group=self.device_group)
407 # Reshape
408 + output_tensor = output_tensor.reshape((world_size, ) + input_size)
409 output_tensor = output_tensor.movedim(0, dim)
410 output_tensor = output_tensor.reshape(input_size[:dim] +
411 (world_size *
vllm/model_executor/models/gpt_bigcode.py
+2
@@ -25,6 +25,7 @@ from torch import nn
25 from transformers import GPTBigCodeConfig
26
27 from vllm.attention import Attention, AttentionMetadata
28 +from vllm.compilation.decorators import support_torch_compile
29 from vllm.config import CacheConfig, LoRAConfig
30 from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
31 from vllm.model_executor.layers.activation import get_act_fn
@@ -187,6 +188,7 @@ class GPTBigCodeBlock(nn.Module):
188 return hidden_states
189
190
191 +@support_torch_compile
192 class GPTBigCodeModel(nn.Module):
193
194 def __init__(
vllm/model_executor/models/gpt_j.py
+2
@@ -23,6 +23,7 @@ from torch import nn
23 from transformers import GPTJConfig
24
25 from vllm.attention import Attention, AttentionMetadata
26 +from vllm.compilation.decorators import support_torch_compile
27 from vllm.config import CacheConfig
28 from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
29 from vllm.model_executor.layers.activation import get_act_fn
@@ -174,6 +175,7 @@ class GPTJBlock(nn.Module):
175 return hidden_states
176
177
178 +@support_torch_compile
179 class GPTJModel(nn.Module):
180
181 def __init__(
vllm/model_executor/models/gpt_neox.py
+2
@@ -23,6 +23,7 @@ from torch import nn
23 from transformers import GPTNeoXConfig
24
25 from vllm.attention import Attention, AttentionMetadata
26 +from vllm.compilation.decorators import support_torch_compile
27 from vllm.config import CacheConfig
28 from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
29 from vllm.model_executor.layers.activation import get_act_fn
@@ -187,6 +188,7 @@ class GPTNeoXLayer(nn.Module):
188 return hidden_states
189
190
191 +@support_torch_compile
192 class GPTNeoXModel(nn.Module):
193
194 def __init__(
vllm/model_executor/models/granite.py
+2
@@ -28,6 +28,7 @@ from torch import nn
28 from transformers import GraniteConfig
29
30 from vllm.attention import Attention, AttentionMetadata
31 +from vllm.compilation.decorators import support_torch_compile
32 from vllm.config import CacheConfig, LoRAConfig
33 from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank,
34 get_tensor_model_parallel_world_size)
@@ -254,6 +255,7 @@ class GraniteDecoderLayer(nn.Module):
255 return hidden_states
256
257
258 +@support_torch_compile
259 class GraniteModel(nn.Module):
260
261 def __init__(
vllm/model_executor/models/internlm2.py
+2
@@ -7,6 +7,7 @@ from torch import nn
7 from transformers import PretrainedConfig
8
9 from vllm.attention import Attention, AttentionMetadata
10 +from vllm.compilation.decorators import support_torch_compile
11 from vllm.config import CacheConfig
12 from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank,
13 get_tensor_model_parallel_world_size,
@@ -230,6 +231,7 @@ class InternLMDecoderLayer(nn.Module):
231 return hidden_states, residual
232
233
234 +@support_torch_compile
235 class InternLM2Model(nn.Module):
236
237 def __init__(