A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[torch.compile] expanding support and fix allgather compilation (#9637)
Signed-off-by: youkaichao <youkaichao@gmail.com> Co-authored-by: youkaichao <youkaichao@gmail.com>
Yongzao committed
Oct 24, 2024 at 16:32 UTC
ad6f78053ed33b2386713b574976523858a879b5
6 files changed
+16
-1
vllm/distributed/parallel_state.py
+6
-1
@@ -392,8 +392,12 @@ class GroupCoordinator:
392
# Convert negative dim to positive.
393
dim += input_.dim()
394
input_size = input_.size()
395
+ # NOTE: we have to use concat-style all-gather here,
396
+ # stack-style all-gather has compatibility issues with
397
+ # torch.compile . see https://github.com/pytorch/pytorch/issues/138795
398
+ output_size = (input_size[0] * world_size, ) + input_size[1:]
399
# Allocate output tensor.
396
- output_tensor = torch.empty((world_size, ) + input_size,
400
+ output_tensor = torch.empty(output_size,
401
dtype=input_.dtype,
402
device=input_.device)
403
# All-gather.
@@ -401,6 +405,7 @@ class GroupCoordinator:
405
input_,
406
group=self.device_group)
407
# Reshape
408
+ output_tensor = output_tensor.reshape((world_size, ) + input_size)
409
output_tensor = output_tensor.movedim(0, dim)
410
output_tensor = output_tensor.reshape(input_size[:dim] +
411
(world_size *
vllm/model_executor/models/gpt_bigcode.py
+2
@@ -25,6 +25,7 @@ from torch import nn
25
from transformers import GPTBigCodeConfig
26
27
from vllm.attention import Attention, AttentionMetadata
28
+from vllm.compilation.decorators import support_torch_compile
29
from vllm.config import CacheConfig, LoRAConfig
30
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
31
from vllm.model_executor.layers.activation import get_act_fn
@@ -187,6 +188,7 @@ class GPTBigCodeBlock(nn.Module):
188
return hidden_states
189
190
191
+@support_torch_compile
192
class GPTBigCodeModel(nn.Module):
193
194
def __init__(
vllm/model_executor/models/gpt_j.py
+2
@@ -23,6 +23,7 @@ from torch import nn
23
from transformers import GPTJConfig
24
25
from vllm.attention import Attention, AttentionMetadata
26
+from vllm.compilation.decorators import support_torch_compile
27
from vllm.config import CacheConfig
28
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
29
from vllm.model_executor.layers.activation import get_act_fn
@@ -174,6 +175,7 @@ class GPTJBlock(nn.Module):
175
return hidden_states
176
177
178
+@support_torch_compile
179
class GPTJModel(nn.Module):
180
181
def __init__(
vllm/model_executor/models/gpt_neox.py
+2
@@ -23,6 +23,7 @@ from torch import nn
23
from transformers import GPTNeoXConfig
24
25
from vllm.attention import Attention, AttentionMetadata
26
+from vllm.compilation.decorators import support_torch_compile
27
from vllm.config import CacheConfig
28
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
29
from vllm.model_executor.layers.activation import get_act_fn
@@ -187,6 +188,7 @@ class GPTNeoXLayer(nn.Module):
188
return hidden_states
189
190
191
+@support_torch_compile
192
class GPTNeoXModel(nn.Module):
193
194
def __init__(
vllm/model_executor/models/granite.py
+2
@@ -28,6 +28,7 @@ from torch import nn
28
from transformers import GraniteConfig
29
30
from vllm.attention import Attention, AttentionMetadata
31
+from vllm.compilation.decorators import support_torch_compile
32
from vllm.config import CacheConfig, LoRAConfig
33
from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank,
34
get_tensor_model_parallel_world_size)
@@ -254,6 +255,7 @@ class GraniteDecoderLayer(nn.Module):
255
return hidden_states
256
257
258
+@support_torch_compile
259
class GraniteModel(nn.Module):
260
261
def __init__(
vllm/model_executor/models/internlm2.py
+2
@@ -7,6 +7,7 @@ from torch import nn
7
from transformers import PretrainedConfig
8
9
from vllm.attention import Attention, AttentionMetadata
10
+from vllm.compilation.decorators import support_torch_compile
11
from vllm.config import CacheConfig
12
from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank,
13
get_tensor_model_parallel_world_size,
@@ -230,6 +231,7 @@ class InternLMDecoderLayer(nn.Module):
231
return hidden_states, residual
232
233
234
+@support_torch_compile
235
class InternLM2Model(nn.Module):
236
237
def __init__(