A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 5, 23:30 UTC
[Model] Add torch.compile support for InternVL vision encoder (#38049)
Signed-off-by: tianrengao <terrygao87@gmail.com>
Terry Gao committed
Mar 25, 2026 at 23:52 UTC
38de822310239a50c610a420e2ed9f6dd670d5a7
2 files changed
+20
-3
vllm/config/utils.py
+9
-1
@@ -296,7 +296,15 @@ def normalize_value(x):
296
297
# PretrainedConfig
298
if hasattr(x, "to_json_string") and callable(x.to_json_string):
299
- return x.to_json_string()
299
+ try:
300
+ return x.to_json_string()
301
+ except (TypeError, ValueError):
302
+ # to_json_string() may fail for trust-remote-code configs
303
+ # with non-JSON-serializable nested objects. Fall back to
304
+ # normalizing the dict representation recursively.
305
+ if hasattr(x, "to_dict") and callable(x.to_dict):
306
+ return normalize_value(x.to_dict())
307
+ raise
308
309
# Unsupported type: e.g., modules, generators, open files, or objects
310
# without a stable JSON/UUID representation. Hard-error to avoid
vllm/model_executor/models/intern_vit.py
+11
-2
@@ -15,6 +15,10 @@ import torch.nn as nn
15
import torch.nn.functional as F
16
from transformers import PretrainedConfig
17
18
+from vllm.compilation.decorators import (
19
+ should_torch_compile_mm_encoder,
20
+ support_torch_compile,
21
+)
22
from vllm.distributed import (
23
divide,
24
get_tensor_model_parallel_rank,
@@ -280,6 +284,11 @@ class InternMLP(nn.Module):
284
return hidden_states
285
286
287
+@support_torch_compile(
288
+ dynamic_arg_dims={"hidden_states": 0},
289
+ enable_if=should_torch_compile_mm_encoder,
290
+ is_encoder=True,
291
+)
292
class InternVisionEncoderLayer(nn.Module):
293
def __init__(
294
self,
@@ -364,8 +373,8 @@ class InternVisionEncoder(nn.Module):
373
self.layers = nn.ModuleList(
374
[
375
self.layer_cls(
367
- config,
368
- quant_config,
376
+ config=config,
377
+ quant_config=quant_config,
378
num_dummy_heads=num_dummy_heads,
379
prefix=f"{prefix}.layers.{layer_idx}",
380
)