@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 6930a41116

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 11:00 UTC

[V1] Add VLLM_ALLOW_INSECURE_SERIALIZATION env var (#17490)

Signed-off-by: Russell Bryant <rbryant@redhat.com> Signed-off-by: Nick Hill <nhill@redhat.com> Co-authored-by: Nick Hill <nhill@redhat.com>

Russell Bryant committed May 8, 2025 at 01:34 UTC 6930a411163c302381e03685fc117c0d2f6a8e90

4 files changed +170 -115

tests/v1/entrypoints/llm/test_struct_output_generate.py

+50 -34

@@ -105,8 +105,9 @@ def test_structured_output(

105 max_tokens=1000,

106 guided_decoding=GuidedDecodingParams(json=sample_json_schema))

107 outputs = llm.generate(prompts=[

108 - f"Give an example JSON for an employee profile "

109 - f"that fits this schema: {sample_json_schema}"

108 + (f"Give an example JSON for an employee profile that fits this "

109 + f"schema. Make the response as short as possible. Schema: "

110 + f"{sample_json_schema}")

111 ] * 2,

112 sampling_params=sampling_params,

113 use_tqdm=True)

@@ -136,7 +137,8 @@ def test_structured_output(

137

138 outputs = llm.generate(

139 prompts=("Generate a JSON object with curly braces for a person with "

139 - "name and age fields for John Smith who is 31 years old."),

140 + "name and age fields for John Smith who is 31 years old. "

141 + "Make the response as short as possible."),

142 sampling_params=sampling_params,

143 use_tqdm=True)

144

@@ -165,19 +167,20 @@ def test_structured_output(

167 with pytest.raises(ValueError,

168 match="The provided JSON schema contains features "

169 "not supported by xgrammar."):

168 - llm.generate(prompts=[

169 - f"Give an example JSON for an employee profile "

170 - f"that fits this schema: {unsupported_json_schema}"

171 - ] * 2,

172 - sampling_params=sampling_params,

173 - use_tqdm=True)

170 + llm.generate(

171 + prompts=[(f"Give an example JSON for an employee profile that "

172 + f"fits this schema: {unsupported_json_schema}. "

173 + f"Make the response as short as possible.")] * 2,

174 + sampling_params=sampling_params,

175 + use_tqdm=True)

176 else:

175 - outputs = llm.generate(

176 - prompts=("Give an example JSON object for a grade "

177 - "that fits this schema: "

178 - f"{unsupported_json_schema}"),

179 - sampling_params=sampling_params,

180 - use_tqdm=True)

177 + outputs = llm.generate(prompts=(

178 + "Give an example JSON object for a grade "

179 + "that fits this schema: "

180 + f"{unsupported_json_schema}. Make the response as short as "

181 + "possible."),

182 + sampling_params=sampling_params,

183 + use_tqdm=True)

184 assert outputs is not None

185 for output in outputs:

186 assert output is not None

@@ -199,8 +202,10 @@ def test_structured_output(

202 max_tokens=1000,

203 guided_decoding=GuidedDecodingParams(grammar=sample_sql_ebnf))

204 outputs = llm.generate(

202 - prompts=("Generate a sql statement that selects col_1 from "

203 - "table_1 where it is equal to 1"),

205 + prompts=(

206 + "Generate a sql statement that selects col_1 from "

207 + "table_1 where it is equal to 1. Make the response as short as "

208 + "possible."),

209 sampling_params=sampling_params,

210 use_tqdm=True,

211 )

@@ -231,8 +236,10 @@ def test_structured_output(

236 max_tokens=1000,

237 guided_decoding=GuidedDecodingParams(grammar=sample_sql_lark))

238 outputs = llm.generate(

234 - prompts=("Generate a sql statement that selects col_1 from "

235 - "table_1 where it is equal to 1"),

239 + prompts=(

240 + "Generate a sql statement that selects col_1 from "

241 + "table_1 where it is equal to 1. Make the response as short as "

242 + "possible."),

243 sampling_params=sampling_params,

244 use_tqdm=True,

245 )

@@ -269,8 +276,10 @@ def test_structured_output(

276 guided_decoding=GuidedDecodingParams(grammar="not a grammar"))

277 with pytest.raises(ValueError, match="Failed to convert the grammar "):

278 llm.generate(

272 - prompts=("Generate a sql statement that selects col_1 from "

273 - "table_1 where it is equal to 1"),

279 + prompts=(

280 + "Generate a sql statement that selects col_1 from "

281 + "table_1 where it is equal to 1. Make the response as short "

282 + "as possible."),

283 sampling_params=sampling_params,

284 use_tqdm=True,

285 )

@@ -284,7 +293,8 @@ def test_structured_output(

293 guided_decoding=GuidedDecodingParams(regex=sample_regex))

294 outputs = llm.generate(

295 prompts=[

287 - f"Give an example IPv4 address with this regex: {sample_regex}"

296 + (f"Give an example IPv4 address with this regex: {sample_regex}. "

297 + f"Make the response as short as possible.")

298 ] * 2,

299 sampling_params=sampling_params,

300 use_tqdm=True,

@@ -309,7 +319,8 @@ def test_structured_output(

319 top_p=0.95,

320 guided_decoding=GuidedDecodingParams(choice=sample_guided_choice))

321 outputs = llm.generate(

312 - prompts="The best language for type-safe systems programming is ",

322 + prompts=("The best language for type-safe systems programming is "

323 + "(Make the response as short as possible.) "),

324 sampling_params=sampling_params,

325 use_tqdm=True)

326 assert outputs is not None

@@ -331,11 +342,12 @@ def test_structured_output(

342 temperature=1.0,

343 max_tokens=1000,

344 guided_decoding=GuidedDecodingParams(json=json_schema))

334 - outputs = llm.generate(

335 - prompts="Generate a JSON with the brand, model and car_type of"

336 - "the most iconic car from the 90's",

337 - sampling_params=sampling_params,

338 - use_tqdm=True)

345 + outputs = llm.generate(prompts=(

346 + "Generate a JSON with the brand, model and car_type of the most "

347 + "iconic car from the 90's. Make the response as short as "

348 + "possible."),

349 + sampling_params=sampling_params,

350 + use_tqdm=True)

351

352 assert outputs is not None

353

@@ -373,7 +385,8 @@ def test_structured_output(

385 guided_decoding=GuidedDecodingParams(json=json_schema))

386

387 outputs = llm.generate(

376 - prompts="Generate a description of a frog using 50 characters.",

388 + prompts=("Generate a description of a frog using 50 characters. "

389 + "Make the response as short as possible."),

390 sampling_params=sampling_params,

391 use_tqdm=True)

392

@@ -452,7 +465,8 @@ Reminder:

465

466 You are a helpful assistant.

467

455 -Given the previous instructions, what is the weather in New York City?

468 +Given the previous instructions, what is the weather in New York City? \

469 +Make the response as short as possible.

470 """

471

472 # Change this once other backends support structural_tag

@@ -509,9 +523,10 @@ def test_structured_output_auto_mode(

523 max_tokens=1000,

524 guided_decoding=GuidedDecodingParams(json=unsupported_json_schema))

525

512 - prompts = ("Give an example JSON object for a grade "

513 - "that fits this schema: "

514 - f"{unsupported_json_schema}")

526 + prompts = (

527 + "Give an example JSON object for a grade "

528 + "that fits this schema: "

529 + f"{unsupported_json_schema}. Make the response as short as possible.")

530 # This would fail with the default of "xgrammar", but in "auto"

531 # we will handle fallback automatically.

532 outputs = llm.generate(prompts=prompts,

@@ -566,7 +581,8 @@ def test_guidance_no_additional_properties(monkeypatch: pytest.MonkeyPatch):

581 prompt = (

582 "<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a "

583 "helpful assistant.<|im_end|>\n<|im_start|>user\nPlease generate a "

569 - "large JSON object with key-value pairs a1=b1, a2=b2, ..., a20=b20"

584 + "large JSON object with key-value pairs a1=b1, a2=b2, ..., a20=b20. "

585 + "Make the response as short as possible."

586 "<|im_end|>\n<|im_start|>assistant\n")

587

588 def generate_with_backend(backend):

tests/v1/test_serial_utils.py

+78 -73

@@ -9,8 +9,8 @@ import pytest

9 import torch

11 from vllm.multimodal.inputs import (MultiModalBatchedField,

12 - MultiModalFieldElem, MultiModalKwargs,

13 - MultiModalKwargsItem,

12 + MultiModalFieldElem, MultiModalFlatField,

13 + MultiModalKwargs, MultiModalKwargsItem,

14 MultiModalSharedField, NestedTensors)

15 from vllm.v1.serial_utils import MsgpackDecoder, MsgpackEncoder

@@ -36,59 +36,62 @@ class MyType:

36 empty_tensor: torch.Tensor

39 -def test_encode_decode():

39 +def test_encode_decode(monkeypatch: pytest.MonkeyPatch):

40 """Test encode/decode loop with zero-copy tensors."""

42 - obj = MyType(

43 - tensor1=torch.randint(low=0,

44 - high=100,

45 - size=(1024, ),

46 - dtype=torch.int32),

47 - a_string="hello",

48 - list_of_tensors=[

49 - torch.rand((1, 10), dtype=torch.float32),

50 - torch.rand((3, 5, 4000), dtype=torch.float64),

51 - torch.tensor(1984), # test scalar too

52 - # Make sure to test bf16 which numpy doesn't support.

53 - torch.rand((3, 5, 1000), dtype=torch.bfloat16),

54 - torch.tensor([float("-inf"), float("inf")] * 1024,

55 - dtype=torch.bfloat16),

56 - ],

57 - numpy_array=np.arange(512),

58 - unrecognized=UnrecognizedType(33),

59 - small_f_contig_tensor=torch.rand(5, 4).t(),

60 - large_f_contig_tensor=torch.rand(1024, 4).t(),

61 - small_non_contig_tensor=torch.rand(2, 4)[:, 1:3],

62 - large_non_contig_tensor=torch.rand(1024, 512)[:, 10:20],

63 - empty_tensor=torch.empty(0),

64 - )

42 + with monkeypatch.context() as m:

43 + m.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")

66 - encoder = MsgpackEncoder(size_threshold=256)

67 - decoder = MsgpackDecoder(MyType)

45 + obj = MyType(

46 + tensor1=torch.randint(low=0,

47 + high=100,

48 + size=(1024, ),

49 + dtype=torch.int32),

50 + a_string="hello",

51 + list_of_tensors=[

52 + torch.rand((1, 10), dtype=torch.float32),

53 + torch.rand((3, 5, 4000), dtype=torch.float64),

54 + torch.tensor(1984), # test scalar too

55 + # Make sure to test bf16 which numpy doesn't support.

56 + torch.rand((3, 5, 1000), dtype=torch.bfloat16),

57 + torch.tensor([float("-inf"), float("inf")] * 1024,

58 + dtype=torch.bfloat16),

59 + ],

60 + numpy_array=np.arange(512),

61 + unrecognized=UnrecognizedType(33),

62 + small_f_contig_tensor=torch.rand(5, 4).t(),

63 + large_f_contig_tensor=torch.rand(1024, 4).t(),

64 + small_non_contig_tensor=torch.rand(2, 4)[:, 1:3],

65 + large_non_contig_tensor=torch.rand(1024, 512)[:, 10:20],

66 + empty_tensor=torch.empty(0),

67 + )

69 - encoded = encoder.encode(obj)

69 + encoder = MsgpackEncoder(size_threshold=256)

70 + decoder = MsgpackDecoder(MyType)

71 - # There should be the main buffer + 4 large tensor buffers

72 - # + 1 large numpy array. "large" is <= 512 bytes.

73 - # The two small tensors are encoded inline.

74 - assert len(encoded) == 8

72 + encoded = encoder.encode(obj)

73 +

74 + # There should be the main buffer + 4 large tensor buffers

75 + # + 1 large numpy array. "large" is <= 512 bytes.

76 + # The two small tensors are encoded inline.

77 + assert len(encoded) == 8

76 - decoded: MyType = decoder.decode(encoded)

79 + decoded: MyType = decoder.decode(encoded)

78 - assert_equal(decoded, obj)

81 + assert_equal(decoded, obj)

80 - # Test encode_into case

83 + # Test encode_into case

82 - preallocated = bytearray()

85 + preallocated = bytearray()

84 - encoded2 = encoder.encode_into(obj, preallocated)

87 + encoded2 = encoder.encode_into(obj, preallocated)

86 - assert len(encoded2) == 8

87 - assert encoded2[0] is preallocated

89 + assert len(encoded2) == 8

90 + assert encoded2[0] is preallocated

89 - decoded2: MyType = decoder.decode(encoded2)

92 + decoded2: MyType = decoder.decode(encoded2)

91 - assert_equal(decoded2, obj)

94 + assert_equal(decoded2, obj)

97 class MyRequest(msgspec.Struct):

@@ -122,7 +125,7 @@ def test_multimodal_kwargs():

125 total_len = sum(memoryview(x).cast("B").nbytes for x in encoded)

126

127 # expected total encoding length, should be 44559, +-20 for minor changes

125 - assert total_len >= 44539 and total_len <= 44579

128 + assert 44539 <= total_len <= 44579

129 decoded: MultiModalKwargs = decoder.decode(encoded).mm[0]

130 assert all(nested_equal(d[k], decoded[k]) for k in d)

131

@@ -135,14 +138,15 @@ def test_multimodal_items_by_modality():

138 "video",

139 "v0",

140 [torch.zeros(1000, dtype=torch.int8) for _ in range(4)],

138 - MultiModalBatchedField(),

141 + MultiModalFlatField(

142 + [[slice(1, 2, 3), slice(4, 5, 6)], [slice(None, 2)]], 0),

143 )

144 e3 = MultiModalFieldElem("image", "i0", torch.zeros(1000,

145 dtype=torch.int32),

146 MultiModalSharedField(4))

143 - e4 = MultiModalFieldElem("image", "i1", torch.zeros(1000,

144 - dtype=torch.int32),

145 - MultiModalBatchedField())

147 + e4 = MultiModalFieldElem(

148 + "image", "i1", torch.zeros(1000, dtype=torch.int32),

149 + MultiModalFlatField([slice(1, 2, 3), slice(4, 5, 6)], 2))

150 audio = MultiModalKwargsItem.from_elems([e1])

151 video = MultiModalKwargsItem.from_elems([e2])

152 image = MultiModalKwargsItem.from_elems([e3, e4])

@@ -161,7 +165,7 @@ def test_multimodal_items_by_modality():

165 total_len = sum(memoryview(x).cast("B").nbytes for x in encoded)

166

167 # expected total encoding length, should be 14255, +-20 for minor changes

164 - assert total_len >= 14235 and total_len <= 14275

168 + assert 14250 <= total_len <= 14300

169 decoded: MultiModalKwargs = decoder.decode(encoded).mm[0]

170

171 # check all modalities were recovered and do some basic sanity checks

@@ -178,8 +182,7 @@ def test_multimodal_items_by_modality():

182 def nested_equal(a: NestedTensors, b: NestedTensors):

183 if isinstance(a, torch.Tensor):

184 return torch.equal(a, b)

181 - else:

182 - return all(nested_equal(x, y) for x, y in zip(a, b))

185 + return all(nested_equal(x, y) for x, y in zip(a, b))

186

187

188 def assert_equal(obj1: MyType, obj2: MyType):

@@ -199,11 +202,10 @@ def assert_equal(obj1: MyType, obj2: MyType):

202 assert torch.equal(obj1.empty_tensor, obj2.empty_tensor)

203

204

202 -@pytest.mark.parametrize("allow_pickle", [True, False])

203 -def test_dict_serialization(allow_pickle: bool):

205 +def test_dict_serialization():

206 """Test encoding and decoding of a generic Python object using pickle."""

205 - encoder = MsgpackEncoder(allow_pickle=allow_pickle)

206 - decoder = MsgpackDecoder(allow_pickle=allow_pickle)

207 + encoder = MsgpackEncoder()

208 + decoder = MsgpackDecoder()

209

210 # Create a sample Python object

211 obj = {"key": "value", "number": 42}

@@ -218,11 +220,10 @@ def test_dict_serialization(allow_pickle: bool):

220 assert obj == decoded, "Decoded object does not match the original object."

221

222

221 -@pytest.mark.parametrize("allow_pickle", [True, False])

222 -def test_tensor_serialization(allow_pickle: bool):

223 +def test_tensor_serialization():

224 """Test encoding and decoding of a torch.Tensor."""

224 - encoder = MsgpackEncoder(allow_pickle=allow_pickle)

225 - decoder = MsgpackDecoder(torch.Tensor, allow_pickle=allow_pickle)

225 + encoder = MsgpackEncoder()

226 + decoder = MsgpackDecoder(torch.Tensor)

227

228 # Create a sample tensor

229 tensor = torch.rand(10, 10)

@@ -238,11 +239,10 @@ def test_tensor_serialization(allow_pickle: bool):

239 tensor, decoded), "Decoded tensor does not match the original tensor."

240

241

241 -@pytest.mark.parametrize("allow_pickle", [True, False])

242 -def test_numpy_array_serialization(allow_pickle: bool):

242 +def test_numpy_array_serialization():

243 """Test encoding and decoding of a numpy array."""

244 - encoder = MsgpackEncoder(allow_pickle=allow_pickle)

245 - decoder = MsgpackDecoder(np.ndarray, allow_pickle=allow_pickle)

244 + encoder = MsgpackEncoder()

245 + decoder = MsgpackDecoder(np.ndarray)

246

247 # Create a sample numpy array

248 array = np.random.rand(10, 10)

@@ -268,26 +268,31 @@ class CustomClass:

268 return isinstance(other, CustomClass) and self.value == other.value

269

270

271 -def test_custom_class_serialization_allowed_with_pickle():

271 +def test_custom_class_serialization_allowed_with_pickle(

272 + monkeypatch: pytest.MonkeyPatch):

273 """Test that serializing a custom class succeeds when allow_pickle=True."""

273 - encoder = MsgpackEncoder(allow_pickle=True)

274 - decoder = MsgpackDecoder(CustomClass, allow_pickle=True)

274

276 - obj = CustomClass("test_value")

275 + with monkeypatch.context() as m:

276 + m.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")

277 + encoder = MsgpackEncoder()

278 + decoder = MsgpackDecoder(CustomClass)

279

278 - # Encode the custom class

279 - encoded = encoder.encode(obj)

280 + obj = CustomClass("test_value")

281

281 - # Decode the custom class

282 - decoded = decoder.decode(encoded)

282 + # Encode the custom class

283 + encoded = encoder.encode(obj)

284

284 - # Verify the decoded object matches the original

285 - assert obj == decoded, "Decoded object does not match the original object."

285 + # Decode the custom class

286 + decoded = decoder.decode(encoded)

287 +

288 + # Verify the decoded object matches the original

289 + assert obj == decoded, (

290 + "Decoded object does not match the original object.")

291

292

293 def test_custom_class_serialization_disallowed_without_pickle():

294 """Test that serializing a custom class fails when allow_pickle=False."""

290 - encoder = MsgpackEncoder(allow_pickle=False)

295 + encoder = MsgpackEncoder()

296

297 obj = CustomClass("test_value")

298

vllm/envs.py

@@ -111,6 +111,7 @@ if TYPE_CHECKING:

111 VLLM_USE_DEEP_GEMM: bool = False

112 VLLM_XGRAMMAR_CACHE_MB: int = 0

113 VLLM_MSGPACK_ZERO_COPY_THRESHOLD: int = 256

114 + VLLM_ALLOW_INSECURE_SERIALIZATION: bool = False

115

116

117 def get_default_cache_root():

@@ -736,6 +737,12 @@ environment_variables: dict[str, Callable[[], Any]] = {

737 # limit will actually be zero-copy decoded.

738 "VLLM_MSGPACK_ZERO_COPY_THRESHOLD":

739 lambda: int(os.getenv("VLLM_MSGPACK_ZERO_COPY_THRESHOLD", "256")),

740 +

741 + # If set, allow insecure serialization using pickle.

742 + # This is useful for environments where it is deemed safe to use the

743 + # insecure method and it is needed for some reason.

744 + "VLLM_ALLOW_INSECURE_SERIALIZATION":

745 + lambda: bool(int(os.getenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "0"))),

746 }

747

748 # end-env-vars-definition

vllm/v1/serial_utils.py

+35 -8

@@ -14,6 +14,7 @@ import zmq

14 from msgspec import msgpack

16 from vllm import envs

17 +from vllm.logger import init_logger

18 from vllm.multimodal.inputs import (BaseMultiModalField,

19 MultiModalBatchedField,

20 MultiModalFieldConfig, MultiModalFieldElem,

@@ -21,6 +22,8 @@ from vllm.multimodal.inputs import (BaseMultiModalField,

22 MultiModalKwargsItem,

23 MultiModalSharedField, NestedTensors)

25 +logger = init_logger(__name__)

26 +

27 CUSTOM_TYPE_PICKLE = 1

28 CUSTOM_TYPE_CLOUDPICKLE = 2

29 CUSTOM_TYPE_RAW_VIEW = 3

@@ -47,9 +50,7 @@ class MsgpackEncoder:

50 via dedicated messages. Note that this is a per-tensor limit.

51 """

50 - def __init__(self,

51 - size_threshold: Optional[int] = None,

52 - allow_pickle: bool = True):

53 + def __init__(self, size_threshold: Optional[int] = None):

54 if size_threshold is None:

55 size_threshold = envs.VLLM_MSGPACK_ZERO_COPY_THRESHOLD

56 self.encoder = msgpack.Encoder(enc_hook=self.enc_hook)

@@ -58,7 +59,10 @@ class MsgpackEncoder:

59 # pass custom data to the hook otherwise.

60 self.aux_buffers: Optional[list[bytestr]] = None

61 self.size_threshold = size_threshold

61 - self.allow_pickle = allow_pickle

62 + if envs.VLLM_ALLOW_INSECURE_SERIALIZATION:

63 + logger.warning(

64 + "Allowing insecure serialization using pickle due to "

65 + "VLLM_ALLOW_INSECURE_SERIALIZATION=1")

67 def encode(self, obj: Any) -> Sequence[bytestr]:

68 try:

@@ -89,6 +93,12 @@ class MsgpackEncoder:

93 if isinstance(obj, np.ndarray) and obj.dtype.kind not in ('O', 'V'):

94 return self._encode_ndarray(obj)

96 + if isinstance(obj, slice):

97 + # We are assuming only int-based values will be used here.

98 + return tuple(

99 + int(v) if v is not None else None

100 + for v in (obj.start, obj.stop, obj.step))

101 +

102 if isinstance(obj, MultiModalKwargs):

103 mm: MultiModalKwargs = obj

104 if not mm.modalities:

@@ -108,7 +118,7 @@ class MsgpackEncoder:

118 for itemlist in mm._items_by_modality.values()

119 for item in itemlist]

120

111 - if not self.allow_pickle:

121 + if not envs.VLLM_ALLOW_INSECURE_SERIALIZATION:

122 raise TypeError(f"Object of type {type(obj)} is not serializable")

123

124 if isinstance(obj, FunctionType):

@@ -185,13 +195,16 @@ class MsgpackDecoder:

195 not thread-safe when encoding tensors / numpy arrays.

196 """

197

188 - def __init__(self, t: Optional[Any] = None, allow_pickle: bool = True):

198 + def __init__(self, t: Optional[Any] = None):

199 args = () if t is None else (t, )

200 self.decoder = msgpack.Decoder(*args,

201 ext_hook=self.ext_hook,

202 dec_hook=self.dec_hook)

203 self.aux_buffers: Sequence[bytestr] = ()

194 - self.allow_pickle = allow_pickle

204 + if envs.VLLM_ALLOW_INSECURE_SERIALIZATION:

205 + logger.warning(

206 + "Allowing insecure deserialization using pickle due to "

207 + "VLLM_ALLOW_INSECURE_SERIALIZATION=1")

208

209 def decode(self, bufs: Union[bytestr, Sequence[bytestr]]) -> Any:

210 if isinstance(bufs, (bytes, bytearray, memoryview, zmq.Frame)):

@@ -212,6 +225,8 @@ class MsgpackDecoder:

225 return self._decode_ndarray(obj)

226 if issubclass(t, torch.Tensor):

227 return self._decode_tensor(obj)

228 + if t is slice:

229 + return slice(*obj)

230 if issubclass(t, MultiModalKwargs):

231 if isinstance(obj, list):

232 return MultiModalKwargs.from_items(

@@ -253,6 +268,12 @@ class MsgpackDecoder:

268 factory_meth_name, *field_args = v["field"]

269 factory_meth = getattr(MultiModalFieldConfig,

270 factory_meth_name)

271 +

272 + # Special case: decode the union "slices" field of

273 + # MultiModalFlatField

274 + if factory_meth_name == "flat":

275 + field_args[0] = self._decode_nested_slices(field_args[0])

276 +

277 v["field"] = factory_meth(None, *field_args).field

278 elems.append(MultiModalFieldElem(**v))

279 decoded_items.append(MultiModalKwargsItem.from_elems(elems))

@@ -269,11 +290,17 @@ class MsgpackDecoder:

290 return self._decode_tensor(obj)

291 return [self._decode_nested_tensors(x) for x in obj]

292

293 + def _decode_nested_slices(self, obj: Any) -> Any:

294 + assert isinstance(obj, (list, tuple))

295 + if obj and not isinstance(obj[0], (list, tuple)):

296 + return slice(*obj)

297 + return [self._decode_nested_slices(x) for x in obj]

298 +

299 def ext_hook(self, code: int, data: memoryview) -> Any:

300 if code == CUSTOM_TYPE_RAW_VIEW:

301 return data

302

276 - if self.allow_pickle:

303 + if envs.VLLM_ALLOW_INSECURE_SERIALIZATION:

304 if code == CUSTOM_TYPE_PICKLE:

305 return pickle.loads(data)

306 if code == CUSTOM_TYPE_CLOUDPICKLE: