A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 23:45 UTC
[Bugfix] Fix isinstance check for tensor types in _load_prompt_embeds to use dtype comparison (#21612)
Signed-off-by: Alexandre Juan <a.juan@netheos.net>
Alexandre JUAN committed
Jul 26, 2025 at 05:11 UTC
2f6e6b33fb0acf5331982c8e1ea620005e3a19ba
1 file changed
+5
-3
vllm/entrypoints/openai/serving_engine.py
+5
-3
@@ -957,9 +957,11 @@ class OpenAIServing:
957
def _load_and_validate_embed(embed: bytes) -> EmbedsPrompt:
958
tensor = torch.load(io.BytesIO(base64.b64decode(embed)),
959
weights_only=True)
960
- assert isinstance(
961
- tensor,
962
- (torch.FloatTensor, torch.BFloat16Tensor, torch.HalfTensor))
960
+ assert isinstance(tensor, torch.Tensor) and tensor.dtype in (
961
+ torch.float32,
962
+ torch.bfloat16,
963
+ torch.float16,
964
+ )
965
if tensor.dim() > 2:
966
tensor = tensor.squeeze(0)
967
assert tensor.dim() == 2