@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / ef68eb28d8
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 03:15 UTC

[Bug] Fix pickling of `ModelConfig` when RunAI Model Streamer is used (#11825)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Jan 8, 2025 at 13:40 UTC ef68eb28d8d45be6e0defe82245e16be9362e375
1 file changed +6 -6
vllm/config.py
+6 -6
@@ -381,16 +381,16 @@ class ModelConfig:
381 """
382 if is_s3(model) or is_s3(tokenizer):
383 if is_s3(model):
384 - self.s3_model = S3Model()
385 - self.s3_model.pull_files(model, allow_pattern=["*config.json"])
384 + s3_model = S3Model()
385 + s3_model.pull_files(model, allow_pattern=["*config.json"])
386 self.model_weights = self.model
387 - self.model = self.s3_model.dir
387 + self.model = s3_model.dir
388
389 if is_s3(tokenizer):
390 - self.s3_tokenizer = S3Model()
391 - self.s3_tokenizer.pull_files(
390 + s3_tokenizer = S3Model()
391 + s3_tokenizer.pull_files(
392 model, ignore_pattern=["*.pt", "*.safetensors", "*.bin"])
393 - self.tokenizer = self.s3_tokenizer.dir
393 + self.tokenizer = s3_tokenizer.dir
394
395 def _init_multimodal_config(
396 self, limit_mm_per_prompt: Optional[Mapping[str, int]]