A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 03:15 UTC
[Bug] Fix pickling of `ModelConfig` when RunAI Model Streamer is used (#11825)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Jan 8, 2025 at 13:40 UTC
ef68eb28d8d45be6e0defe82245e16be9362e375
1 file changed
+6
-6
vllm/config.py
+6
-6
@@ -381,16 +381,16 @@ class ModelConfig:
381
"""
382
if is_s3(model) or is_s3(tokenizer):
383
if is_s3(model):
384
- self.s3_model = S3Model()
385
- self.s3_model.pull_files(model, allow_pattern=["*config.json"])
384
+ s3_model = S3Model()
385
+ s3_model.pull_files(model, allow_pattern=["*config.json"])
386
self.model_weights = self.model
387
- self.model = self.s3_model.dir
387
+ self.model = s3_model.dir
388
389
if is_s3(tokenizer):
390
- self.s3_tokenizer = S3Model()
391
- self.s3_tokenizer.pull_files(
390
+ s3_tokenizer = S3Model()
391
+ s3_tokenizer.pull_files(
392
model, ignore_pattern=["*.pt", "*.safetensors", "*.bin"])
393
- self.tokenizer = self.s3_tokenizer.dir
393
+ self.tokenizer = s3_tokenizer.dir
394
395
def _init_multimodal_config(
396
self, limit_mm_per_prompt: Optional[Mapping[str, int]]