@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 029c71de11
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 18:15 UTC

[CI/Build] Avoid downloading all HF files in `RemoteOpenAIServer` (#7836)

Cyrus Leung committed Aug 26, 2024 at 13:31 UTC 029c71de11bc3bcf84a1b3cf9d91e79ab6949799
2 files changed +27 -15
tests/utils.py
+26 -14
@@ -11,13 +11,14 @@ from typing import Any, Callable, Dict, List, Optional
11
12 import openai
13 import requests
14 -from huggingface_hub import snapshot_download
14 from transformers import AutoTokenizer
15 from typing_extensions import ParamSpec
16
17 from vllm.distributed import (ensure_model_parallel_initialized,
18 init_distributed_environment)
19 +from vllm.engine.arg_utils import AsyncEngineArgs
20 from vllm.entrypoints.openai.cli_args import make_arg_parser
21 +from vllm.model_executor.model_loader.loader import DefaultModelLoader
22 from vllm.platforms import current_platform
23 from vllm.utils import FlexibleArgumentParser, get_open_port, is_hip
24
@@ -60,39 +61,50 @@ class RemoteOpenAIServer:
61
62 def __init__(self,
63 model: str,
63 - cli_args: List[str],
64 + vllm_serve_args: List[str],
65 *,
66 env_dict: Optional[Dict[str, str]] = None,
67 auto_port: bool = True,
68 max_wait_seconds: Optional[float] = None) -> None:
68 - if not model.startswith("/"):
69 - # download the model if it's not a local path
70 - # to exclude the model download time from the server start time
71 - snapshot_download(model)
69 if auto_port:
73 - if "-p" in cli_args or "--port" in cli_args:
74 - raise ValueError("You have manually specified the port"
70 + if "-p" in vllm_serve_args or "--port" in vllm_serve_args:
71 + raise ValueError("You have manually specified the port "
72 "when `auto_port=True`.")
73
77 - cli_args = cli_args + ["--port", str(get_open_port())]
74 + # Don't mutate the input args
75 + vllm_serve_args = vllm_serve_args + [
76 + "--port", str(get_open_port())
77 + ]
78
79 parser = FlexibleArgumentParser(
80 description="vLLM's remote OpenAI server.")
81 parser = make_arg_parser(parser)
82 - args = parser.parse_args(cli_args)
82 + args = parser.parse_args(["--model", model, *vllm_serve_args])
83 self.host = str(args.host or 'localhost')
84 self.port = int(args.port)
85
86 + # download the model before starting the server to avoid timeout
87 + is_local = os.path.isdir(model)
88 + if not is_local:
89 + engine_args = AsyncEngineArgs.from_cli_args(args)
90 + engine_config = engine_args.create_engine_config()
91 + dummy_loader = DefaultModelLoader(engine_config.load_config)
92 + dummy_loader._prepare_weights(engine_config.model_config.model,
93 + engine_config.model_config.revision,
94 + fall_back_to_pt=True)
95 +
96 env = os.environ.copy()
97 # the current process might initialize cuda,
98 # to be safe, we should use spawn method
99 env['VLLM_WORKER_MULTIPROC_METHOD'] = 'spawn'
100 if env_dict is not None:
101 env.update(env_dict)
92 - self.proc = subprocess.Popen(["vllm", "serve"] + [model] + cli_args,
93 - env=env,
94 - stdout=sys.stdout,
95 - stderr=sys.stderr)
102 + self.proc = subprocess.Popen(
103 + ["vllm", "serve", model, *vllm_serve_args],
104 + env=env,
105 + stdout=sys.stdout,
106 + stderr=sys.stderr,
107 + )
108 max_wait_seconds = max_wait_seconds or 240
109 self._wait_for_server(url=self.url_for("health"),
110 timeout=max_wait_seconds)
vllm/engine/arg_utils.py
+1 -1
@@ -742,7 +742,7 @@ class EngineArgs:
742 engine_args = cls(**{attr: getattr(args, attr) for attr in attrs})
743 return engine_args
744
745 - def create_engine_config(self, ) -> EngineConfig:
745 + def create_engine_config(self) -> EngineConfig:
746 # gguf file needs a specific model loader and doesn't use hf_repo
747 if self.model.endswith(".gguf"):
748 self.quantization = self.load_format = "gguf"