@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 386087970a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[CI/Build] build on empty device for better dev experience (#4773)

tomeras91 committed Aug 11, 2024 at 23:09 UTC 386087970ac220ad675bd1ab7c0e4abb309d4e8a
2 files changed +21 -7
requirements-cuda.txt
+2 -2
@@ -7,5 +7,5 @@ nvidia-ml-py # for pynvml package
7 torch == 2.4.0
8 # These must be updated alongside torch
9 torchvision == 0.19 # Required for phi3v processor. See https://github.com/pytorch/vision?tab=readme-ov-file#installation for corresponding version
10 -xformers == 0.0.27.post2 # Requires PyTorch 2.4.0
11 -vllm-flash-attn == 2.6.1 # Requires PyTorch 2.4.0
10 +xformers == 0.0.27.post2; platform_system == 'Linux' and platform_machine == 'x86_64' # Requires PyTorch 2.4.0
11 +vllm-flash-attn == 2.6.1; platform_system == 'Linux' and platform_machine == 'x86_64' # Requires PyTorch 2.4.0
setup.py
+19 -5
@@ -61,9 +61,12 @@ envs = load_module_from_path('envs', os.path.join(ROOT_DIR, 'vllm', 'envs.py'))
61
62 VLLM_TARGET_DEVICE = envs.VLLM_TARGET_DEVICE
63
64 -# vLLM only supports Linux platform
65 -assert sys.platform.startswith(
66 - "linux"), "vLLM only supports Linux platform (including WSL)."
64 +if not sys.platform.startswith("linux"):
65 + logger.warning(
66 + "vLLM only supports Linux platform (including WSL). "
67 + "Building on %s, "
68 + "so vLLM may not be able to run correctly", sys.platform)
69 + VLLM_TARGET_DEVICE = "empty"
70
71 MAIN_CUDA_VERSION = "12.1"
72
@@ -231,6 +234,10 @@ class cmake_build_ext(build_ext):
234 subprocess.check_call(["cmake", *build_args], cwd=self.build_temp)
235
236
237 +def _no_device() -> bool:
238 + return VLLM_TARGET_DEVICE == "empty"
239 +
240 +
241 def _is_cuda() -> bool:
242 has_cuda = torch.version.cuda is not None
243 return (VLLM_TARGET_DEVICE == "cuda" and has_cuda
@@ -350,7 +357,9 @@ def find_version(filepath: str) -> str:
357 def get_vllm_version() -> str:
358 version = find_version(get_path("vllm", "version.py"))
359
353 - if _is_cuda():
360 + if _no_device():
361 + version += "+empty"
362 + elif _is_cuda():
363 cuda_version = str(get_nvcc_cuda_version())
364 if cuda_version != MAIN_CUDA_VERSION:
365 cuda_version_str = cuda_version.replace(".", "")[:3]
@@ -404,7 +413,9 @@ def get_requirements() -> List[str]:
413 resolved_requirements.append(line)
414 return resolved_requirements
415
407 - if _is_cuda():
416 + if _no_device():
417 + requirements = _read_requirements("requirements-cuda.txt")
418 + elif _is_cuda():
419 requirements = _read_requirements("requirements-cuda.txt")
420 cuda_major, cuda_minor = torch.version.cuda.split(".")
421 modified_requirements = []
@@ -453,6 +464,9 @@ if envs.VLLM_USE_PRECOMPILED:
464 ext_modules = []
465 package_data["vllm"].append("*.so")
466
467 +if _no_device():
468 + ext_modules = []
469 +
470 setup(
471 name="vllm",
472 version=get_vllm_version(),