A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[CI/Build] build on empty device for better dev experience (#4773)
tomeras91 committed
Aug 11, 2024 at 23:09 UTC
386087970ac220ad675bd1ab7c0e4abb309d4e8a
2 files changed
+21
-7
requirements-cuda.txt
+2
-2
@@ -7,5 +7,5 @@ nvidia-ml-py # for pynvml package
7
torch == 2.4.0
8
# These must be updated alongside torch
9
torchvision == 0.19 # Required for phi3v processor. See https://github.com/pytorch/vision?tab=readme-ov-file#installation for corresponding version
10
-xformers == 0.0.27.post2 # Requires PyTorch 2.4.0
11
-vllm-flash-attn == 2.6.1 # Requires PyTorch 2.4.0
10
+xformers == 0.0.27.post2; platform_system == 'Linux' and platform_machine == 'x86_64' # Requires PyTorch 2.4.0
11
+vllm-flash-attn == 2.6.1; platform_system == 'Linux' and platform_machine == 'x86_64' # Requires PyTorch 2.4.0
setup.py
+19
-5
@@ -61,9 +61,12 @@ envs = load_module_from_path('envs', os.path.join(ROOT_DIR, 'vllm', 'envs.py'))
61
62
VLLM_TARGET_DEVICE = envs.VLLM_TARGET_DEVICE
63
64
-# vLLM only supports Linux platform
65
-assert sys.platform.startswith(
66
- "linux"), "vLLM only supports Linux platform (including WSL)."
64
+if not sys.platform.startswith("linux"):
65
+ logger.warning(
66
+ "vLLM only supports Linux platform (including WSL). "
67
+ "Building on %s, "
68
+ "so vLLM may not be able to run correctly", sys.platform)
69
+ VLLM_TARGET_DEVICE = "empty"
70
71
MAIN_CUDA_VERSION = "12.1"
72
@@ -231,6 +234,10 @@ class cmake_build_ext(build_ext):
234
subprocess.check_call(["cmake", *build_args], cwd=self.build_temp)
235
236
237
+def _no_device() -> bool:
238
+ return VLLM_TARGET_DEVICE == "empty"
239
+
240
+
241
def _is_cuda() -> bool:
242
has_cuda = torch.version.cuda is not None
243
return (VLLM_TARGET_DEVICE == "cuda" and has_cuda
@@ -350,7 +357,9 @@ def find_version(filepath: str) -> str:
357
def get_vllm_version() -> str:
358
version = find_version(get_path("vllm", "version.py"))
359
353
- if _is_cuda():
360
+ if _no_device():
361
+ version += "+empty"
362
+ elif _is_cuda():
363
cuda_version = str(get_nvcc_cuda_version())
364
if cuda_version != MAIN_CUDA_VERSION:
365
cuda_version_str = cuda_version.replace(".", "")[:3]
@@ -404,7 +413,9 @@ def get_requirements() -> List[str]:
413
resolved_requirements.append(line)
414
return resolved_requirements
415
407
- if _is_cuda():
416
+ if _no_device():
417
+ requirements = _read_requirements("requirements-cuda.txt")
418
+ elif _is_cuda():
419
requirements = _read_requirements("requirements-cuda.txt")
420
cuda_major, cuda_minor = torch.version.cuda.split(".")
421
modified_requirements = []
@@ -453,6 +464,9 @@ if envs.VLLM_USE_PRECOMPILED:
464
ext_modules = []
465
package_data["vllm"].append("*.so")
466
467
+if _no_device():
468
+ ext_modules = []
469
+
470
setup(
471
name="vllm",
472
version=get_vllm_version(),