A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 15:45 UTC
[misc][installation] build from source without compilation (#8818)
youkaichao committed
Sep 26, 2024 at 13:19 UTC
70de39f6b46f6b90aecba52358825127a50b3921
2 files changed
+85
-3
docs/source/getting_started/installation.rst
+31
-3
@@ -58,13 +58,41 @@ You can install vLLM using pip:
58
$ # export VLLM_COMMIT=...
59
$ # pip install https://vllm-wheels.s3.us-west-2.amazonaws.com/${VLLM_COMMIT}/vllm-${VLLM_VERSION}-cp38-abi3-manylinux1_x86_64.whl
60
61
+Build from source (without compilation)
62
+---------------------------------------
63
+
64
+If you want to develop vLLM, and you only need to change the Python code, you can build vLLM without compilation.
65
+
66
+The first step is to follow the previous instructions to install the latest vLLM wheel:
67
+
68
+.. code-block:: console
69
+
70
+ $ export VLLM_VERSION=0.6.1.post1
71
+ $ pip install https://vllm-wheels.s3.us-west-2.amazonaws.com/nightly/vllm-${VLLM_VERSION}-cp38-abi3-manylinux1_x86_64.whl
72
+
73
+After verifying that the installation is successful, we have a script for you to copy and link directories, so that you can edit the Python code directly:
74
+
75
+.. code-block:: console
76
+
77
+ $ git clone https://github.com/vllm-project/vllm.git
78
+ $ cd vllm
79
+ $ python python_only_dev.py
80
+
81
+It will:
82
+
83
+- Find the installed vLLM in the current environment.
84
+- Copy built files to the current directory.
85
+- Rename the installed vLLM
86
+- Symbolically link the current directory to the installed vLLM.
87
+
88
+This way, you can edit the Python code in the current directory, and the changes will be reflected in the installed vLLM.
89
90
.. _build_from_source:
91
64
-Build from source
65
------------------
92
+Build from source (with compilation)
93
+------------------------------------
94
67
-You can also build and install vLLM from source:
95
+If you need to touch the C++ or CUDA code, you need to build vLLM from source:
96
97
.. code-block:: console
98
python_only_dev.py
new
+54
@@ -0,0 +1,54 @@
1
+# enable python only development
2
+# copy compiled files to the current directory directly
3
+
4
+import os
5
+import shutil
6
+import subprocess
7
+import sys
8
+
9
+# cannot directly `import vllm` , because it will try to
10
+# import from the current directory
11
+output = subprocess.run([sys.executable, "-m", "pip", "show", "vllm"],
12
+ capture_output=True)
13
+
14
+assert output.returncode == 0, "vllm is not installed"
15
+
16
+text = output.stdout.decode("utf-8")
17
+
18
+package_path = None
19
+for line in text.split("\n"):
20
+ if line.startswith("Location: "):
21
+ package_path = line.split(": ")[1]
22
+ break
23
+
24
+assert package_path is not None, "could not find package path"
25
+
26
+cwd = os.getcwd()
27
+
28
+assert cwd != package_path, "should not import from the current directory"
29
+
30
+files_to_copy = [
31
+ "vllm/_C.abi3.so",
32
+ "vllm/_core_C.abi3.so",
33
+ "vllm/_moe_C.abi3.so",
34
+ "vllm/vllm_flash_attn/vllm_flash_attn_c.abi3.so",
35
+ "vllm/vllm_flash_attn/flash_attn_interface.py",
36
+ "vllm/vllm_flash_attn/__init__.py",
37
+ # "vllm/_version.py", # not available in nightly wheels yet
38
+]
39
+
40
+for file in files_to_copy:
41
+ src = os.path.join(package_path, file)
42
+ dst = file
43
+ print(f"Copying {src} to {dst}")
44
+ shutil.copyfile(src, dst)
45
+
46
+pre_built_vllm_path = os.path.join(package_path, "vllm")
47
+tmp_path = os.path.join(package_path, "vllm_pre_built")
48
+current_vllm_path = os.path.join(cwd, "vllm")
49
+
50
+print(f"Renaming {pre_built_vllm_path} to {tmp_path}")
51
+os.rename(pre_built_vllm_path, tmp_path)
52
+
53
+print(f"linking {current_vllm_path} to {pre_built_vllm_path}")
54
+os.symlink(current_vllm_path, pre_built_vllm_path)