A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 23:30 UTC
[Misc] AMD Build Improvements (#12923)
Shiyan Deng committed
Feb 12, 2025 at 02:36 UTC
f1042e86f05cfe93bcadac445e78671ed2e8fddb
4 files changed
+14
-7
csrc/moe/moe_align_sum_kernels.cu
+1
-1
@@ -3,7 +3,7 @@
3
#include <c10/cuda/CUDAGuard.h>
4
5
#include <ATen/ATen.h>
6
-#include <THC/THCAtomics.cuh>
6
+#include <ATen/cuda/Atomic.cuh>
7
8
#include "../cuda_compat.h"
9
#include "../dispatch_utils.h"
csrc/rocm/attention.cu
+1
-1
@@ -1122,4 +1122,4 @@ void paged_attention(
1122
#undef WARP_SIZE
1123
#undef MAX
1124
#undef MIN
1125
-#undef DIVIDE_ROUND_UP
\ No newline at end of file
1125
+#undef DIVIDE_ROUND_UP
vllm/model_executor/models/registry.py
+11
-4
@@ -205,6 +205,14 @@ _VLLM_MODELS = {
205
**_FALLBACK_MODEL,
206
}
207
208
+# This variable is used as the args for subprocess.run(). We
209
+# can modify this variable to alter the args if needed. e.g.
210
+# when we use par format to pack things together, sys.executable
211
+# might not be the target we want to run.
212
+_SUBPROCESS_COMMAND = [
213
+ sys.executable, "-m", "vllm.model_executor.models.registry"
214
+]
215
+
216
217
@dataclass(frozen=True)
218
class _ModelInfo:
@@ -502,10 +510,9 @@ def _run_in_subprocess(fn: Callable[[], _T]) -> _T:
510
511
# cannot use `sys.executable __file__` here because the script
512
# contains relative imports
505
- returned = subprocess.run(
506
- [sys.executable, "-m", "vllm.model_executor.models.registry"],
507
- input=input_bytes,
508
- capture_output=True)
513
+ returned = subprocess.run(_SUBPROCESS_COMMAND,
514
+ input=input_bytes,
515
+ capture_output=True)
516
517
# check if the subprocess is successful
518
try:
vllm/transformers_utils/configs/__init__.py
+1
-1
@@ -45,4 +45,4 @@ __all__ = [
45
"SolarConfig",
46
"Telechat2Config",
47
"UltravoxConfig",
48
-]
\ No newline at end of file
48
+]