A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
[Kernel] changing fused moe kernel chunk size default to 32k (#7995)
Avshalom Manevich committed
Aug 30, 2024 at 11:11 UTC
34a0e96d463d37cf85cee9c2cd01397034e97573
1 file changed
+1
-1
vllm/envs.py
+1
-1
@@ -352,7 +352,7 @@ environment_variables: Dict[str, Callable[[], Any]] = {
352
os.path.join(get_default_cache_root(), "vllm", "xla_cache"),
353
)),
354
"VLLM_FUSED_MOE_CHUNK_SIZE":
355
- lambda: int(os.getenv("VLLM_FUSED_MOE_CHUNK_SIZE", "65536")),
355
+ lambda: int(os.getenv("VLLM_FUSED_MOE_CHUNK_SIZE", "32768")),
356
357
# If set, vllm will skip the deprecation warnings.
358
"VLLM_NO_DEPRECATION_WARNING":