@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / f7a8fa39d8
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 22:00 UTC

[Kernel] reloading fused_moe config on the last chunk (#6210)

Avshalom Manevich committed Jul 8, 2024 at 18:00 UTC f7a8fa39d828136d0b8bbdd20c262602e5543ffd
1 file changed +36 -15
vllm/model_executor/layers/fused_moe/fused_moe.py
+36 -15
@@ -332,6 +332,31 @@ def get_default_config(
332 return config
333
334
335 +def try_get_optimal_moe_config(
336 + w1_shape: Tuple[int, ...],
337 + w2_shape: Tuple[int, ...],
338 + top_k: int,
339 + dtype: Optional[str],
340 + M: int,
341 + override_config: Optional[Dict[str, Any]] = None,
342 +):
343 + if override_config:
344 + config = override_config
345 + else:
346 + # First try to load optimal config from the file
347 + E, _, N = w2_shape
348 + configs = get_moe_configs(E, N, dtype)
349 +
350 + if configs:
351 + # If an optimal configuration map has been found, look up the
352 + # optimal config
353 + config = configs[min(configs.keys(), key=lambda x: abs(x - M))]
354 + else:
355 + # Else use the default config
356 + config = get_default_config(M, E, N, w1_shape[2], top_k, dtype)
357 + return config
358 +
359 +
360 def fused_topk(
361 hidden_states: torch.Tensor,
362 gating_output: torch.Tensor,
@@ -428,22 +453,16 @@ def fused_experts(hidden_states: torch.Tensor,
453 CHUNK_SIZE = envs.VLLM_FUSED_MOE_CHUNK_SIZE
454 M = min(num_tokens, CHUNK_SIZE)
455
431 - if override_config:
432 - config = override_config
433 - else:
434 - # First try to load optimal config from the file
435 - configs = get_moe_configs(E, w2.shape[2],
436 - "float8" if use_fp8 else None)
456 + get_config_func = functools.partial(
457 + try_get_optimal_moe_config,
458 + w1.shape,
459 + w2.shape,
460 + topk_ids.shape[1],
461 + "float8" if use_fp8 else None,
462 + override_config=override_config,
463 + )
464
438 - if configs:
439 - # If an optimal configuration map has been found, look up the
440 - # optimal config
441 - config = configs[min(configs.keys(), key=lambda x: abs(x - M))]
442 - else:
443 - # Else use the default config
444 - config = get_default_config(M, E, N, w1.shape[2],
445 - topk_ids.shape[1],
446 - "float8" if use_fp8 else None)
465 + config = get_config_func(M)
466
467 intermediate_cache1 = torch.empty((M, topk_ids.shape[1], N),
468 device=hidden_states.device,
@@ -478,6 +497,8 @@ def fused_experts(hidden_states: torch.Tensor,
497 intermediate_cache1 = intermediate_cache1[:tokens_in_chunk]
498 intermediate_cache2 = intermediate_cache2[:tokens_in_chunk]
499 intermediate_cache3 = intermediate_cache3[:tokens_in_chunk]
500 + # reload config to get better performance on the last chunk
501 + config = get_config_func(tokens_in_chunk)
502
503 curr_topk_ids = topk_ids[begin_chunk_idx:end_chunk_idx]
504 curr_topk_weights = topk_weights[begin_chunk_idx:end_chunk_idx]