A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
[tpu][misc] fix typo (#8260)
youkaichao committed
Sep 6, 2024 at 22:40 UTC
ce2702a92356b69ec1ea35ecd46263ddf98e8e2c
3 files changed
+5
-5
tests/compile/test_wrapper.py
+2
-2
@@ -2,7 +2,7 @@ from typing import Optional
2
3
import torch
4
5
-from vllm.compilation.wrapper import TorchCompileWrapperWithCustomDispacther
5
+from vllm.compilation.wrapper import TorchCompileWrapperWithCustomDispatcher
6
7
8
class MyMod(torch.nn.Module):
@@ -13,7 +13,7 @@ class MyMod(torch.nn.Module):
13
return x * 2
14
15
16
-class MyWrapper(TorchCompileWrapperWithCustomDispacther):
16
+class MyWrapper(TorchCompileWrapperWithCustomDispatcher):
17
18
def __init__(self, model):
19
self.model = model
vllm/compilation/wrapper.py
+1
-1
@@ -10,7 +10,7 @@ import torch
10
import vllm.envs as envs
11
12
13
-class TorchCompileWrapperWithCustomDispacther:
13
+class TorchCompileWrapperWithCustomDispatcher:
14
"""
15
A wrapper class for torch.compile, with a custom dispatch logic.
16
Subclasses should:
vllm/worker/tpu_model_runner.py
+2
-2
@@ -11,7 +11,7 @@ import torch_xla.core.xla_model as xm
11
import torch_xla.runtime as xr
12
13
from vllm.attention import AttentionMetadata, get_attn_backend
14
-from vllm.compilation.wrapper import TorchCompileWrapperWithCustomDispacther
14
+from vllm.compilation.wrapper import TorchCompileWrapperWithCustomDispatcher
15
from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, ModelConfig,
16
ParallelConfig, SchedulerConfig)
17
from vllm.logger import init_logger
@@ -611,7 +611,7 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
611
return [SamplerOutput(sampler_outputs)]
612
613
614
-class ModelWrapper(TorchCompileWrapperWithCustomDispacther):
614
+class ModelWrapper(TorchCompileWrapperWithCustomDispatcher):
615
616
def __init__(self, model: nn.Module):
617
self.model = model