@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / a9a2e74d21
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 20:45 UTC

[Misc] Use `torch.Tensor` for type annotation (#6505)

Woosuk Kwon committed Jul 17, 2024 at 06:01 UTC a9a2e74d21d7ef893ff1b7d2c34f390ab3b7736c
2 files changed +18 -18
benchmarks/cutlass_benchmarks/w8a8_benchmarks.py
+17 -17
@@ -20,18 +20,18 @@ DEFAULT_TP_SIZES = [1]
20 # helpers
21
22
23 -def to_fp8(tensor: torch.tensor) -> torch.tensor:
23 +def to_fp8(tensor: torch.Tensor) -> torch.Tensor:
24 finfo = torch.finfo(torch.float8_e4m3fn)
25 return torch.round(tensor.clamp(
26 min=finfo.min, max=finfo.max)).to(dtype=torch.float8_e4m3fn)
27
28
29 -def to_int8(tensor: torch.tensor) -> torch.tensor:
29 +def to_int8(tensor: torch.Tensor) -> torch.Tensor:
30 return torch.round(tensor.clamp(min=-128, max=127)).to(dtype=torch.int8)
31
32
33 def make_rand_tensors(dtype: torch.dtype, m: int, n: int,
34 - k: int) -> Tuple[torch.tensor, torch.tensor]:
34 + k: int) -> Tuple[torch.Tensor, torch.Tensor]:
35
36 a = torch.randn((m, k), device='cuda') * 5
37 b = torch.randn((n, k), device='cuda').t() * 5
@@ -47,15 +47,15 @@ def make_rand_tensors(dtype: torch.dtype, m: int, n: int,
47 # impl
48
49
50 -def pytorch_mm_impl(a: torch.tensor, b: torch.tensor, scale_a: torch.tensor,
51 - scale_b: torch.tensor,
52 - out_dtype: torch.dtype) -> torch.tensor:
50 +def pytorch_mm_impl(a: torch.Tensor, b: torch.Tensor, scale_a: torch.Tensor,
51 + scale_b: torch.Tensor,
52 + out_dtype: torch.dtype) -> torch.Tensor:
53 return torch.mm(a, b)
54
55
56 -def pytorch_fp8_impl(a: torch.tensor, b: torch.tensor, scale_a: torch.tensor,
57 - scale_b: torch.tensor,
58 - out_dtype: torch.dtype) -> torch.tensor:
56 +def pytorch_fp8_impl(a: torch.Tensor, b: torch.Tensor, scale_a: torch.Tensor,
57 + scale_b: torch.Tensor,
58 + out_dtype: torch.dtype) -> torch.Tensor:
59 return torch._scaled_mm(a,
60 b,
61 scale_a=scale_a,
@@ -63,9 +63,9 @@ def pytorch_fp8_impl(a: torch.tensor, b: torch.tensor, scale_a: torch.tensor,
63 out_dtype=out_dtype)
64
65
66 -def pytorch_fp8_impl_fast_accum(a: torch.tensor, b: torch.tensor,
67 - scale_a: torch.tensor, scale_b: torch.tensor,
68 - out_dtype: torch.dtype) -> torch.tensor:
66 +def pytorch_fp8_impl_fast_accum(a: torch.Tensor, b: torch.Tensor,
67 + scale_a: torch.Tensor, scale_b: torch.Tensor,
68 + out_dtype: torch.dtype) -> torch.Tensor:
69 return torch._scaled_mm(a,
70 b,
71 scale_a=scale_a,
@@ -74,15 +74,15 @@ def pytorch_fp8_impl_fast_accum(a: torch.tensor, b: torch.tensor,
74 use_fast_accum=True)
75
76
77 -def cutlass_impl(a: torch.tensor, b: torch.tensor, scale_a: torch.tensor,
78 - scale_b: torch.tensor,
79 - out_dtype: torch.dtype) -> torch.tensor:
77 +def cutlass_impl(a: torch.Tensor, b: torch.Tensor, scale_a: torch.Tensor,
78 + scale_b: torch.Tensor,
79 + out_dtype: torch.dtype) -> torch.Tensor:
80 return ops.cutlass_scaled_mm(a, b, scale_a, scale_b, out_dtype=out_dtype)
81
82
83 # bench
84 -def bench_fn(a: torch.tensor, b: torch.tensor, scale_a: torch.tensor,
85 - scale_b: torch.tensor, out_dtype: torch.dtype, label: str,
84 +def bench_fn(a: torch.Tensor, b: torch.Tensor, scale_a: torch.Tensor,
85 + scale_b: torch.Tensor, out_dtype: torch.dtype, label: str,
86 sub_label: str, fn: Callable, description: str) -> TMeasurement:
87
88 min_run_time = 1
vllm/worker/worker.py
+1 -1
@@ -105,7 +105,7 @@ class Worker(LocalOrDistributedWorkerBase):
105 # initialize_cache.
106 self.cache_engine: List[CacheEngine]
107 # Initialize gpu_cache as embedding models don't initialize kv_caches
108 - self.gpu_cache: Optional[List[List[torch.tensor]]] = None
108 + self.gpu_cache: Optional[List[List[torch.Tensor]]] = None
109
110 def init_device(self) -> None:
111 if self.device_config.device.type == "cuda":