A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 23:15 UTC
[Bugfix] Fix typo in scheduler.py (requeset -> request) (#5470)
Michael Goin committed
Jun 12, 2024 at 17:59 UTC
94a07bbdd813a0121d01a852ab03fb2430e73548
1 file changed
+10
-10
vllm/core/scheduler.py
+10
-10
@@ -50,8 +50,8 @@ class SchedulingBudget:
50
"""
51
token_budget: int
52
max_num_seqs: int
53
- _requeset_ids_num_batched_tokens: Set[str] = field(default_factory=set)
54
- _requeset_ids_num_curr_seqs: Set[str] = field(default_factory=set)
53
+ _request_ids_num_batched_tokens: Set[str] = field(default_factory=set)
54
+ _request_ids_num_curr_seqs: Set[str] = field(default_factory=set)
55
_num_batched_tokens: int = 0
56
_num_curr_seqs: int = 0
57
@@ -65,28 +65,28 @@ class SchedulingBudget:
65
return self.token_budget - self.num_batched_tokens
66
67
def add_num_batched_tokens(self, req_id: str, num_batched_tokens: int):
68
- if req_id in self._requeset_ids_num_batched_tokens:
68
+ if req_id in self._request_ids_num_batched_tokens:
69
return
70
71
- self._requeset_ids_num_batched_tokens.add(req_id)
71
+ self._request_ids_num_batched_tokens.add(req_id)
72
self._num_batched_tokens += num_batched_tokens
73
74
def subtract_num_batched_tokens(self, req_id: str,
75
num_batched_tokens: int):
76
- if req_id in self._requeset_ids_num_batched_tokens:
77
- self._requeset_ids_num_batched_tokens.remove(req_id)
76
+ if req_id in self._request_ids_num_batched_tokens:
77
+ self._request_ids_num_batched_tokens.remove(req_id)
78
self._num_batched_tokens -= num_batched_tokens
79
80
def add_num_seqs(self, req_id: str, num_curr_seqs: int):
81
- if req_id in self._requeset_ids_num_curr_seqs:
81
+ if req_id in self._request_ids_num_curr_seqs:
82
return
83
84
- self._requeset_ids_num_curr_seqs.add(req_id)
84
+ self._request_ids_num_curr_seqs.add(req_id)
85
self._num_curr_seqs += num_curr_seqs
86
87
def subtract_num_seqs(self, req_id: str, num_curr_seqs: int):
88
- if req_id in self._requeset_ids_num_curr_seqs:
89
- self._requeset_ids_num_curr_seqs.remove(req_id)
88
+ if req_id in self._request_ids_num_curr_seqs:
89
+ self._request_ids_num_curr_seqs.remove(req_id)
90
self._num_curr_seqs -= num_curr_seqs
91
92
@property