A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
lint: format all python file instead of just source code (#2567)
Simon Mo committed
Jan 23, 2024 at 15:53 UTC
1e4277d2d1eceedbe0d00ba9e2c1bef88145df7b
5 files changed
+10
-18
.github/workflows/yapf.yml
+1
-1
@@ -28,4 +28,4 @@ jobs:
28
pip install toml==0.10.2
29
- name: Running yapf
30
run: |
31
- yapf --diff --recursive vllm tests
31
+ yapf --diff --recursive .
benchmarks/benchmark_serving.py
+7
-13
@@ -92,17 +92,9 @@ async def get_request(
92
await asyncio.sleep(interval)
93
94
95
-async def send_request(
96
- backend: str,
97
- model: str,
98
- api_url: str,
99
- prompt: str,
100
- prompt_len: int,
101
- output_len: int,
102
- best_of: int,
103
- use_beam_search: bool,
104
- pbar: tqdm
105
-) -> None:
95
+async def send_request(backend: str, model: str, api_url: str, prompt: str,
96
+ prompt_len: int, output_len: int, best_of: int,
97
+ use_beam_search: bool, pbar: tqdm) -> None:
98
request_start_time = time.perf_counter()
99
100
headers = {"User-Agent": "Benchmark Client"}
@@ -155,7 +147,6 @@ async def send_request(
147
pbar.update(1)
148
149
158
-
150
async def benchmark(
151
backend: str,
152
model: str,
@@ -217,7 +208,10 @@ if __name__ == "__main__":
208
type=str,
209
default="vllm",
210
choices=["vllm", "tgi"])
220
- parser.add_argument("--protocol", type=str, default="http", choices=["http", "https"])
211
+ parser.add_argument("--protocol",
212
+ type=str,
213
+ default="http",
214
+ choices=["http", "https"])
215
parser.add_argument("--host", type=str, default="localhost")
216
parser.add_argument("--port", type=int, default=8000)
217
parser.add_argument("--endpoint", type=str, default="/generate")
examples/openai_chatcompletion_client.py
-1
@@ -32,6 +32,5 @@ chat_completion = client.chat.completions.create(
32
model=model,
33
)
34
35
-
35
print("Chat completion results:")
36
print(chat_completion)
examples/openai_completion_client.py
+1
-2
@@ -21,8 +21,7 @@ completion = client.completions.create(
21
echo=False,
22
n=2,
23
stream=stream,
24
- logprobs=3
25
-)
24
+ logprobs=3)
25
26
print("Completion results:")
27
if stream:
format.sh
+1
-1
@@ -71,7 +71,7 @@ format_changed() {
71
72
# Format all files
73
format_all() {
74
- yapf --in-place "${YAPF_FLAGS[@]}" "${YAPF_EXCLUDES[@]}" vllm tests
74
+ yapf --in-place "${YAPF_FLAGS[@]}" "${YAPF_EXCLUDES[@]}" .
75
}
76
77
## This flag formats individual files. --files *must* be the first command line