A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 10:45 UTC
[ci][test] use load dummy for testing (#9165)
youkaichao committed
Oct 9, 2024 at 00:38 UTC
c8627cd41b10747da393b76c382de5ef0eb635a2
3 files changed
+20
-1
.buildkite/test-pipeline.yaml
+1
-1
@@ -269,7 +269,7 @@ steps:
269
- csrc/
270
- vllm/model_executor/layers/quantization
271
- tests/quantization
272
- command: pytest -v -s quantization
272
+ command: VLLM_TEST_FORCE_LOAD_FORMAT=auto pytest -v -s quantization
273
274
- label: LM Eval Small Models # 53min
275
working_dir: "/vllm-workspace/.buildkite/lm-eval-harness"
tests/utils.py
+17
@@ -16,6 +16,7 @@ import requests
16
from openai.types.completion import Completion
17
from typing_extensions import ParamSpec, assert_never
18
19
+import vllm.envs as envs
20
from tests.models.utils import TextTextLogprobs
21
from vllm.distributed import (ensure_model_parallel_initialized,
22
init_distributed_environment)
@@ -352,10 +353,26 @@ def compare_all_settings(model: str,
353
tokenizer_mode=tokenizer_mode,
354
)
355
356
+ can_force_load_format = True
357
+
358
+ for args in all_args:
359
+ if "--load-format" in args:
360
+ can_force_load_format = False
361
+ break
362
+
363
prompt = "Hello, my name is"
364
token_ids = tokenizer(prompt).input_ids
365
ref_results: List = []
366
for i, (args, env) in enumerate(zip(all_args, all_envs)):
367
+ if can_force_load_format:
368
+ # we are comparing the results and
369
+ # usually we don't need real weights.
370
+ # we force to use dummy weights by default,
371
+ # and it should work for most of the cases.
372
+ # if not, we can use VLLM_TEST_FORCE_LOAD_FORMAT
373
+ # environment variable to force the load format,
374
+ # e.g. in quantization tests.
375
+ args = args + ["--load-format", envs.VLLM_TEST_FORCE_LOAD_FORMAT]
376
compare_results: List = []
377
results = ref_results if i == 0 else compare_results
378
with RemoteOpenAIServer(model,
vllm/envs.py
+2
@@ -397,6 +397,8 @@ environment_variables: Dict[str, Callable[[], Any]] = {
397
lambda:
398
(os.environ.get("VLLM_TEST_FORCE_FP8_MARLIN", "0").strip().lower() in
399
("1", "true")),
400
+ "VLLM_TEST_FORCE_LOAD_FORMAT":
401
+ lambda: os.getenv("VLLM_TEST_FORCE_LOAD_FORMAT", "dummy"),
402
403
# Time in ms for the zmq client to wait for a response from the backend
404
# server for simple data operations