@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c8627cd41b
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 10:45 UTC

[ci][test] use load dummy for testing (#9165)

youkaichao committed Oct 9, 2024 at 00:38 UTC c8627cd41b10747da393b76c382de5ef0eb635a2
3 files changed +20 -1
.buildkite/test-pipeline.yaml
+1 -1
@@ -269,7 +269,7 @@ steps:
269 - csrc/
270 - vllm/model_executor/layers/quantization
271 - tests/quantization
272 - command: pytest -v -s quantization
272 + command: VLLM_TEST_FORCE_LOAD_FORMAT=auto pytest -v -s quantization
273
274 - label: LM Eval Small Models # 53min
275 working_dir: "/vllm-workspace/.buildkite/lm-eval-harness"
tests/utils.py
+17
@@ -16,6 +16,7 @@ import requests
16 from openai.types.completion import Completion
17 from typing_extensions import ParamSpec, assert_never
18
19 +import vllm.envs as envs
20 from tests.models.utils import TextTextLogprobs
21 from vllm.distributed import (ensure_model_parallel_initialized,
22 init_distributed_environment)
@@ -352,10 +353,26 @@ def compare_all_settings(model: str,
353 tokenizer_mode=tokenizer_mode,
354 )
355
356 + can_force_load_format = True
357 +
358 + for args in all_args:
359 + if "--load-format" in args:
360 + can_force_load_format = False
361 + break
362 +
363 prompt = "Hello, my name is"
364 token_ids = tokenizer(prompt).input_ids
365 ref_results: List = []
366 for i, (args, env) in enumerate(zip(all_args, all_envs)):
367 + if can_force_load_format:
368 + # we are comparing the results and
369 + # usually we don't need real weights.
370 + # we force to use dummy weights by default,
371 + # and it should work for most of the cases.
372 + # if not, we can use VLLM_TEST_FORCE_LOAD_FORMAT
373 + # environment variable to force the load format,
374 + # e.g. in quantization tests.
375 + args = args + ["--load-format", envs.VLLM_TEST_FORCE_LOAD_FORMAT]
376 compare_results: List = []
377 results = ref_results if i == 0 else compare_results
378 with RemoteOpenAIServer(model,
vllm/envs.py
+2
@@ -397,6 +397,8 @@ environment_variables: Dict[str, Callable[[], Any]] = {
397 lambda:
398 (os.environ.get("VLLM_TEST_FORCE_FP8_MARLIN", "0").strip().lower() in
399 ("1", "true")),
400 + "VLLM_TEST_FORCE_LOAD_FORMAT":
401 + lambda: os.getenv("VLLM_TEST_FORCE_LOAD_FORMAT", "dummy"),
402
403 # Time in ms for the zmq client to wait for a response from the backend
404 # server for simple data operations