@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / a88081bf76
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[CI] Disable non-lazy string operation on logging (#4326)

Co-authored-by: Danny Guinther <dguinther@neuralmagic.com>

SangBin Cho committed Apr 26, 2024 at 16:16 UTC a88081bf768fcc1c662e4f588bd01ca9ddcc6aad
31 files changed +176 -149
docs/source/conf.py
+3 -2
@@ -98,9 +98,10 @@ autodoc_mock_imports = [
98 for mock_target in autodoc_mock_imports:
99 if mock_target in sys.modules:
100 logger.info(
101 - f"Potentially problematic mock target ({mock_target}) found; "
101 + "Potentially problematic mock target (%s) found; "
102 "autodoc_mock_imports cannot mock modules that have already "
103 - "been loaded into sys.modules when the sphinx build starts.")
103 + "been loaded into sys.modules when the sphinx build starts.",
104 + mock_target)
105
106
107 class MockedClassDocumenter(autodoc.ClassDocumenter):
pyproject.toml
+1
@@ -32,6 +32,7 @@ select = [
32 "SIM",
33 # isort
34 # "I",
35 + "G",
36 ]
37 ignore = [
38 # star imports
setup.py
+4 -3
@@ -63,7 +63,7 @@ class cmake_build_ext(build_ext):
63 num_jobs = os.environ.get("MAX_JOBS", None)
64 if num_jobs is not None:
65 num_jobs = int(num_jobs)
66 - logger.info(f"Using MAX_JOBS={num_jobs} as the number of jobs.")
66 + logger.info("Using MAX_JOBS=%d as the number of jobs.", num_jobs)
67 else:
68 try:
69 # os.sched_getaffinity() isn't universally available, so fall
@@ -81,8 +81,9 @@ class cmake_build_ext(build_ext):
81 nvcc_threads = os.getenv("NVCC_THREADS", None)
82 if nvcc_threads is not None:
83 nvcc_threads = int(nvcc_threads)
84 - logger.info(f"Using NVCC_THREADS={nvcc_threads} as the number"
85 - " of nvcc threads.")
84 + logger.info(
85 + "Using NVCC_THREADS=%d as the number of nvcc threads.",
86 + nvcc_threads)
87 else:
88 nvcc_threads = 1
89 num_jobs = max(1, num_jobs // nvcc_threads)
vllm/config.py
+8 -8
@@ -167,9 +167,9 @@ class ModelConfig:
167 f"supported in ROCm.")
168 if self.quantization != "marlin":
169 logger.warning(
170 - f"{self.quantization} quantization is not fully "
170 + "%s quantization is not fully "
171 "optimized yet. The speed can be slower than "
172 - "non-quantized models.")
172 + "non-quantized models.", self.quantization)
173
174 def _verify_cuda_graph(self) -> None:
175 if self.max_context_len_to_capture is None:
@@ -360,7 +360,7 @@ class CacheConfig:
360 if cpu_memory_usage > 0.7 * total_cpu_memory:
361 raise ValueError("Too large swap space. " + msg)
362 elif cpu_memory_usage > 0.4 * total_cpu_memory:
363 - logger.warning("Possibly too large swap space. " + msg)
363 + logger.warning("Possibly too large swap space. %s", msg)
364
365
366 @dataclass
@@ -898,8 +898,8 @@ class LoRAConfig:
898 "awq", "gptq"
899 ]:
900 # TODO support marlin and squeezellm
901 - logger.warning(f"{model_config.quantization} quantization is not "
902 - "tested with LoRA yet.")
901 + logger.warning("%s quantization is not tested with LoRA yet.",
902 + model_config.quantization)
903
904 def verify_with_scheduler_config(self, scheduler_config: SchedulerConfig):
905 if scheduler_config.max_num_batched_tokens > 65528:
@@ -1008,7 +1008,7 @@ def _get_and_verify_dtype(
1008 pass
1009 else:
1010 # Casting between float16 and bfloat16 is allowed with a warning.
1011 - logger.warning(f"Casting {config_dtype} to {torch_dtype}.")
1011 + logger.warning("Casting %s to %s.", config_dtype, torch_dtype)
1012
1013 return torch_dtype
1014
@@ -1051,8 +1051,8 @@ def _get_and_verify_max_len(
1051 logger.warning(
1052 "The model's config.json does not contain any of the following "
1053 "keys to determine the original maximum length of the model: "
1054 - f"{possible_keys}. Assuming the model's maximum length is "
1055 - f"{default_max_len}.")
1054 + "%d. Assuming the model's maximum length is %d.", possible_keys,
1055 + default_max_len)
1056 derived_max_model_len = default_max_len
1057
1058 rope_scaling = getattr(hf_config, "rope_scaling", None)
vllm/core/scheduler.py
+6 -4
@@ -617,8 +617,9 @@ class Scheduler:
617
618 if num_new_tokens > self.prompt_limit:
619 logger.warning(
620 - f"Input prompt ({num_new_tokens} tokens) is too long"
621 - f" and exceeds limit of {self.prompt_limit}")
620 + "Input prompt (%d tokens) is too long"
621 + " and exceeds limit of %d", num_new_tokens,
622 + self.prompt_limit)
623 for seq in waiting_seqs:
624 seq.status = SequenceStatus.FINISHED_IGNORED
625 ignored_seq_groups.append(seq_group)
@@ -631,8 +632,9 @@ class Scheduler:
632 break
633 elif can_allocate == AllocStatus.NEVER:
634 logger.warning(
634 - f"Input prompt ({num_new_tokens} tokens) is too long"
635 - f" and exceeds the capacity of block_manager")
635 + "Input prompt (%d tokens) is too long"
636 + " and exceeds the capacity of block_manager",
637 + num_new_tokens)
638 for seq in waiting_seqs:
639 seq.status = SequenceStatus.FINISHED_IGNORED
640 ignored_seq_groups.append(seq_group)
vllm/distributed/device_communicators/custom_all_reduce.py
+4 -4
@@ -37,7 +37,7 @@ def init_custom_ar() -> None:
37 return
38
39 if world_size not in _SUPPORTED_WORLD_SIZES:
40 - logger.warn(
40 + logger.warning(
41 "Custom allreduce is disabled due to an unsupported world size: "
42 "%d. Supported world sizes: %s. To silence this warning, specify"
43 " disable_custom_all_reduce=True explicitly.", world_size,
@@ -47,7 +47,7 @@ def init_custom_ar() -> None:
47 # note: num dev can be larger than world_size if we're only using
48 # first few GPUs
49 if num_dev < world_size:
50 - logger.warn(
50 + logger.warning(
51 "Cannot test GPU P2P because not all GPUs are visible to the "
52 "current process. This might be the case if 'CUDA_VISIBLE_DEVICES'"
53 " is set.")
@@ -62,7 +62,7 @@ def init_custom_ar() -> None:
62 # this checks hardware and driver support for NVLink
63 full_nvlink = _is_full_nvlink(device_ids)
64 if world_size > 2 and not full_nvlink:
65 - logger.warn(
65 + logger.warning(
66 "Custom allreduce is disabled because it's not supported on more"
67 " than two PCIe-only GPUs. To silence this warning, specify"
68 " disable_custom_all_reduce=True explicitly.")
@@ -71,7 +71,7 @@ def init_custom_ar() -> None:
71 # this is expensive to compute at the first time
72 # then we cache the result
73 if not _can_p2p(rank, world_size):
74 - logger.warn(
74 + logger.warning(
75 "Custom allreduce is disabled because your platform lacks GPU P2P"
76 " capability or P2P test failed. To silence this warning, specify"
77 " disable_custom_all_reduce=True explicitly.")
vllm/distributed/device_communicators/pynccl.py
+8 -7
@@ -43,15 +43,16 @@ try:
43 nccl = ctypes.CDLL(so_file)
44 except Exception as e:
45 logger.error(
46 - f"Failed to load NCCL library from {so_file} ."
46 + "Failed to load NCCL library from %s ."
47 "It is expected if you are not running on NVIDIA/AMD GPUs."
48 "Otherwise, the nccl library might not exist, be corrupted "
49 - f"or it does not support the current platform {platform.platform()}."
50 - f"One solution is to download libnccl2 version 2.18 from "
51 - f"https://developer.download.nvidia.com/compute/cuda/repos/ "
52 - f"and extract the libnccl.so.2 file. If you already have the "
53 - f"library, please set the environment variable VLLM_NCCL_SO_PATH"
54 - " to point to the correct nccl library path.")
49 + "or it does not support the current platform %s."
50 + "One solution is to download libnccl2 version 2.18 from "
51 + "https://developer.download.nvidia.com/compute/cuda/repos/ "
52 + "and extract the libnccl.so.2 file. If you already have the "
53 + "library, please set the environment variable VLLM_NCCL_SO_PATH"
54 + " to point to the correct nccl library path.", so_file,
55 + platform.platform())
56 raise e
57
58 # === export types and functions from nccl to Python ===
vllm/distributed/device_communicators/pynccl_utils.py
+2 -2
@@ -14,7 +14,7 @@ try:
14 except Exception as e:
15 # in non-NVIDIA environments, we can't import the nccl module
16 # e.g. when running on machines with AMD GPUs
17 - logger.info(f"Failed to import NCCL library: {e}")
17 + logger.info("Failed to import NCCL library: %s", e)
18 logger.info("It is expected if you are not running on NVIDIA GPUs.")
19 pass
20
@@ -40,7 +40,7 @@ def set_pynccl_stream(stream: torch.cuda.Stream):
40 def init_process_group(group: Optional[ProcessGroup] = None) -> None:
41 assert not is_initialized()
42 global comm
43 - logger.info(f"vLLM is using nccl=={ncclGetVersion()}")
43 + logger.info("vLLM is using nccl==%s", ncclGetVersion())
44 comm = NCCLCommunicator(group=group)
45
46
vllm/distributed/parallel_state.py
+4 -2
@@ -57,8 +57,10 @@ def init_distributed_environment(
57 local_rank: int = -1,
58 backend: str = "nccl",
59 ):
60 - logger.debug(f"{world_size=} {rank=} {local_rank=} "
61 - f"{distributed_init_method=} {backend=}")
60 + logger.debug(
61 + "world_size=%d rank=%d local_rank=%d "
62 + "distributed_init_method=%s backend=%s", world_size, rank, local_rank,
63 + distributed_init_method, backend)
64 if not torch.distributed.is_initialized():
65 assert distributed_init_method is not None, (
66 "distributed_init_method must be provided when initializing "
vllm/distributed/utils.py
+2 -2
@@ -112,7 +112,7 @@ def gpu_p2p_access_check(i: int, j: int) -> bool:
112 and (not os.path.exists(path)):
113 # only the local master process (with local_rank == 0) can
114 # enter this block to calculate the cache
115 - logger.info(f"generating GPU P2P access cache for in {path}")
115 + logger.info("generating GPU P2P access cache for in %s", path)
116 cache = {}
117 for _i in range(num_dev):
118 for _j in range(num_dev):
@@ -126,7 +126,7 @@ def gpu_p2p_access_check(i: int, j: int) -> bool:
126 if is_distributed:
127 cpu_world_group = get_cpu_world_group()
128 dist.barrier(cpu_world_group)
129 - logger.info(f"reading GPU P2P access cache from {path}")
129 + logger.info("reading GPU P2P access cache from %s", path)
130 with open(path, "r") as f:
131 cache = json.load(f)
132 _gpu_p2p_access_cache = cache
vllm/engine/async_llm_engine.py
+9 -9
@@ -117,7 +117,7 @@ class RequestTracker:
117 self._request_streams[request_id].put(request_output)
118 if request_output.finished:
119 if verbose:
120 - logger.info(f"Finished request {request_id}.")
120 + logger.info("Finished request %s.", request_id)
121 self.abort_request(request_id)
122
123 def process_exception(self,
@@ -128,7 +128,7 @@ class RequestTracker:
128 """Propagate an exception from the engine."""
129 self._request_streams[request_id].put(exception)
130 if verbose:
131 - logger.info(f"Finished request {request_id}.")
131 + logger.info("Finished request %s.", request_id)
132 self.abort_request(request_id)
133
134 def add_request(self, request_id: str,
@@ -151,7 +151,7 @@ class RequestTracker:
151 def abort_request(self, request_id: str, *, verbose: bool = False) -> None:
152 """Abort a request during next background loop iteration."""
153 if verbose:
154 - logger.info(f"Aborted request {request_id}.")
154 + logger.info("Aborted request %s.", request_id)
155
156 self._finished_requests.put_nowait(request_id)
157
@@ -521,11 +521,11 @@ class AsyncLLMEngine:
521 if shortened_token_ids is not None:
522 shortened_token_ids = shortened_token_ids[:self.
523 max_log_len]
524 - logger.info(f"Received request {request_id}: "
525 - f"prompt: {shortened_prompt!r}, "
526 - f"sampling_params: {sampling_params}, "
527 - f"prompt_token_ids: {shortened_token_ids}, "
528 - f"lora_request: {lora_request}.")
524 + logger.info(
525 + "Received request %s: prompt: %r, "
526 + "sampling_params: %s, prompt_token_ids: %s, "
527 + "lora_request: %s.", request_id, shortened_prompt,
528 + sampling_params, shortened_token_ids, lora_request)
529
530 if not self.is_running:
531 if self.start_engine_loop:
@@ -717,4 +717,4 @@ class AsyncLLMEngine:
717 raise RuntimeError("Engine is dead.") from e
718 else:
719 await self.engine.check_health_async()
720 - logger.debug(f"Health check took {time.perf_counter()-t}s")
720 + logger.debug("Health check took %fs", time.perf_counter() - t)
vllm/engine/llm_engine.py
+36 -25
@@ -96,29 +96,38 @@ class LLMEngine:
96 usage_context: UsageContext = UsageContext.ENGINE_CONTEXT,
97 ) -> None:
98 logger.info(
99 - f"Initializing an LLM engine (v{vllm.__version__}) with config: "
100 - f"model={model_config.model!r}, "
101 - f"speculative_config={speculative_config!r}, "
102 - f"tokenizer={model_config.tokenizer!r}, "
103 - f"skip_tokenizer_init={model_config.skip_tokenizer_init}, "
104 - f"tokenizer_mode={model_config.tokenizer_mode}, "
105 - f"revision={model_config.revision}, "
106 - f"tokenizer_revision={model_config.tokenizer_revision}, "
107 - f"trust_remote_code={model_config.trust_remote_code}, "
108 - f"dtype={model_config.dtype}, "
109 - f"max_seq_len={model_config.max_model_len}, "
110 - f"download_dir={load_config.download_dir!r}, "
111 - f"load_format={load_config.load_format}, "
112 - f"tensor_parallel_size={parallel_config.tensor_parallel_size}, "
113 - f"disable_custom_all_reduce="
114 - f"{parallel_config.disable_custom_all_reduce}, "
115 - f"quantization={model_config.quantization}, "
116 - f"enforce_eager={model_config.enforce_eager}, "
117 - f"kv_cache_dtype={cache_config.cache_dtype}, "
118 - f"quantization_param_path={model_config.quantization_param_path}, "
119 - f"device_config={device_config.device}, "
120 - f"decoding_config={decoding_config!r}, "
121 - f"seed={model_config.seed})")
99 + "Initializing an LLM engine (v%s) with config: "
100 + "model=%r, speculative_config=%r, tokenizer=%r, "
101 + "skip_tokenizer_init=%s, tokenizer_mode=%s, revision=%s, "
102 + "tokenizer_revision=%s, trust_remote_code=%s, dtype=%s, "
103 + "max_seq_len=%d, download_dir=%r, load_format=%s, "
104 + "tensor_parallel_size=%d, disable_custom_all_reduce=%s"
105 + "quantization=%s, enforce_eager=%s, kv_cache_dtype=%s, "
106 + "quantization_param_path=%s, device_config=%s, "
107 + "decoding_config=%r, seed=%d)",
108 + vllm.__version__,
109 + model_config.model,
110 + speculative_config,
111 + model_config.tokenizer,
112 + model_config.skip_tokenizer_init,
113 + model_config.tokenizer_mode,
114 + model_config.revision,
115 + model_config.tokenizer_revision,
116 + model_config.trust_remote_code,
117 + model_config.dtype,
118 + model_config.max_model_len,
119 + load_config.download_dir,
120 + load_config.load_format,
121 + parallel_config.tensor_parallel_size,
122 + parallel_config.disable_custom_all_reduce,
123 + model_config.quantization,
124 + model_config.enforce_eager,
125 + cache_config.cache_dtype,
126 + model_config.quantization_param_path,
127 + device_config.device,
128 + decoding_config,
129 + model_config.seed,
130 + )
131 # TODO(woosuk): Print more configs in debug mode.
132
133 self.model_config = model_config
@@ -237,8 +246,10 @@ class LLMEngine:
246
247 if self.cache_config.num_gpu_blocks_override is not None:
248 num_gpu_blocks_override = self.cache_config.num_gpu_blocks_override
240 - logger.info(f"Overriding {num_gpu_blocks=} with "
241 - f"{num_gpu_blocks_override=}")
249 + logger.info(
250 + "Overriding num_gpu_blocks=%d with "
251 + "num_gpu_blocks_override=%d", num_gpu_blocks,
252 + num_gpu_blocks_override)
253 num_gpu_blocks = num_gpu_blocks_override
254
255 self.cache_config.num_gpu_blocks = num_gpu_blocks
vllm/engine/metrics.py
+13 -8
@@ -227,14 +227,19 @@ class StatLogger:
227
228 # Log to stdout.
229 logger.info(
230 - f"Avg prompt throughput: {prompt_throughput:.1f} tokens/s, "
231 - f"Avg generation throughput: "
232 - f"{generation_throughput:.1f} tokens/s, "
233 - f"Running: {stats.num_running} reqs, "
234 - f"Swapped: {stats.num_swapped} reqs, "
235 - f"Pending: {stats.num_waiting} reqs, "
236 - f"GPU KV cache usage: {stats.gpu_cache_usage * 100:.1f}%, "
237 - f"CPU KV cache usage: {stats.cpu_cache_usage * 100:.1f}%")
230 + "Avg prompt throughput: %.1f tokens/s, "
231 + "Avg generation throughput: %.1f tokens/s, "
232 + "Running: %d reqs, Swapped: %d reqs, "
233 + "Pending: %d reqs, GPU KV cache usage: %.1f%, "
234 + "CPU KV cache usage: %.1f%",
235 + prompt_throughput,
236 + generation_throughput,
237 + stats.num_running,
238 + stats.num_swapped,
239 + stats.num_waiting,
240 + stats.gpu_cache_usage * 100,
241 + stats.cpu_cache_usage * 100,
242 + )
243
244 # Reset tracked stats for next interval.
245 self.num_prompt_tokens = []
vllm/entrypoints/openai/api_server.py
+2 -2
@@ -148,8 +148,8 @@ if __name__ == "__main__":
148 raise ValueError(f"Invalid middleware {middleware}. "
149 f"Must be a function or a class.")
150
151 - logger.info(f"vLLM API server version {vllm.__version__}")
152 - logger.info(f"args: {args}")
151 + logger.info("vLLM API server version %s", vllm.__version__)
152 + logger.info("args: %s", args)
153
154 if args.served_model_name is not None:
155 served_model_names = args.served_model_name
vllm/entrypoints/openai/serving_chat.py
+5 -6
@@ -57,8 +57,7 @@ class OpenAIServingChat(OpenAIServing):
57 tokenize=False,
58 add_generation_prompt=request.add_generation_prompt)
59 except Exception as e:
60 - logger.error(
61 - f"Error in applying chat template from request: {str(e)}")
60 + logger.error("Error in applying chat template from request: %s", e)
61 return self.create_error_response(str(e))
62
63 request_id = f"cmpl-{random_uuid()}"
@@ -338,11 +337,11 @@ class OpenAIServingChat(OpenAIServing):
337 tokenizer.chat_template = codecs.decode(
338 chat_template, "unicode_escape")
339
341 - logger.info(
342 - f"Using supplied chat template:\n{tokenizer.chat_template}")
340 + logger.info("Using supplied chat template:\n%s",
341 + tokenizer.chat_template)
342 elif tokenizer.chat_template is not None:
344 - logger.info(
345 - f"Using default chat template:\n{tokenizer.chat_template}")
343 + logger.info("Using default chat template:\n%s",
344 + tokenizer.chat_template)
345 else:
346 logger.warning(
347 "No chat template provided. Chat API will not work.")
vllm/executor/cpu_executor.py
+1 -1
@@ -69,7 +69,7 @@ class CPUExecutor(ExecutorBase):
69 # NOTE: `cpu block` for CPU backend is located on CPU memory but is
70 # referred as `gpu block`. Because we want to reuse the existing block
71 # management procedure.
72 - logger.info(f"# CPU blocks: {num_gpu_blocks}")
72 + logger.info("# CPU blocks: %d", num_gpu_blocks)
73 self.driver_worker.initialize_cache(num_gpu_blocks, num_cpu_blocks)
74
75 def execute_model(self,
vllm/executor/gpu_executor.py
+2 -2
@@ -116,8 +116,8 @@ class GPUExecutor(ExecutorBase):
116 # NOTE: This is logged in the executor because there can be >1 worker
117 # with other executors. We could log in the engine level, but work
118 # remains to abstract away the device for non-GPU configurations.
119 - logger.info(f"# GPU blocks: {num_gpu_blocks}, "
120 - f"# CPU blocks: {num_cpu_blocks}")
119 + logger.info("# GPU blocks: %d, # CPU blocks: %d", num_gpu_blocks,
120 + num_cpu_blocks)
121
122 self.driver_worker.initialize_cache(num_gpu_blocks, num_cpu_blocks)
123
vllm/executor/ray_gpu_executor.py
+2 -2
@@ -214,8 +214,8 @@ class RayGPUExecutor(ExecutorBase):
214 # NOTE: We log here to avoid multiple logs when number of workers is
215 # greater than one. We could log in the engine, but not all executors
216 # have GPUs.
217 - logger.info(f"# GPU blocks: {num_gpu_blocks}, "
218 - f"# CPU blocks: {num_cpu_blocks}")
217 + logger.info("# GPU blocks: %d, # CPU blocks: %d", num_gpu_blocks,
218 + num_cpu_blocks)
219
220 self.cache_config.num_gpu_blocks = num_gpu_blocks
221 self.cache_config.num_cpu_blocks = num_cpu_blocks
vllm/executor/ray_utils.py
+3 -3
@@ -43,9 +43,9 @@ try:
43 return output
44
45 except ImportError as e:
46 - logger.warning(f"Failed to import Ray with {e!r}. "
47 - "For distributed inference, please install Ray with "
48 - "`pip install ray`.")
46 + logger.warning(
47 + "Failed to import Ray with %r. For distributed inference, "
48 + "please install Ray with `pip install ray`.", e)
49 ray = None # type: ignore
50 RayWorkerWrapper = None # type: ignore
51
vllm/logger.py
+1 -1
@@ -126,7 +126,7 @@ def enable_trace_function_call(log_file_path: str,
126 "VLLM_TRACE_FUNCTION is enabled. It will record every"
127 " function executed by Python. This will slow down the code. It "
128 "is suggested to be used for debugging hang or crashes only.")
129 - logger.info(f"Trace frame log is saved to {log_file_path}")
129 + logger.info("Trace frame log is saved to %s", log_file_path)
130 if root_dir is None:
131 # by default, this is the vllm root directory
132 root_dir = os.path.dirname(os.path.dirname(__file__))
vllm/lora/models.py
+3 -3
@@ -345,8 +345,8 @@ class LoRAModelManager:
345 index, _ = first_free_slot
346 self._active_loras[lora_id] = None
347 lora_model = self._registered_loras[lora_id]
348 - logger.debug(
349 - f"Activating LoRA. int id: {lora_model.id}, slot index: {index}")
348 + logger.debug("Activating LoRA. int id: %d, slot index: %d",
349 + lora_model.id, index)
350 self.lora_index_to_id[index] = lora_model.id
351 for module_name, module in self.modules.items():
352 module_lora = lora_model.get_lora(module_name)
@@ -567,7 +567,7 @@ class LoRALRUCache(LRUCache[LoRAModel]):
567 self.deactivate_lora_fn = deactivate_lora_fn
568
569 def _on_remove(self, key: int, value: LoRAModel):
570 - logger.debug(f"Removing LoRA. int id: {key}")
570 + logger.debug("Removing LoRA. int id: %d", key)
571 self.deactivate_lora_fn(key)
572 return super()._on_remove(key, value)
573
vllm/model_executor/layers/fused_moe/fused_moe.py
+2 -2
@@ -296,8 +296,8 @@ def get_moe_configs(E: int, N: int,
296 os.path.dirname(os.path.realpath(__file__)), "configs", json_file_name)
297 if os.path.exists(config_file_path):
298 with open(config_file_path) as f:
299 - logger.info(
300 - f"Using configuration from {config_file_path} for MoE layer.")
299 + logger.info("Using configuration from %s for MoE layer.",
300 + config_file_path)
301 # If a configuration has been found, return it
302 return {int(key): val for key, val in json.load(f).items()}
303
vllm/model_executor/model_loader/tensorizer.py
+4 -4
@@ -334,10 +334,10 @@ class TensorizerAgent:
334 per_second = convert_bytes(deserializer.total_tensor_bytes / duration)
335 after_mem = get_mem_usage()
336 deserializer.close()
337 - logger.info(f"Deserialized {total_bytes_str} in "
338 - f"{end - start:0.2f}s, {per_second}/s")
339 - logger.info(f"Memory usage before: {before_mem}")
340 - logger.info(f"Memory usage after: {after_mem}")
337 + logger.info("Deserialized %s in %0.2fs, %f/s", total_bytes_str,
338 + end - start, per_second)
339 + logger.info("Memory usage before: %s", before_mem)
340 + logger.info("Memory usage after: %s", after_mem)
341
342 self._check_tensors_on_meta_device()
343 self._resize_lora_embeddings()
vllm/model_executor/model_loader/weight_utils.py
+7 -7
@@ -190,7 +190,7 @@ def download_weights_from_hf(model_name_or_path: str,
190 allow_patterns = [pattern]
191 break
192
193 - logger.info(f"Using model weights format {allow_patterns}")
193 + logger.info("Using model weights format %s", allow_patterns)
194 # Use file lock to prevent multiple processes from
195 # downloading the same model weights at the same time.
196 with get_lock(model_name_or_path, cache_dir):
@@ -310,17 +310,17 @@ def kv_cache_scales_loader(
310 return layer_scales_map.items()
311
312 except FileNotFoundError:
313 - logger.error(f"File or directory '{filename}' not found.")
313 + logger.error("File or directory '%s' not found.", filename)
314 except json.JSONDecodeError:
315 - logger.error(f"Error decoding JSON in file '{filename}'.")
315 + logger.error("Error decoding JSON in file '%s'.", filename)
316 except Exception as e:
317 - logger.error(f"An error occurred while reading '{filename}': {e}")
317 + logger.error("An error occurred while reading '%s': %s", filename, e)
318 # This section is reached if and only if any of the excepts are hit
319 # Return an empty iterable (list) => no KV cache scales are loaded
320 # which ultimately defaults to 1.0 scales
321 - logger.warning("Defaulting to KV cache scaling factors = 1.0 "
322 - f"for all layers in TP rank {tp_rank} "
323 - "as an error occurred during loading.")
321 + logger.warning(
322 + "Defaulting to KV cache scaling factors = 1.0 for all "
323 + "layers in TP rank %d as an error occurred during loading.", tp_rank)
324 return []
325
326
vllm/model_executor/models/__init__.py
+5 -5
@@ -91,8 +91,8 @@ class ModelRegistry:
91 "ROCm for now.")
92 if model_arch in _ROCM_PARTIALLY_SUPPORTED_MODELS:
93 logger.warning(
94 - f"Model architecture {model_arch} is partially supported "
95 - "by ROCm: " + _ROCM_PARTIALLY_SUPPORTED_MODELS[model_arch])
94 + "Model architecture %s is partially supported by ROCm: %s",
95 + model_arch, _ROCM_PARTIALLY_SUPPORTED_MODELS[model_arch])
96
97 module_name, model_cls_name = _MODELS[model_arch]
98 module = importlib.import_module(
@@ -107,9 +107,9 @@ class ModelRegistry:
107 def register_model(model_arch: str, model_cls: Type[nn.Module]):
108 if model_arch in _MODELS:
109 logger.warning(
110 - f"Model architecture {model_arch} is already registered, "
111 - "and will be overwritten by the new model "
112 - f"class {model_cls.__name__}.")
110 + "Model architecture %s is already registered, and will be "
111 + "overwritten by the new model class %s.", model_arch,
112 + model_cls.__name__)
113 global _OOT_MODELS
114 _OOT_MODELS[model_arch] = model_cls
115
vllm/model_executor/models/gemma.py
+3 -3
@@ -55,10 +55,10 @@ def _get_gemma_act_fn(
55 "in the config JSON file when it was initially released. "
56 "Changing the activation function to approximate GeLU "
57 "(`gelu_pytorch_tanh`). If you want to use the legacy "
58 - f"`{hidden_act}`, edit the config JSON to set "
59 - f"`hidden_activation={hidden_act}` instead of `hidden_act`. "
58 + "`%s`, edit the config JSON to set "
59 + "`hidden_activation=%s` instead of `hidden_act`. "
60 "See https://github.com/huggingface/transformers/pull/29402 "
61 - "for more details.")
61 + "for more details.", hidden_act, hidden_act)
62 return GeluAndMul(approximate="tanh")
63 elif hidden_activation == "gelu_pytorch_tanh":
64 return GeluAndMul(approximate="tanh")
vllm/spec_decode/spec_decode_worker.py
+2 -1
@@ -183,7 +183,8 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
183 "speculative decoding "
184 "requires non-None seq_group_metadata_list")
185
186 - logger.info(f"spec_decode_worker.execute_model {num_lookahead_slots=}")
186 + logger.info("spec_decode_worker.execute_model num_lookahead_slots=%d",
187 + num_lookahead_slots)
188
189 # If no spec tokens, call the proposer and scorer workers normally.
190 # Used for prefill.
vllm/transformers_utils/configs/dbrx.py
+7 -6
@@ -72,9 +72,10 @@ class DbrxAttentionConfig(PretrainedConfig):
72 and config_dict["model_type"] != cls.model_type
73 ):
74 logger.warning(
75 - f"You are using a model of type {config_dict['model_type']} to instantiate a model of type "
76 - + f"{cls.model_type}. This is not supported for all configurations of models and can yield errors."
77 - )
75 + "You are using a model of type %s to instantiate a model of "
76 + "type %s. This is not supported for all configurations of "
77 + "models and can yield errors.",
78 + config_dict["model_type"], cls.model_type)
79
80 return cls.from_dict(config_dict, **kwargs)
81
@@ -151,9 +152,9 @@ class DbrxFFNConfig(PretrainedConfig):
152 and config_dict["model_type"] != cls.model_type
153 ):
154 logger.warning(
154 - f"You are using a model of type {config_dict['model_type']} to instantiate a model of type "
155 - + f"{cls.model_type}. This is not supported for all configurations of models and can yield errors."
156 - )
155 + "You are using a model of type %s to instantiate a model of "
156 + "type %s. This is not supported for all "
157 + "configurations of models and can yield errors.", config_dict["model_type"], cls.model_type)
158
159 return cls.from_dict(config_dict, **kwargs)
160
vllm/transformers_utils/tokenizer.py
+2 -3
@@ -138,9 +138,8 @@ def get_lora_tokenizer(lora_request: LoRARequest, *args,
138 # No tokenizer was found in the LoRA folder,
139 # use base model tokenizer
140 logger.warning(
141 - f"No tokenizer found in {lora_request.lora_local_path}, "
142 - "using base model tokenizer instead. "
143 - f"(Exception: {str(e)})")
141 + "No tokenizer found in %s, using base model tokenizer instead. "
142 + "(Exception: %s)", lora_request.lora_local_path, e)
143 tokenizer = None
144 return tokenizer
145
vllm/utils.py
+11 -9
@@ -289,8 +289,9 @@ def get_open_port() -> int:
289 def update_environment_variables(envs: Dict[str, str]):
290 for k, v in envs.items():
291 if k in os.environ and os.environ[k] != v:
292 - logger.warning(f"Overwriting environment variable {k} "
293 - f"from '{os.environ[k]}' to '{v}'")
292 + logger.warning(
293 + "Overwriting environment variable %s "
294 + "from '%s' to '%s'", k, os.environ[k], v)
295 os.environ[k] = v
296
297
@@ -310,11 +311,12 @@ def get_nvcc_cuda_version() -> Optional[Version]:
311 if not cuda_home:
312 cuda_home = '/usr/local/cuda'
313 if os.path.isfile(cuda_home + '/bin/nvcc'):
313 - logger.info(f'CUDA_HOME is not found in the environment. '
314 - f'Using {cuda_home} as CUDA_HOME.')
314 + logger.info(
315 + 'CUDA_HOME is not found in the environment. '
316 + 'Using %s as CUDA_HOME.', cuda_home)
317 else:
316 - logger.warning(
317 - f'Not found nvcc in {cuda_home}. Skip cuda version check!')
318 + logger.warning('Not found nvcc in %s. Skip cuda version check!',
319 + cuda_home)
320 return None
321 nvcc_output = subprocess.check_output([cuda_home + "/bin/nvcc", "-V"],
322 universal_newlines=True)
@@ -599,8 +601,8 @@ def find_nccl_library():
601 # manually load the nccl library
602 if so_file:
603 logger.info(
602 - f"Found nccl from environment variable VLLM_NCCL_SO_PATH={so_file}"
603 - )
604 + "Found nccl from environment variable VLLM_NCCL_SO_PATH=%s",
605 + so_file)
606 else:
607 if torch.version.cuda is not None:
608 so_file = vllm_nccl_path or find_library("libnccl.so.2")
@@ -608,7 +610,7 @@ def find_nccl_library():
610 so_file = find_library("librccl.so.1")
611 else:
612 raise ValueError("NCCL only supports CUDA and ROCm backends.")
611 - logger.info(f"Found nccl from library {so_file}")
613 + logger.info("Found nccl from library %s", so_file)
614 return so_file
615
616
vllm/worker/model_runner.py
+14 -13
@@ -170,8 +170,8 @@ class ModelRunner:
170 )
171
172 self.model_memory_usage = m.consumed_memory
173 - logger.info(f"Loading model weights took "
174 - f"{self.model_memory_usage / float(2**30):.4f} GB")
173 + logger.info("Loading model weights took %.4f GB",
174 + self.model_memory_usage / float(2**30))
175
176 if self.lora_config:
177 assert hasattr(self.model, "supported_lora_modules"
@@ -196,18 +196,19 @@ class ModelRunner:
196 self.model.load_kv_cache_scales(
197 self.model_config.quantization_param_path)
198 else:
199 - raise RuntimeError("Using FP8 KV cache and scaling "
200 - "factors provided but model "
201 - f"{self.model.__class__} does not "
202 - "support loading scaling factors.")
199 + raise RuntimeError(
200 + "Using FP8 KV cache and scaling factors provided but "
201 + "model %s does not support loading scaling factors.",
202 + self.model.__class__)
203 else:
204 - logger.warn("Using FP8 KV cache but no scaling factors "
205 - "provided. Defaulting to scaling factors of 1.0. "
206 - "This may lead to less accurate results!")
204 + logger.warning(
205 + "Using FP8 KV cache but no scaling factors "
206 + "provided. Defaulting to scaling factors of 1.0. "
207 + "This may lead to less accurate results!")
208 elif self.model_config.quantization_param_path is not None:
208 - logger.warn("KV cache scaling factors provided, "
209 - "but the KV cache data type is not FP8. "
210 - "KV cache scaling factors will not be used.")
209 + logger.warning("KV cache scaling factors provided, "
210 + "but the KV cache data type is not FP8. "
211 + "KV cache scaling factors will not be used.")
212
213 def set_block_size(self, block_size: int) -> None:
214 self.block_size = block_size
@@ -1054,7 +1055,7 @@ class ModelRunner:
1055 end_time = time.perf_counter()
1056 elapsed_time = end_time - start_time
1057 # This usually takes < 10 seconds.
1057 - logger.info(f"Graph capturing finished in {elapsed_time:.0f} secs.")
1058 + logger.info("Graph capturing finished in %.0f secs.", elapsed_time)
1059
1060 def __del__(self) -> None:
1061 # Delete the CUDA graphs before deleting the pynccl communicator.