A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 07:00 UTC
[Core] remove temporary local variables in LLMEngine.__init__ (#10577)
Signed-off-by: Russell Bryant <rbryant@redhat.com>
Russell Bryant committed
Nov 22, 2024 at 19:22 UTC
28598f3939f9a04800f514e7fe62ab9bb8f617ec
1 file changed
+66
-77
vllm/engine/llm_engine.py
+66
-77
@@ -231,19 +231,18 @@ class LLMEngine:
231
use_cached_outputs: bool = False,
232
) -> None:
233
234
- # TODO: remove the local variables and use self.* throughout the class.
235
- model_config = self.model_config = vllm_config.model_config
236
- cache_config = self.cache_config = vllm_config.cache_config
237
- lora_config = self.lora_config = vllm_config.lora_config
238
- parallel_config = self.parallel_config = vllm_config.parallel_config
239
- scheduler_config = self.scheduler_config = vllm_config.scheduler_config
240
- device_config = self.device_config = vllm_config.device_config
241
- speculative_config = self.speculative_config = vllm_config.speculative_config # noqa
242
- load_config = self.load_config = vllm_config.load_config
243
- decoding_config = self.decoding_config = vllm_config.decoding_config or DecodingConfig( # noqa
234
+ self.model_config = vllm_config.model_config
235
+ self.cache_config = vllm_config.cache_config
236
+ self.lora_config = vllm_config.lora_config
237
+ self.parallel_config = vllm_config.parallel_config
238
+ self.scheduler_config = vllm_config.scheduler_config
239
+ self.device_config = vllm_config.device_config
240
+ self.speculative_config = vllm_config.speculative_config # noqa
241
+ self.load_config = vllm_config.load_config
242
+ self.decoding_config = vllm_config.decoding_config or DecodingConfig( # noqa
243
)
245
- prompt_adapter_config = self.prompt_adapter_config = vllm_config.prompt_adapter_config # noqa
246
- observability_config = self.observability_config = vllm_config.observability_config or ObservabilityConfig( # noqa
244
+ self.prompt_adapter_config = vllm_config.prompt_adapter_config # noqa
245
+ self.observability_config = vllm_config.observability_config or ObservabilityConfig( # noqa
246
)
247
248
logger.info(
@@ -265,54 +264,43 @@ class LLMEngine:
264
"mm_processor_kwargs=%s, pooler_config=%r,"
265
"compilation_config=%r",
266
VLLM_VERSION,
268
- model_config.model,
269
- speculative_config,
270
- model_config.tokenizer,
271
- model_config.skip_tokenizer_init,
272
- model_config.tokenizer_mode,
273
- model_config.revision,
274
- model_config.override_neuron_config,
275
- model_config.tokenizer_revision,
276
- model_config.trust_remote_code,
277
- model_config.dtype,
278
- model_config.max_model_len,
279
- load_config.download_dir,
280
- load_config.load_format,
281
- parallel_config.tensor_parallel_size,
282
- parallel_config.pipeline_parallel_size,
283
- parallel_config.disable_custom_all_reduce,
284
- model_config.quantization,
285
- model_config.enforce_eager,
286
- cache_config.cache_dtype,
287
- model_config.quantization_param_path,
288
- device_config.device,
289
- decoding_config,
290
- observability_config,
291
- model_config.seed,
292
- model_config.served_model_name,
293
- scheduler_config.num_scheduler_steps,
294
- scheduler_config.chunked_prefill_enabled,
295
- scheduler_config.multi_step_stream_outputs,
296
- cache_config.enable_prefix_caching,
297
- model_config.use_async_output_proc,
267
+ self.model_config.model,
268
+ self.speculative_config,
269
+ self.model_config.tokenizer,
270
+ self.model_config.skip_tokenizer_init,
271
+ self.model_config.tokenizer_mode,
272
+ self.model_config.revision,
273
+ self.model_config.override_neuron_config,
274
+ self.model_config.tokenizer_revision,
275
+ self.model_config.trust_remote_code,
276
+ self.model_config.dtype,
277
+ self.model_config.max_model_len,
278
+ self.load_config.download_dir,
279
+ self.load_config.load_format,
280
+ self.parallel_config.tensor_parallel_size,
281
+ self.parallel_config.pipeline_parallel_size,
282
+ self.parallel_config.disable_custom_all_reduce,
283
+ self.model_config.quantization,
284
+ self.model_config.enforce_eager,
285
+ self.cache_config.cache_dtype,
286
+ self.model_config.quantization_param_path,
287
+ self.device_config.device,
288
+ self.decoding_config,
289
+ self.observability_config,
290
+ self.model_config.seed,
291
+ self.model_config.served_model_name,
292
+ self.scheduler_config.num_scheduler_steps,
293
+ self.scheduler_config.chunked_prefill_enabled,
294
+ self.scheduler_config.multi_step_stream_outputs,
295
+ self.cache_config.enable_prefix_caching,
296
+ self.model_config.use_async_output_proc,
297
use_cached_outputs,
299
- model_config.mm_processor_kwargs,
300
- model_config.pooler_config,
298
+ self.model_config.mm_processor_kwargs,
299
+ self.model_config.pooler_config,
300
vllm_config.compilation_config,
301
)
302
# TODO(woosuk): Print more configs in debug mode.
304
- self.model_config = model_config
305
- self.cache_config = cache_config
306
- self.lora_config = lora_config
307
- self.parallel_config = parallel_config
308
- self.scheduler_config = scheduler_config
309
- self.device_config = device_config
310
- self.speculative_config = speculative_config
311
- self.load_config = load_config
312
- self.decoding_config = decoding_config or DecodingConfig()
313
- self.prompt_adapter_config = prompt_adapter_config
314
- self.observability_config = observability_config or ObservabilityConfig(
315
- )
303
+
304
self.log_stats = log_stats
305
self.use_cached_outputs = use_cached_outputs
306
@@ -334,15 +322,15 @@ class LLMEngine:
322
323
self.seq_counter = Counter()
324
self.generation_config_fields = _load_generation_config_dict(
337
- model_config)
325
+ self.model_config)
326
339
- self.input_preprocessor = InputPreprocessor(model_config,
327
+ self.input_preprocessor = InputPreprocessor(self.model_config,
328
self.tokenizer,
329
mm_registry)
330
331
self.input_registry = input_registry
332
self.input_processor = input_registry.create_input_processor(
345
- model_config)
333
+ self.model_config)
334
335
self.model_executor = executor_class(vllm_config=vllm_config, )
336
@@ -354,36 +342,36 @@ class LLMEngine:
342
from vllm.model_executor.model_loader import (
343
get_architecture_class_name)
344
usage_message.report_usage(
357
- get_architecture_class_name(model_config),
345
+ get_architecture_class_name(self.model_config),
346
usage_context,
347
extra_kvs={
348
# Common configuration
349
"dtype":
362
- str(model_config.dtype),
350
+ str(self.model_config.dtype),
351
"tensor_parallel_size":
364
- parallel_config.tensor_parallel_size,
352
+ self.parallel_config.tensor_parallel_size,
353
"block_size":
366
- cache_config.block_size,
354
+ self.cache_config.block_size,
355
"gpu_memory_utilization":
368
- cache_config.gpu_memory_utilization,
356
+ self.cache_config.gpu_memory_utilization,
357
358
# Quantization
359
"quantization":
372
- model_config.quantization,
360
+ self.model_config.quantization,
361
"kv_cache_dtype":
374
- str(cache_config.cache_dtype),
362
+ str(self.cache_config.cache_dtype),
363
364
# Feature flags
365
"enable_lora":
378
- bool(lora_config),
366
+ bool(self.lora_config),
367
"enable_prompt_adapter":
380
- bool(prompt_adapter_config),
368
+ bool(self.prompt_adapter_config),
369
"enable_prefix_caching":
382
- cache_config.enable_prefix_caching,
370
+ self.cache_config.enable_prefix_caching,
371
"enforce_eager":
384
- model_config.enforce_eager,
372
+ self.model_config.enforce_eager,
373
"disable_custom_all_reduce":
386
- parallel_config.disable_custom_all_reduce,
374
+ self.parallel_config.disable_custom_all_reduce,
375
})
376
377
if self.tokenizer:
@@ -402,7 +390,7 @@ class LLMEngine:
390
for _ in range(self.parallel_config.pipeline_parallel_size)
391
]
392
405
- if model_config.use_async_output_proc:
393
+ if self.model_config.use_async_output_proc:
394
process_model_outputs = weak_bind(self._process_model_outputs)
395
396
self.async_callbacks = [
@@ -422,11 +410,11 @@ class LLMEngine:
410
# GPU and CPU blocks, which are profiled in the distributed executor.
411
self.scheduler = [
412
Scheduler(
425
- scheduler_config, cache_config, lora_config,
426
- parallel_config.pipeline_parallel_size,
413
+ self.scheduler_config, self.cache_config, self.lora_config,
414
+ self.parallel_config.pipeline_parallel_size,
415
self.async_callbacks[v_id]
428
- if model_config.use_async_output_proc else None)
429
- for v_id in range(parallel_config.pipeline_parallel_size)
416
+ if self.model_config.use_async_output_proc else None)
417
+ for v_id in range(self.parallel_config.pipeline_parallel_size)
418
]
419
420
# Metric Logging.
@@ -448,7 +436,8 @@ class LLMEngine:
436
"prometheus":
437
PrometheusStatLogger(
438
local_interval=_LOCAL_LOGGING_INTERVAL_SEC,
451
- labels=dict(model_name=model_config.served_model_name),
439
+ labels=dict(
440
+ model_name=self.model_config.served_model_name),
441
max_model_len=self.model_config.max_model_len),
442
}
443
self.stat_loggers["prometheus"].info("cache_config",