@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 28598f3939
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 07:00 UTC

[Core] remove temporary local variables in LLMEngine.__init__ (#10577)

Signed-off-by: Russell Bryant <rbryant@redhat.com>

Russell Bryant committed Nov 22, 2024 at 19:22 UTC 28598f3939f9a04800f514e7fe62ab9bb8f617ec
1 file changed +66 -77
vllm/engine/llm_engine.py
+66 -77
@@ -231,19 +231,18 @@ class LLMEngine:
231 use_cached_outputs: bool = False,
232 ) -> None:
233
234 - # TODO: remove the local variables and use self.* throughout the class.
235 - model_config = self.model_config = vllm_config.model_config
236 - cache_config = self.cache_config = vllm_config.cache_config
237 - lora_config = self.lora_config = vllm_config.lora_config
238 - parallel_config = self.parallel_config = vllm_config.parallel_config
239 - scheduler_config = self.scheduler_config = vllm_config.scheduler_config
240 - device_config = self.device_config = vllm_config.device_config
241 - speculative_config = self.speculative_config = vllm_config.speculative_config # noqa
242 - load_config = self.load_config = vllm_config.load_config
243 - decoding_config = self.decoding_config = vllm_config.decoding_config or DecodingConfig( # noqa
234 + self.model_config = vllm_config.model_config
235 + self.cache_config = vllm_config.cache_config
236 + self.lora_config = vllm_config.lora_config
237 + self.parallel_config = vllm_config.parallel_config
238 + self.scheduler_config = vllm_config.scheduler_config
239 + self.device_config = vllm_config.device_config
240 + self.speculative_config = vllm_config.speculative_config # noqa
241 + self.load_config = vllm_config.load_config
242 + self.decoding_config = vllm_config.decoding_config or DecodingConfig( # noqa
243 )
245 - prompt_adapter_config = self.prompt_adapter_config = vllm_config.prompt_adapter_config # noqa
246 - observability_config = self.observability_config = vllm_config.observability_config or ObservabilityConfig( # noqa
244 + self.prompt_adapter_config = vllm_config.prompt_adapter_config # noqa
245 + self.observability_config = vllm_config.observability_config or ObservabilityConfig( # noqa
246 )
247
248 logger.info(
@@ -265,54 +264,43 @@ class LLMEngine:
264 "mm_processor_kwargs=%s, pooler_config=%r,"
265 "compilation_config=%r",
266 VLLM_VERSION,
268 - model_config.model,
269 - speculative_config,
270 - model_config.tokenizer,
271 - model_config.skip_tokenizer_init,
272 - model_config.tokenizer_mode,
273 - model_config.revision,
274 - model_config.override_neuron_config,
275 - model_config.tokenizer_revision,
276 - model_config.trust_remote_code,
277 - model_config.dtype,
278 - model_config.max_model_len,
279 - load_config.download_dir,
280 - load_config.load_format,
281 - parallel_config.tensor_parallel_size,
282 - parallel_config.pipeline_parallel_size,
283 - parallel_config.disable_custom_all_reduce,
284 - model_config.quantization,
285 - model_config.enforce_eager,
286 - cache_config.cache_dtype,
287 - model_config.quantization_param_path,
288 - device_config.device,
289 - decoding_config,
290 - observability_config,
291 - model_config.seed,
292 - model_config.served_model_name,
293 - scheduler_config.num_scheduler_steps,
294 - scheduler_config.chunked_prefill_enabled,
295 - scheduler_config.multi_step_stream_outputs,
296 - cache_config.enable_prefix_caching,
297 - model_config.use_async_output_proc,
267 + self.model_config.model,
268 + self.speculative_config,
269 + self.model_config.tokenizer,
270 + self.model_config.skip_tokenizer_init,
271 + self.model_config.tokenizer_mode,
272 + self.model_config.revision,
273 + self.model_config.override_neuron_config,
274 + self.model_config.tokenizer_revision,
275 + self.model_config.trust_remote_code,
276 + self.model_config.dtype,
277 + self.model_config.max_model_len,
278 + self.load_config.download_dir,
279 + self.load_config.load_format,
280 + self.parallel_config.tensor_parallel_size,
281 + self.parallel_config.pipeline_parallel_size,
282 + self.parallel_config.disable_custom_all_reduce,
283 + self.model_config.quantization,
284 + self.model_config.enforce_eager,
285 + self.cache_config.cache_dtype,
286 + self.model_config.quantization_param_path,
287 + self.device_config.device,
288 + self.decoding_config,
289 + self.observability_config,
290 + self.model_config.seed,
291 + self.model_config.served_model_name,
292 + self.scheduler_config.num_scheduler_steps,
293 + self.scheduler_config.chunked_prefill_enabled,
294 + self.scheduler_config.multi_step_stream_outputs,
295 + self.cache_config.enable_prefix_caching,
296 + self.model_config.use_async_output_proc,
297 use_cached_outputs,
299 - model_config.mm_processor_kwargs,
300 - model_config.pooler_config,
298 + self.model_config.mm_processor_kwargs,
299 + self.model_config.pooler_config,
300 vllm_config.compilation_config,
301 )
302 # TODO(woosuk): Print more configs in debug mode.
304 - self.model_config = model_config
305 - self.cache_config = cache_config
306 - self.lora_config = lora_config
307 - self.parallel_config = parallel_config
308 - self.scheduler_config = scheduler_config
309 - self.device_config = device_config
310 - self.speculative_config = speculative_config
311 - self.load_config = load_config
312 - self.decoding_config = decoding_config or DecodingConfig()
313 - self.prompt_adapter_config = prompt_adapter_config
314 - self.observability_config = observability_config or ObservabilityConfig(
315 - )
303 +
304 self.log_stats = log_stats
305 self.use_cached_outputs = use_cached_outputs
306
@@ -334,15 +322,15 @@ class LLMEngine:
322
323 self.seq_counter = Counter()
324 self.generation_config_fields = _load_generation_config_dict(
337 - model_config)
325 + self.model_config)
326
339 - self.input_preprocessor = InputPreprocessor(model_config,
327 + self.input_preprocessor = InputPreprocessor(self.model_config,
328 self.tokenizer,
329 mm_registry)
330
331 self.input_registry = input_registry
332 self.input_processor = input_registry.create_input_processor(
345 - model_config)
333 + self.model_config)
334
335 self.model_executor = executor_class(vllm_config=vllm_config, )
336
@@ -354,36 +342,36 @@ class LLMEngine:
342 from vllm.model_executor.model_loader import (
343 get_architecture_class_name)
344 usage_message.report_usage(
357 - get_architecture_class_name(model_config),
345 + get_architecture_class_name(self.model_config),
346 usage_context,
347 extra_kvs={
348 # Common configuration
349 "dtype":
362 - str(model_config.dtype),
350 + str(self.model_config.dtype),
351 "tensor_parallel_size":
364 - parallel_config.tensor_parallel_size,
352 + self.parallel_config.tensor_parallel_size,
353 "block_size":
366 - cache_config.block_size,
354 + self.cache_config.block_size,
355 "gpu_memory_utilization":
368 - cache_config.gpu_memory_utilization,
356 + self.cache_config.gpu_memory_utilization,
357
358 # Quantization
359 "quantization":
372 - model_config.quantization,
360 + self.model_config.quantization,
361 "kv_cache_dtype":
374 - str(cache_config.cache_dtype),
362 + str(self.cache_config.cache_dtype),
363
364 # Feature flags
365 "enable_lora":
378 - bool(lora_config),
366 + bool(self.lora_config),
367 "enable_prompt_adapter":
380 - bool(prompt_adapter_config),
368 + bool(self.prompt_adapter_config),
369 "enable_prefix_caching":
382 - cache_config.enable_prefix_caching,
370 + self.cache_config.enable_prefix_caching,
371 "enforce_eager":
384 - model_config.enforce_eager,
372 + self.model_config.enforce_eager,
373 "disable_custom_all_reduce":
386 - parallel_config.disable_custom_all_reduce,
374 + self.parallel_config.disable_custom_all_reduce,
375 })
376
377 if self.tokenizer:
@@ -402,7 +390,7 @@ class LLMEngine:
390 for _ in range(self.parallel_config.pipeline_parallel_size)
391 ]
392
405 - if model_config.use_async_output_proc:
393 + if self.model_config.use_async_output_proc:
394 process_model_outputs = weak_bind(self._process_model_outputs)
395
396 self.async_callbacks = [
@@ -422,11 +410,11 @@ class LLMEngine:
410 # GPU and CPU blocks, which are profiled in the distributed executor.
411 self.scheduler = [
412 Scheduler(
425 - scheduler_config, cache_config, lora_config,
426 - parallel_config.pipeline_parallel_size,
413 + self.scheduler_config, self.cache_config, self.lora_config,
414 + self.parallel_config.pipeline_parallel_size,
415 self.async_callbacks[v_id]
428 - if model_config.use_async_output_proc else None)
429 - for v_id in range(parallel_config.pipeline_parallel_size)
416 + if self.model_config.use_async_output_proc else None)
417 + for v_id in range(self.parallel_config.pipeline_parallel_size)
418 ]
419
420 # Metric Logging.
@@ -448,7 +436,8 @@ class LLMEngine:
436 "prometheus":
437 PrometheusStatLogger(
438 local_interval=_LOCAL_LOGGING_INTERVAL_SEC,
451 - labels=dict(model_name=model_config.served_model_name),
439 + labels=dict(
440 + model_name=self.model_config.served_model_name),
441 max_model_len=self.model_config.max_model_len),
442 }
443 self.stat_loggers["prometheus"].info("cache_config",