@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 5d80a9178b
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 05:30 UTC

Minor fix in prefill cache example (#2494)

Jason Zhu committed Jan 18, 2024 at 09:40 UTC 5d80a9178b48f211a6fa02a7b5ddc0a0ae29aa44
1 file changed +10 -2
examples/offline_inference_with_prefix.py
+10 -2
@@ -40,8 +40,16 @@ print("-" * 80)
40 # -1 since the last token can change when concatenating prompts.
41 prefix_pos = len(llm.llm_engine.tokenizer.encode(prefix)) - 1
42
43 -# Generate with prefix
44 -outputs = llm.generate(generating_prompts, sampling_params,
43 +# The llm.generate call will batch all prompts and send the batch at once if resources allow.
44 +# The prefix will only be cached after the first batch is processed, so we need to call generate once
45 +# to calculate the prefix and cache it.
46 +outputs = llm.generate(generating_prompts[0],
47 + sampling_params,
48 + prefix_pos=[prefix_pos])
49 +
50 +# Subsequent batches can leverage the cached prefix
51 +outputs = llm.generate(generating_prompts,
52 + sampling_params,
53 prefix_pos=[prefix_pos] * len(generating_prompts))
54
55 # Print the outputs. You should see the same outputs as before