Offline Inference With Prefix#

Source vllm-project/vllm.

 1# flake8: noqa
 2# UPSTREAM SYNC: noqa is required for passing ruff run on nm-automation
 3
 4from vllm import LLM, SamplingParams
 5
 6prefix = (
 7    "You are an expert school principal, skilled in effectively managing "
 8    "faculty and staff. Draft 10-15 questions for a potential first grade "
 9    "Head Teacher for my K-12, all-girls', independent school that emphasizes "
10    "community, joyful discovery, and life-long learning. The candidate is "
11    "coming in for a first-round panel interview for a 8th grade Math "
12    "teaching role. They have 5 years of previous teaching experience "
13    "as an assistant teacher at a co-ed, public school with experience "
14    "in middle school math teaching. Based on these information, fulfill "
15    "the following paragraph: ")
16
17# Sample prompts.
18prompts = [
19    "Hello, my name is",
20    "The president of the United States is",
21    "The capital of France is",
22    "The future of AI is",
23]
24# Create a sampling params object.
25sampling_params = SamplingParams(temperature=0.0)
26
27# Create an LLM.
28llm = LLM(model="facebook/opt-125m", enable_prefix_caching=True)
29
30generating_prompts = [prefix + prompt for prompt in prompts]
31
32# Generate texts from the prompts. The output is a list of RequestOutput objects
33# that contain the prompt, generated text, and other information.
34outputs = llm.generate(generating_prompts, sampling_params)
35# Print the outputs.
36for output in outputs:
37    prompt = output.prompt
38    generated_text = output.outputs[0].text
39    print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
40
41print("-" * 80)
42
43# The llm.generate call will batch all prompts and send the batch at once
44# if resources allow. The prefix will only be cached after the first batch
45# is processed, so we need to call generate once to calculate the prefix
46# and cache it.
47outputs = llm.generate(generating_prompts[0], sampling_params)
48
49# Subsequent batches can leverage the cached prefix
50outputs = llm.generate(generating_prompts, sampling_params)
51
52# Print the outputs. You should see the same outputs as before
53for output in outputs:
54    prompt = output.prompt
55    generated_text = output.outputs[0].text
56    print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")