Llama 本地推理huggingFace模型的两种方式_huggingface opt模型推理代码

 方式1：

from transformers import LlamaForCausalLM, AutoTokenizer
 
#下载好的hf模型地址
hf_model_path = './Llama-2-7b'
model = LlamaForCausalLM.from_pretrained(hf_model_path, device_map="auto")
tokenizer = AutoTokenizer.from_pretrained(hf_model_path)
 
prompt = "Hey, are you conscious? Can you talk to me?"
inputs = tokenizer(prompt, return_tensors="pt")
 
# Generate
generate_ids = model.generate(inputs.input_ids, max_length=30)
res = tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
print(res)

方式2:

import transformers,torch
from transformers import LlamaForCausalLM, AutoTokenizer
 
#下载好的hf模型地址
hf_model_path = './Llama-2-7b'
 
tokenizer = AutoTokenizer.from_pretrained(hf_model_path)
pipeline = transformers.pipeline(
    "text-generation",
    model=hf_model_path,
    torch_dtype=torch.float16,
    device_map="auto",
)
 
sequences = pipeline(
    'I liked "Breaking Bad" and "Band of Brothers". Do you have any recommendations of other shows I might like?\n',
    do_sample=True,
    top_k=10,
    num_return_sequences=1,
    eos_token_id=tokenizer.eos_token_id,
    max_length=200,
)
 
for seq in sequences:
    print(f"Result: {seq['generated_text']}")