创建环境：
conda create -n llama3_env python=3.10
conda activate llama3_env
conda install pytorch torchvision torchaudio cudatoolkit=11.7 -c pytorch
安装Hugging Face的Transformers库：
pip install transformers sentencepiece
下载模型
https://huggingface.co/shenzhi-wang/Llama3-8B-Chinese-Chat/tree/main
编写代码调用
import torch
from transformers import AutoModelForCausalLM, AutoTokenizer
# 检查CUDA是否可用，并设置设备
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
print(torch.cuda.is_available())
print(device)
# 加载模型和tokenizer
model_name = "F:\\ollama_models\\Llama3-8B-Chinese-Chat"
model = AutoModelForCausalLM.from_pretrained(model_name).to(device)
tokenizer = AutoTokenizer.from_pretrained(model_name)
# 编写推理函数
# def generate_text(prompt):
# inputs = tokenizer(prompt, return_tensors="pt").to(device)
# outputs = model.generate(inputs['input_ids'], max_length=100)
# return tokenizer.decode(outputs[0], skip_special_tokens=True)
#
# # 示例使用
# prompt = "写一首诗吧，以春天为主题"
# print(generate_text(prompt))
messages = [
{"role": "user", "content": "写一首诗吧"},
]
input_ids = tokenizer.apply_chat_template(
messages, add_generation_prompt=True, return_tensors="pt"
).to(model.device)
outputs = model.generate(
input_ids,
max_new_tokens=8192,
do_sample=True,
temperature=0.6,
top_p=0.9,
)
response = outputs[0][input_ids.shape[-1]:]
print(tokenizer.decode(response, skip_special_tokens=True))
非常慢，大概用了一两分钟回答一个问题。
还是老实用ollama跑qwen吧
