from typing import List
import numpy as np
import pandas as pd
from sklearn.metrics import accuracy_score, precision_score, f1_score
from sklearn.metrics import recall_score, confusion_matrix
import os
import time
from transformers import AutoModelForMaskedLM, AutoTokenizer, get_scheduler
from pet_config import*import sys
sys.path.append('/Users/ligang/PycharmProjects/llm/prompt_tasks/PET/data_handle')
sys.path.append('/Users/ligang/PycharmProjects/llm/prompt_tasks/PET/utils')from utils.metirc_utils import ClassEvaluator
from utils.common_utils import*from data_handle.data_loader import*from utils.verbalizer import Verbalizer
from pet_config import*
pc = ProjectConfig()
定义model2train()函数
defmodel2train():
model = AutoModelForMaskedLM.from_pretrained(pc.pre_model)
tokenizer = AutoTokenizer.from_pretrained(pc.pre_model)
verbalizer = Verbalizer(verbalizer_file=pc.verbalizer,
tokenizer=tokenizer,
max_label_len=pc.max_label_len)#对参数做权重衰减是为了使函数平滑,然而bias和layernorm的权重参数不影响函数的平滑性。#他们起到的作用仅仅是缩放平移,因此不需要权重衰减
no_decay =["bias","LayerNorm.weight"]
optimizer_grouped_parameters =[{"params":[p for n, p in model.named_parameters()ifnotany(nd in n for nd in no_decay)],"weight_decay": pc.weight_decay,},{"params":[p for n, p in model.named_parameters()ifany(nd in n for nd in no_decay)],"weight_decay":0.0,},]
optimizer = torch.optim.AdamW(optimizer_grouped_parameters, lr=pc.learning_rate)
model.to(pc.device)
train_dataloader, dev_dataloader = get_data()# 根据训练轮数计算最大训练步数,以便于scheduler动态调整lr
num_update_steps_per_epoch =len(train_dataloader)#指定总的训练步数,它会被学习率调度器用来确定学习率的变化规律,确保学习率在整个训练过程中得以合理地调节
max_train_steps = pc.epochs * num_update_steps_per_epoch
warm_steps =int(pc.warmup_ratio * max_train_steps)# 预热阶段的训练步数
lr_scheduler = get_scheduler(
name='linear',
optimizer=optimizer,
num_warmup_steps=warm_steps,
num_training_steps=max_train_steps,)
loss_list =[]
tic_train = time.time()
metric = ClassEvaluator()
criterion = torch.nn.CrossEntropyLoss()
global_step, best_f1 =0,0print('开始训练:')for epoch inrange(pc.epochs):for batch in train_dataloader:
logits = model(input_ids=batch['input_ids'].to(pc.device),
token_type_ids=batch['token_type_ids'].to(pc.device),
attention_mask=batch['attention_mask'].to(pc.device)).logits
# print(f'模型训练得到的结果logits-->{logits.size()}')# 真实标签
mask_labels = batch['mask_labels'].numpy().tolist()
sub_labels = verbalizer.batch_find_sub_labels(mask_labels)
sub_labels =[ele['token_ids']for ele in sub_labels]# print(f'sub_labels--->{sub_labels}')
loss = mlm_loss(logits,
batch['mask_positions'].to(pc.device),
sub_labels,
criterion,
pc.device,1.0)
optimizer.zero_grad()
loss.backward()
optimizer.step()
lr_scheduler.step()
loss_list.append(float(loss.cpu().detach()))# #
global_step +=1if global_step % pc.logging_steps ==0:
time_diff = time.time()- tic_train
loss_avg =sum(loss_list)/len(loss_list)print("global step %d, epoch: %d, loss: %.5f, speed: %.2f step/s"%(global_step, epoch, loss_avg, pc.logging_steps / time_diff))
tic_train = time.time()if global_step % pc.valid_steps ==0:
cur_save_dir = os.path.join(pc.save_dir,"model_%d"% global_step)ifnot os.path.exists(cur_save_dir):
os.makedirs(cur_save_dir)
model.save_pretrained(os.path.join(cur_save_dir))
tokenizer.save_pretrained(os.path.join(cur_save_dir))
acc, precision, recall, f1, class_metrics = evaluate_model(model,
metric,
dev_dataloader,
tokenizer,
verbalizer)print("Evaluation precision: %.5f, recall: %.5f, F1: %.5f"%(precision, recall, f1))if f1 > best_f1:print(f"best F1 performence has been updated: {best_f1:.5f} --> {f1:.5f}")print(f'Each Class Metrics are: {class_metrics}')
best_f1 = f1
cur_save_dir = os.path.join(pc.save_dir,"model_best")ifnot os.path.exists(cur_save_dir):
os.makedirs(cur_save_dir)
model.save_pretrained(os.path.join(cur_save_dir))
tokenizer.save_pretrained(os.path.join(cur_save_dir))
tic_train = time.time()print('训练结束')
import time
from typing import List
import torch
from rich importprintfrom transformers import AutoTokenizer, AutoModelForMaskedLM
import sys
sys.path.append('/Users/**/PycharmProjects/llm/prompt_tasks/PET/data_handle')
sys.path.append('/Users/**/PycharmProjects/llm/prompt_tasks/PET/utils')from utils.verbalizer import Verbalizer
from data_handle.template import HardTemplate
from data_handle.data_preprocess import convert_example
from utils.common_utils import convert_logits_to_ids
Retrieval-Augmented Generation for Knowledge-Intensive NLP Tasks Lewis P, Perez E, Piktus A, et al. Retrieval-augmented generation for knowledge-intensive nlp tasks[J]. Advances in Neural Information Processing Systems, 2020, 33: 9459-9474. RAG结合了信息检…
文章目录 1 AI框架1.1 Spring AI 简介1.2 Spring AI 使用1.2.1 pom.xml1.2.2 可实现的功能 1.3 Spring Cloud Alibaba AI1.4 Spring Cloud Alibaba AI 实践操作1.4.1 pom.xml1.4.2 配置文件1.4.3 对接文本模型1.4.4 文生图模型1.4.5 语音合成模型 1 AI框架
1.1 Spring AI 简介…