github地址: https://github.com/CodeWang-Ay/DataProcess
类型1
需求1:
类似于数据格式: https://blog.csdn.net/qq_44072222/article/details/120884158
目标json格式
{"位置": 1, "名称": "nba球员", "国家": "美国", "级别": "top10", "level2": [{"模块": "jordan", "球衣号码": 23, "球队名称": "公牛", "level3": [{"技术": "中投", "分数": 10}, {"技术": "三分", "分数": 5}, {"技术": "篮下", "分数": 8}, {"技术": "扣篮", "分数": 9}, {"技术": "忠诚度", "分数": 9}, {"技术": "人品", "分数": 6}]}, {"模块": "james", "球衣号码": "8、24", "球队名称": "骑士", "level3": [{"技术": "三分", "分数": 7}, {"技术": "篮下", "分数": 10}, {"技术": "扣篮", "分数": 9}, {"技术": "忠诚度", "分数": 5}]}, {"模块": "kobe", "球衣号码": "23、6", "球队名称": "湖人", "level3": [{"技术": "中投", "分数": 10}, {"技术": "忠诚度", "分数": 10}]}]}
转json格式链接: https://www.json.cn/#google_vignette
整体思路
数据格式相等于是一个三层的的树，一次遍历第一层，第二层， 第三层然后构建一颗树即可
第一层: 列范围 1-4
第二层: 列范围 5-7
第三层: 列范围 8-9
关键参数
表示三层
每次定位到第一层然后范围依次缩小取位置
col_level_index = [1, 5, 8, 10] # level1, level2, level3, level4
使用 openpyxl 工具包
openpyxl 中文文档: https://openpyxl-chinese-docs.readthedocs.io/zh-cn/latest/tutorial.html
解决代码:
import json
import openpyxl as xl
import pandas as pd
def get_merge_cell_by_no(sheet_, column_no):
"""
:param sheet: sheet对象
:param column_no: 列索引 从1开始
:return: 返回给定的索引下的所有合并单元格
"""
merger_cell = [] # 第一列合并的单元格
merged_ranges = sheet_.merged_cells.ranges # 获取当前工作表的所有合并区域列表
for merged_cell_range in merged_ranges:
if merged_cell_range.min_col == column_no and merged_cell_range.max_col == column_no:
merger_cell.append(merged_cell_range)
return sorted(merger_cell, key=lambda x: x.min_row) # 排序返回 默认不排序
def get_merge_cell_by_special(sheet, start_row, end_row, start_col, end_col):
"""
在特定范围内的合并单元格坐标
:param sheet:
:param start_row:
:param end_row:
:param start_col:
:param end_col:
:return:
"""
merger_cell = [] # 第一列合并的单元格
merged_ranges = sheet.merged_cells.ranges # 获取当前工作表的所有合并区域列表
for merged_cell_range in merged_ranges:
up = merged_cell_range.min_row >= start_row
down = merged_cell_range.max_row <= end_row
left = merged_cell_range.min_col >= start_col
right = merged_cell_range.max_col <= end_col
if up and down and left and right:
merger_cell.append(merged_cell_range)
return sorted(merger_cell, key=lambda x: x.min_row) # 排序返回 默认不排序
def get_filed_by_row_id(sheet_, keys, row_id, col_level_index):
"""
get_filed_by_row_id 通过行id得到所有的json
"""
# for row_id in row_list:
filed_json = {}
for row in sheet_.iter_rows(min_row=row_id, max_row=row_id, min_col=col_level_index[2], max_col=col_level_index[3], values_only=True):
filed_json = get_dict_by_list(keys, row)
# print(row)
return filed_json
def get_dict_by_list(keys, values):
"""
get_dict_by_list
"""
data_dict = {}
for key, value in zip(keys, values):
data_dict[key] = value
return data_dict
def save_jsonline_json(json_list, target_path):
"""
json_list ---> jsonline
:param json_list:
:param target_path:
:return:
"""
with open(target_path, 'w', encoding="utf-8") as json_file:
for item in json_list:
json.dump(item, json_file, ensure_ascii=False)
json_file.write("\n")
print("Json列表已经保存到 {} 文件中。 每一行为一个json对象".format(target_path))
def excel_to_json_tree_e2e(sheet_, col_level_index):
alphabet = " ABCDEFGHIJKLMNOPQRSTUVWXYZ"
level_merge_list = get_merge_cell_by_no(sheet_, col_level_index[0]) # 一级逻辑关系所有的列\
json_list = []
for index_level1, merge_level1 in enumerate(level_merge_list):
start_row_level1 = merge_level1.min_row
end_row_level1 = merge_level1.max_row
level1_map = {}
for col in range(col_level_index[0], col_level_index[1]): # 列A是1，列D是4
key = sheet_[alphabet[col] + str(1)].value
cell_ref = sheet_[alphabet[col] + str(start_row_level1)].value # 因为第二行，所以+1
level1_map[key] = cell_ref # 第一个阶段结束
level1_map["level2"] = []
level2_merge_list = get_merge_cell_by_special(sheet_, start_row_level1, end_row_level1,
col_level_index[1], col_level_index[1])
level2_list = []
for index_level2, merge_cell_level2 in enumerate(level2_merge_list):
start_row_level2 = merge_cell_level2.min_row
end_row_level2 = merge_cell_level2.max_row
level2_list.append([i for i in range(start_row_level2, end_row_level2 + 1)])
level2_map = {}
for col in range(col_level_index[1], col_level_index[2]): # 列A是1，列D是4
key = sheet_[alphabet[col] + str(1)].value
cell_ref = sheet_[alphabet[col] + str(start_row_level2)].value # 因为第二行，所以+1
level2_map[key] = cell_ref # 第一个阶段结束
column_name_list = []
for col in range(col_level_index[2], col_level_index[3]): # 列A是1，列D是4
key = sheet_[alphabet[col] + str(1)].value
column_name_list.append(key)
level2_map["level3"] = []
for row_id in range(start_row_level2, end_row_level2+1):
current_filed_json = get_filed_by_row_id(sheet_, column_name_list, row_id, col_level_index)
level2_map["level3"].append(current_filed_json)
level1_map["level2"].append(level2_map)
json_list.append(level1_map)
return json_list
if __name__ == '__main__':
excel_file = "excel/merge_new.xlsx"
output_path = "output/nba_json.json"
wb = xl.load_workbook(excel_file)
sheet = wb["nba"]
col_level_index = [1, 5, 8, 10] # level1, level2, level3, level4
json_list = excel_to_json_tree_e2e(sheet, col_level_index)
save_jsonline_json(json_list, output_path)
类型2
需求2:
转成:
数据格式相等于是一个三层的的树，一次遍历第一层，第二层， 第三层然后构建一颗树即可
第一层: 列范围 H
第二层: 列范围 I
第三层: 列范围 J
第四层: 列范围 KLMN
关键参数
每次定位到第一层然后范围依次缩小取位置
{"source": "北京时间3月30日，26届Cubal东南赛区决赛落幕，卫冕冠军广东工业大学66-65险胜宁波大学，时隔6年再次夺得东南王！广工首发5人全部得分上双，宁大三分15中2成输球关键。\n", "label": {"relation": "且", "conditions": [{"relation": "或", "conditions": [{"relation": "且", "conditions": [{"description": "最高速度", "option": "等于", "requirement": "210km/h"}, {"description": "车身重量", "option": "等于", "requirement": "1980kg"}]}, {"relation": "且", "conditions": [{"description": "最高速度", "option": "等于", "requirement": "265km/h"}, {"description": "车身重量", "option": "等于", "requirement": "2090kg"}]}, {"relation": "且", "conditions": [{"description": "最高速度", "option": "等于", "requirement": "265km/h"}, {"description": "车身重量", "option": "等于", "requirement": "2205kg"}]}]}, {"description": "车载智能系统", "option": "等于", "requirement": "澎湃os"}, {"description": "车身颜色", "option": "等于", "requirement": "橙、紫、绿、藏青、黑色"}, {"relation": "或", "conditions": [{"description": "电池类型", "option": "等于", "requirement": "磷酸铁锂电池"}, {"description": "电池类型", "option": "等于", "requirement": "三元锂电池"}]}]}}
解决代码:
import openpyxl as xl
import json
import itertools
import copy
def get_merge_cell_by_no(sheet_, column_no):
"""
:param sheet:
:param column_no:
:return:
"""
merger_cell = [] # 第一列合并的单元格
merged_ranges = sheet_.merged_cells.ranges # 获取当前工作表的所有合并区域列表
for merged_cell_range in merged_ranges:
if merged_cell_range.min_col == column_no and merged_cell_range.max_col == column_no:
merger_cell.append(merged_cell_range)
return sorted(merger_cell, key=lambda x: x.min_row) # 排序返回 默认不排序
def get_merge_cell_by_special(sheet, start_row, end_row, start_col, end_col):
"""
在特定范围内的合并单元格坐标
:param sheet:
:param start_row:
:param end_row:
:param start_col:
:param end_col:
:return:
"""
merger_cell = [] # 第一列合并的单元格
merged_ranges = sheet.merged_cells.ranges # 获取当前工作表的所有合并区域列表
for merged_cell_range in merged_ranges:
up = merged_cell_range.min_row >= start_row
down = merged_cell_range.max_row <= end_row
left = merged_cell_range.min_col >= start_col
right = merged_cell_range.max_col <= end_col
if up and down and left and right:
merger_cell.append(merged_cell_range)
return sorted(merger_cell, key=lambda x: x.min_row) # 排序返回 默认不排序
def get_dict_by_list(keys, values):
"""
get_dict_by_list
"""
data_dict = {}
for key, value in zip(keys, values):
data_dict[key] = value
return data_dict
def get_filed_by_row_id(sheet_, keys, row_id, use_align):
"""
get_filed_by_row_id 通过行id得到所有的json
"""
# for row_id in row_list:
filed_json = {}
for row in sheet_.iter_rows(min_row=row_id, max_row=row_id, min_col=11, max_col=14, values_only=True):
if use_align:
row = [row[0], row[2], row[3]] # 取数配置那一列不要
else:
row = [row[1], row[2], row[3]] # 取数配置那一列不要
filed_json = get_dict_by_list(keys, row)
# print(row)
return filed_json
def custom_sort_key(item):
"""
合并单元格排序
"""
if isinstance(item, list):
return item[0]
else:
return item
def excel_to_json_tree_e2e(sheet_, column_name_list, logic_level, level1_col_index, isvalid_col, use_align):
"""
excel_to_json_tree. 读取excel文件，转换成json树结构
:param sheet_:
:param column_name_list: key_list
:param logic_level: summary， 一级， 二级， 三级列名
:param level1_col_index: 第一级逻辑关系的索引 8
:param isvalid_col: 是否有效列, 有效我们才进行转化
:return:
"""
level_merge_list = get_merge_cell_by_no(sheet_, level1_col_index) # 一级逻辑关系所有的列
e2e_json_list = []
pre_json_list = []
post_json_list = []
relation_key = "relation"
condition_key = "conditions"
for index_level1, merge_cell_level1 in enumerate(level_merge_list):
e2e_json = {}
pre_json_total = {}
post_json_label = {}
post_json_total = {}
start_row_level1 = merge_cell_level1.min_row
end_row_level1 = merge_cell_level1.max_row
level1_merge = [i for i in range(start_row_level1, end_row_level1 + 1)] # level的所有列索引
summary_content = sheet_[logic_level[0] + str(start_row_level1)].value # G2 内容用于构建 source: target使用
level1_json = {}
level1_logic = sheet_[logic_level[1] + str(start_row_level1)].value # H2
level1_json[relation_key] = level1_logic # relation : logic
level1_json[condition_key] = [] #
level1_json_pre = copy.deepcopy(level1_json) # 拷贝
isvalid = sheet_[isvalid_col + str(start_row_level1)].value # F2 验证有效字段是否有效 无效则直接跳过
if isvalid != 1: # 验证是否有效字段
continue
# post
post_json_label[condition_key] = []
for row_id in level1_merge:
current_post_json = get_filed_by_row_id(sheet_, column_name_list, row_id, use_align)
post_json_label[condition_key].append(current_post_json)
level2_merge = []
level2_merge_cell_list = get_merge_cell_by_special(sheet_, start_row_level1, end_row_level1,
level1_col_index + 1, level1_col_index + 1)
for index_level2, merge_cell_level2 in enumerate(level2_merge_cell_list):
start_row_level2 = merge_cell_level2.min_row
end_row_level2 = merge_cell_level2.max_row
level2_merge.append([i for i in range(start_row_level2, end_row_level2 + 1)])
level2_merge_one_dimension = list(itertools.chain.from_iterable(level2_merge))
level2_single = [i for i in level1_merge if i not in level2_merge_one_dimension]
level2 = level2_merge + level2_single
level2 = sorted(level2, key=custom_sort_key) # 为了保证顺序
for element in level2:
if type(element) is int:
# 没有二级逻辑关系
row_id = element
current_filed_json = get_filed_by_row_id(sheet_, column_name_list, row_id, use_align)
level1_json[condition_key].append(current_filed_json)
value = current_filed_json[column_name_list[0]] # pre dataset
level1_json_pre[condition_key].append(value)
elif type(element) is list:
# 二级逻辑关系
current_level2_merge = element
start = element[0]
end = element[-1]
level3_merge_list = get_merge_cell_by_special(sheet_, start, end,
level1_col_index + 2, level1_col_index + 2)
level2_json = {}
level2_logic = sheet_[logic_level[2] + str(start)].value
level2_json[relation_key] = level2_logic
level2_json[condition_key] = []
level2_json_pre = copy.deepcopy(level2_json)
if len(level3_merge_list) == 0:
# 没有三级逻辑关系
for row_id in element:
current_filed_json = get_filed_by_row_id(sheet_, column_name_list, row_id, use_align)
level2_json[condition_key].append(current_filed_json)
level2_json_pre[condition_key].append(current_filed_json[column_name_list[0]])
else:
# 有三级逻辑关系
level3_merge = []
for merge_cell_level3 in level3_merge_list:
start_row_level3 = merge_cell_level3.min_row
end_row_level3 = merge_cell_level3.max_row
level3_merge.append([i for i in range(start_row_level3, end_row_level3 + 1)])
level3_merge_one_dimension = list(itertools.chain.from_iterable(level3_merge))
level3_single = [i for i in current_level2_merge if i not in level3_merge_one_dimension]
level3 = level3_single + level3_merge
level3 = sorted(level3, key=custom_sort_key)
for element_level3 in level3:
if type(element_level3) is int:
current_filed_json = get_filed_by_row_id(sheet_, column_name_list, element_level3, use_align)
level2_json[condition_key].append(current_filed_json)
level2_json_pre[condition_key].append(current_filed_json[column_name_list[0]])
elif type(element_level3) is list:
level3_start = element_level3[0]
level3_end = element_level3[-1]
level3_json = {}
level3_logic = sheet_[logic_level[3] + str(level3_start)].value
level3_json[relation_key] = level3_logic
level3_json[condition_key] = []
level3_json_pre = copy.deepcopy(level3_json)
for row_id in element_level3:
current_filed_json = get_filed_by_row_id(sheet_, column_name_list, row_id, use_align)
level3_json[condition_key].append(current_filed_json)
field = current_filed_json[column_name_list[0]]
level3_json_pre[condition_key].append(field)
level2_json[condition_key].append(level3_json)
level2_json_pre[condition_key].append(level3_json_pre)
level1_json[condition_key].append(level2_json)
level1_json_pre[condition_key].append(level2_json_pre)
e2e_json["source"] = summary_content
e2e_json["label"] = level1_json
pre_json_total["source"] = summary_content
pre_json_total["label"] = level1_json_pre
post_json_total["source"] = summary_content
post_json_total["label"] = post_json_label
# print(json.dumps(e2e_json, ensure_ascii=False))
# print(json.dumps(pre_json_total, ensure_ascii=False))
# print(json.dumps(post_json_total, ensure_ascii=False))
e2e_json_list.append(e2e_json)
pre_json_list.append(pre_json_total)
post_json_list.append(post_json_total)
return e2e_json_list, pre_json_list, post_json_list
def save_jsonline_json(json_list, target_path):
"""
json_list ---> jsonline
:param json_list:
:param target_path:
:return:
"""
with open(target_path, 'w', encoding="utf-8") as json_file:
for item in json_list:
json.dump(item, json_file, ensure_ascii=False)
json_file.write("\n")
print("Json列表已经保存到 {} 文件中。 每一行为一个json对象".format(target_path))
# 按顺序读取
"""
openpyxl 中文文档
https://openpyxl-chinese-docs.readthedocs.io/zh-cn/latest/tutorial.html
"""
if __name__ == "__main__":
source_file = "excel/merge_new.xlsx" # excel path
wb = xl.load_workbook(source_file)
sheet = wb["news"] # 子表对象
key_name_list = ["description", "option", "requirement"]
target_e2e_path = "output/train_e2e.json"
target_pre_path = "output/train_pre.json"
target_post_path = "output/train_post.json"
target_logic_level = ["G", "H", "I", "J"] # 逻辑关系所对应的列
valid_col = "F" # 是否有效列的字母
align_flag = 0 # 是否使用对齐列
e2e_list, pre_list, post_list = excel_to_json_tree_e2e(sheet, key_name_list, target_logic_level,
8, valid_col, align_flag)
# json_list = excel_to_json_tree_post(sheet, key_name_list, target_logic_level, 8, isvalid_col)
save_jsonline_json(e2e_list, target_e2e_path)
save_jsonline_json(pre_list, target_pre_path)
save_jsonline_json(post_list, target_post_path)
