json.loads和eval 速度对比
- 代码1
- 结果图
- 代码2
- 参考地址
代码1
import json
import time
import pandas as pd
data_sets = pd.read_pickle("val_token_id.pandas_pickle")
data_sets=[str(i) for i in data_sets]
start=time.time()
[json.loads(i) for i in data_sets]
print(time.time()-start)
start=time.time()
[eval(i) for i in data_sets]
print(time.time()-start)
结果图
代码2
import json
import time
from multiprocessing import Process, Manager, freeze_support
import pandas as pd
from tqdm import tqdm
def json_loads_data(return_list,one_data):
return_list+=[json.loads(i) for i in tqdm(one_data)]
if __name__ == '__main__':
freeze_support()
data_sets = pd.read_pickle("val_token_id.pandas_pickle")
data_sets = [str(i) for i in data_sets]
start = time.time()
data = Manager().list()
num = 5
p_list = []
for i in range(0, len(data_sets), len(data_sets)//num):
j = i + len(data_sets)//num
p = Process(target=json_loads_data, args=(data, data_sets[i:j]))
p.start()
p_list.append(p)
for p in p_list:
p.join()
print("multi_json_loads", time.time() - start)
start = time.time()
[json.loads(i) for i in data_sets]
print("json_loads", time.time() - start)
start = time.time()
pd.DataFrame(data_sets)[0].apply(lambda x: json.loads(x)).values.tolist()
print("dataFrame_apply", time.time() - start)
start = time.time()
json.loads(str(data_sets).replace("'", ""))
print("json_loads_str", time.time() - start)
start = time.time()
[eval(i) for i in data_sets]
print("eval", time.time() - start)
参考地址
https://blog.csdn.net/qq_35869630/article/details/105919104
Python 在大数据处理下的优化(一)用json.loads比eval快10倍!