电商API接口|爬虫案例|采集某东商品评论信息

news2024/11/24 14:24:05

前言：

平常大家都有网上购物的习惯，在商品下面卖的好的产品基本都会有评论，当然也不排除有刷评论的情况，因为评论会影响我们的购物决策。今天主要分享用python+re正则表达式获取京东商品评论。API接口获取京东平台商品详情SKU数据！

环境准备：

pyhon编译器版本python3.7.4

集成开发环境(IDE)pycharm版本2020.1.5

相关代码如下

运行结果部分数据展示：

总共获取500条评论。

完整代码：

import requests
import re
import json
import pandas as pd
# 基本url

BASE_URL = "https://club.jd.com/comment/productPageComments.action?

callback=fetchJSON_comment98&productId=100035712310&score=0&sortType=5

&page={}&pageSize=10&isShadowSku=0&rid=0&fold=1"

# 定义函数访问页面
def scrape_url(n):
url = BASE_URL.format(n)
return url

def get_reviews_html(url):
# 请求头
headers = {
'cookie': '__jdu=1659013711311387516036; shshshfpa=96b8fa44-7940-4a82-dc6b-ae9ec11053d8-1671423464; shshshfpb=l3JPGq-Nsv3ryl-UYczuTVg; unpl=JF8EAMhnNSttXEhSBh0LG0IZTlsBW11YGx4LbDAHBllbHANXEwFIFBl7XlVdXxRKFB9sYxRUXVNLVA4ZBisSEXteXVdZDEsWC2tXVgQFDQ8VXURJQlZAFDNVCV9dSRZRZjJWBFtdT1xWSAYYRRMfDlAKDlhCR1FpMjVkXlh7VAQrAhwWGEpdVlhdCEkXA21uA1BdX0pWAisDKxUge21WWVwPSRYzblcEZB8MF1cBHwEZF11LWlBWXAhJEQNvZQVUX1FNUAUcAxkVIEptVw; __jdv=76161171|baidu-pinzhuan|t_288551095_baidupinzhuan|cpc|0f3d30c8dba7459bb52f2eb5eba8ac7d_0_5272781a847d41aa892f3c52ffc92b78|1677294929187; areaId=19; PCSYCityID=CN_440000_440300_0; shshshfpx=96b8fa44-7940-4a82-dc6b-ae9ec11053d8-1671423464; shshshfp=edea99a27fbc1b9e7a0a677bc566df49; __jdc=122270672; ip_cityCode=1601; ipLoc-djd=19-1607-4773-62121; jwotest_product=99; 3AB9D23F7A4B3C9B=JLXTWBOH4BNFVY37YNSE4B6N2OLXI7T6WIX336O237ZEYUEAF4RFRCLPN3AEBN5FK556TM2FSRQABIGENHAVETWTGA; jsavif=1; __jda=122270672.1659013711311387516036.1659013711.1677312527.1677330268.7; __jdb=122270672.1.1659013711311387516036|7.1677330268; shshshsID=426d0c1911285cc51524a751262039de_1_1677330269246; JSESSIONID=85D1CE4A20581F98C688EA0EFE2CB175.s1',
'referer': 'https://item.jd.com/',
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36',
}
# 发送请求
response = requests.get(url=url,headers=headers)
# 获取响应内容
result = response.text
print(result)
return result
# 创建空列表，用来存放信息
data = []
def get_goods_contents(result):
# 提取评论内容
content = re.findall('"guid".*?"content":"(.*?)"',result)
# print(content)
# 提取产品尺寸
productSize = re.findall('productColor":"(.*?)"',result)
# print(productSize)
# 提取产品配置
productConfig = re.findall('"productSize":"(.*?)"',result)
# print(productConfig)
# 提取评论创建时间
creationTime = re.findall('"creationTime":"(.*?)"',result)
# 用for 循环批次把信息储存到data
for i in range(len(productSize)):
data.append([productSize[i],creationTime[i],productConfig[i],content[i]])
# print(data)
return data
def save_reviews(data):
columns = ['产品尺寸','评论时间','产品配置','产品评论']
df = pd.DataFrame(data,columns=columns)
df.to_excel('电脑评论.xlsx',encoding='utf-8')
# 运行主程序
if __name__=='__main__':
for i in range(0,50):
# 调用函数依次访问页面
url =scrape_url(n=i)
# 调用函数获取响应内容
result = get_reviews_html(url)
# 调用函数获取数据
data = get_goods_contents(result)
# 调用函数保存信息