支持2020-2023所有图文数据采集

#!/usr/bin/python3
# -*- coding:utf-8 -*-
import requests
import re
from lxml import etree
from sdk.utils.util_decorate import retry

@retry(retry=3,sleep=5)
def get_html(url):
    response = requests.get(url)
    response.encoding = "utf-8"
    if response.status_code == 200:
        return response.text
    else:
        print(response.status_code)
        return "ERROR"

def get_text(text):
    if isinstance(text,str):
        return re.sub("\\r|\\n|\\t|　| ", "", text).strip(" ")
    elif isinstance(text,list):
        return "".join([re.sub("\\r|\\n|\\t|　| ", "", i).strip(" ") for i in text])


def anlise_detail(detail_html):
    tree = etree.HTML(detail_html)
    lis = tree.xpath('//div[@class="article"]|//div[@class="text_c"]')
    for li in lis:
        title = get_text(li.xpath('./h1/text()'))
        print("标题",title)
        title2 = get_text(li.xpath('./h2/text()')).strip("\n")
        if title2:
            print("副标题",title2)
        pusblish_info = get_text(li.xpath('.//span[@class="date"]/text()|//div[@class="lai"]//text()'))
        print("文章信息",pusblish_info)
        content = get_text(li.xpath('.//div[@id="ozoom"]//p/text()'))
        print(content)
        img_list = [i.replace("../../../", "http://paper.people.com.cn/rmrb/") if not i.startswith("http://") else i for
                    i in
                    li.xpath('.//img/@src')]
        if img_list:
            print(img_list)


year_list = [str(i) for i in range(2020,2024)]
month_list = [str(i).zfill(2) for i in range(1,13)]
day_list = [str(i).zfill(2) for i in range(1,32)]

for year in year_list:
    for month in month_list:
        for day in day_list:
            head = "http://paper.people.com.cn/rmrb/html/{}-{}/{}/".format(year,month,day)
            for i in range(1,21):
                url = "{}nbs.D110000renmrb_{}.htm".format(head,str(i).zfill(2))
                # print(url)
                html = get_html(url)["msg"]
                if html != "ERROR":
                    tree = etree.HTML(html)
                    lis = tree.xpath('//div[@class="news"]/ul|//div[@id="titleList"]/ul')
                    for li in lis:
                        detail_url_list = li.xpath('./li/a/@href')
                        name_list = li.xpath('./li/a//text()')
                        for name,_url in zip(name_list, detail_url_list):
                            detail_url = "{}{}".format(head,_url)
                            name = re.findall('document\.write\(view\(\"(.*?)\"\)\)',name)[0].strip()
                            print(name,detail_url)
                            detail_html = get_html(detail_url)["msg"]
                            if detail_html != "ERROR":
                                anlise_detail(detail_html)