用途：个人学习笔记，有所借鉴，欢迎指正

前言：

主要包含对requests库和Web爬虫解析库的使用，python爬虫自动化，批量信息收集

Python开发工具：PyCharm 2022.1 激活破解码_安装教程 (2022年8月25日更新~)-小白学堂

一、EDUSRC平台爬取接收漏洞的教育机构名称

1、爬取目标：EDUSRC平台此网页第1-209页的所有教育机构名称，保存到txt文件中

​

2、Python代码实现： 

import requests,time
from bs4 import BeautifulSoup

#<tr>
#    <td class="am-text-center">1</td>
#    <td class="am-text-center">
#        <a href="/list/firm/3761">上海交通大学</a>
#    </td>
#    <td class="am-text-center">3994</td>
#    <td class="am-text-center">10523</td>
#</tr>

def get_eduName():
    for i in range(1,209):
        url = 'https://src.sjtu.edu.cn/rank/firm/0/?page=%s'%str(i)
        try:
            s=requests.get(url).text
            print('------->正在获取第%s页面数据'%str(i))
            soup = BeautifulSoup(s, 'lxml')
            edu1=soup.find_all('tr')
            for edu in edu1:
                edu_name=edu.a.string
                print(edu_name)
                with open('eduname.txt','a+',encoding='utf-8') as f:
                    f.write(edu_name+'\n')
                    f.close()
        
        except Exception as e:
            time.sleep(1)
            pass

if __name__ == '__main__':
    get_eduName()

二、利用FOFA搜索引擎批量爬取与目标相关的URL地址

1、FOFA搜索语法：收集目标名称相关所有URL地址

 2、Python代码实现

import requests
from bs4 import BeautifulSoup

header={
    #登录fofa,浏览器查看数据包中的登录凭证fofa_token
    'cookie':'fofa_token=eyJhbGciOiJIUzUxMiIsImtpZCI6Ik5XWTVZakF4TVRkalltSTJNRFZsWXpRM05EWXdaakF3TURVMlkyWTNZemd3TUdRd1pUTmpZUT09IiwidHlwIjoiSldUIn0.eyJpZCI6MjgyNzMsIm1pZCI6MTAwMDIxOTg4LCJ1c2VybmFtZSI6InhpYW9kaXNlYyIsImV4cCI6MTY3MTI4MjUzOH0.0ukMGFIrIvzDOzpUl9JglOoMpzbIPCczGRDeqKdmFYHfStd2jdwc6LGby3Ke0UR2suvErzhOTPYL2ACe4Goi8Q; '
}

url='https://fofa.info/result?qbase64=dGl0bGU9IuS4iua1t%2BS6pOmAmuWkp%2BWtpiIgJiYgY291bnRyeT0iQ04i'
s=requests.get(url,headers=header).text
soup = BeautifulSoup(s, 'lxml')
#获取页数
edu1=soup.find_all('p',attrs={'class': 'hsxa-nav-font-size'})
for edu in edu1:
    edu_name = edu.span.get_text()
    i=int(edu_name)/10
    yeshu=int(i)+1
    print(yeshu)
    for ye in range(1,yeshu+1):
        url = 'https://fofa.info/result?qbase64=dGl0bGU9IuS4iua1t%2BS6pOmAmuWkp%2BWtpiIgJiYgY291bnRyeT0iQ04i&page='+str(ye)+'&page_size=10'
        print(url)
        s = requests.get(url, headers=header).text
        edu1=soup.find_all('span',attrs={'class': 'hsxa-host'})
        for edu in edu1:
            edu_name = edu.a.get_text().strip()
            print(edu_name)

3、使用Goby新建扫描任务，导入收集到的URL目标，批量扫描漏洞

三、 使用FOFA查询接口批量查询收集URL

Python代码实现：

import requests
import base64

#https://fofa.info/api/v1/search/all?email=your_email&key=your_key&qbase64=dGl0bGU9ImJpbmci

def get_fofa_data(email,apikey):
    for eduname in open('eduname.txt',encoding='utf-8'):
        e=eduname.strip()
        search='"%s" && country="CN" && title=="Error 404--Not Found"'%e
        b=base64.b64encode(search.encode('utf-8'))
        b=b.decode('utf-8')
        url='https://fofa.info/api/v1/search/all?email=%s&key=%s&qbase64=%s'%(email,apikey,b)
        s=requests.get(url).json()
        print('查询->'+eduname)
        print(url)
        if s['size'] != 0:
            print(eduname+'有数据啦！')
            for ip in s['results']:
                print(ip[0])
        else:
            print('没有数据')

if __name__ == '__main__':
    email='471656814@qq.com'   #自己fofa账号
    apikey='0fccc926c6d0c4922cbdc620659b9a42' #fofa个人中心的apikey
    get_fofa_data(email,apikey)