一、需求：错误日志Top10告警发送
二、需求分解
- jsoup实现登录，获取到cookie和token等用户鉴权信息
- 获取接口相应的key值
- 调用日志平台错误日志Top榜接口，查询到结果集
- 调用企业微信机器人发送消息接口
- 加上定时任务，可以实现定时发送错误日志告警的功能（后续加进定时任务里面去）
jsoup是java的爬虫框架，可以爬取网页数据，这里没有重点使用，只是做了个登录功能。后续可以专门它写一份爬虫的程序。。
<dependency>
<groupId>org.jsoup</groupId>
<artifactId>jsoup</artifactId>
<version>1.12.1</version>
</dependency>package com.smy.cbs.task;
import cn.hutool.core.date.DateUtil;
import cn.hutool.core.util.StrUtil;
import cn.hutool.http.HttpRequest;
import cn.hutool.http.HttpResponse;
import cn.hutool.http.HttpUtil;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.google.common.collect.Lists;
import com.smy.cbs.util.RetryUtil;
import com.smy.framework.core.support.SpringTestCase;
import org.apache.commons.collections4.CollectionUtils;
import org.jsoup.Connection;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.junit.Test;
import org.springframework.scheduling.concurrent.CustomizableThreadFactory;
import javax.annotation.Resource;
import javax.net.ssl.*;
import java.io.IOException;
import java.math.BigDecimal;
import java.security.SecureRandom;
import java.security.cert.CertificateException;
import java.security.cert.X509Certificate;
import java.util.*;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
/**
* @author youlu
* @ClassName HttpTestt
* @Date 2023/11/22 17:14
* @Version V1.0
**/
public class HttpTestt2 extends SpringTestCase {
@Resource
private RetryUtil retryUtil;
public static final String LOG_CENTER_BASE_URL = "https://logxxxxx.com/api/v1/";
public static final String USER_NAME = "aaaaa";
public static final String PASSWORD = "bbbbb";
public static String X_Csrf_Token = "";
public static String COOKIE = "";
public static String id = "";
public static int maxCount = Integer.MAX_VALUE;//达到阈值则告警
public static List<String> filterContent = Lists.newArrayList();//过滤内容
public static int SUB_CONTENT_LENGTH = 240;//截取报错内容字符串长度
public static int SUB_TITLE_LENGTH = 20;//截取报错类型字符串长度
public static int PERIOD_HOUR = 24*7;//24小时内的错误日志
public static List<String> SEARCH_SYSTEM = Lists.newArrayList("cbs_core", "adv_core", "rls_core", "uts_core");//查询的系统
//public static List<String> SEARCH_SYSTEM = Lists.newArrayList("adv_core");//查询的系统
public static String WX_ROBOT_URL = "https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=05e1f4a8-aaaaaa" /*+ ",https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=87140cb7-bbbbbb"*/;
public static final String TEMPLATE = "{\"markdown\":{\"content\":\"%s\"},\"msgtype\":\"markdown\"}";
public static final String QUERY_TEMPLATE = "{\n" +
" \"app\": \"system\",\n" +
" \"source\": \"other\",\n" +
" \"query\": \"repo=\\\"smy_%s\\\" origin=\\\"*\\\" AND \\\"ERROR\\\"\\n| where level=\\\"ERROR\\\"\\n| eval err_type=arr_index(split(arr_index(split(_raw, \\\" - \\\"), 1), \\\"\\\\d+\\\"), 0)\\n| stats count() as num by err_type\\n| sort 10 by num\\n| join type=inner err_type [\\n repo=\\\"smy_%s\\\" origin=\\\"*\\\" AND \\\"ERROR\\\"\\n | where level=\\\"ERROR\\\"\\n | eval err_type=arr_index(split(arr_index(split(_raw, \\\" - \\\"), 1), \\\"\\\\d+\\\"), 0)\\n | fields + err_type, _raw\\n | dedup err_type\\n]\\n| rename _raw as 原始日志, num as 统计, err_type as 错误类型\",\n" +
" \"mode\": \"smart\",\n" +
" \"preview\": false,\n" +
" \"collectSize\": -1,\n" +
" \"timeout\": 1000,\n" +
" \"sorts\": []\n" +
"}";
public static ThreadPoolExecutor threadPool = new ThreadPoolExecutor(10, 10, 0, TimeUnit.MINUTES,
new LinkedBlockingQueue<Runnable>(100), new CustomizableThreadFactory("Log_Center-pool-"));
@Test
public void LogCenterData() throws IOException {
//1.模拟登录 jsoup
jsoupLogin();
for (String systemName : SEARCH_SYSTEM) {
try {
Thread.sleep(1000);
//2.获取key
String id = getKey(systemName);
Thread.sleep(8000);
List<Map<String, Object>> logCenterList = retryUtil.doRetry(4, () -> {
//3.获取查询结果
List<Map<String, Object>> contentList = getContentList(systemName, id);
if (CollectionUtils.isEmpty(contentList)) {
Thread.sleep(8000);
throw new Exception(systemName + "未查询到数据，需要重试！");
}
return contentList;
}, systemName + "获取日志数据");
//4.调微信发送短信
//if ("adv_core".equals(systemName) && CollectionUtils.isNotEmpty(logCenterList)) {
// List<List<Map<String, Object>>> partition = Lists.partition(logCenterList, 5);
// partition.stream().forEach(k -> sendWxMessage(systemName, id, k));
//} else {
sendWxMessage(systemName, id, logCenterList);
//}
} catch (Exception e) {
e.printStackTrace();
}
}
}
public static String getKey(String systemName) {
Date endDate = new Date();
Date startDate = DateUtil.offsetHour(endDate, -PERIOD_HOUR);
//请求参数
JSONObject paramJson = JSON.parseObject(String.format(QUERY_TEMPLATE, systemName, systemName));
paramJson.put("startTime",startDate.getTime());
paramJson.put("endTime",endDate.getTime());
HttpResponse execute = HttpRequest.post(LOG_CENTER_BASE_URL + "jobs")
//设置请求头(可任意加)
.header("X-Csrf-Token", X_Csrf_Token)
.header("Cookie", COOKIE)
.header("Content-Type", "application/json")
.header("Connection","keep-alive")
//请求参数
.body(paramJson.toJSONString())
.timeout(40000)
.execute();
String body1 = execute.body();
String id = JSON.parseObject(body1).getString("id");
return id;
}
public static List<Map<String,Object>> getContentList(String systemName,String id) {
String url = LOG_CENTER_BASE_URL + "jobs/" + id + "/results";
url += "?_=" + System.currentTimeMillis();
HttpResponse execute = HttpRequest.get(url)
//设置请求头(可任意加)
.header("X-Csrf-Token", X_Csrf_Token)
.header("Cookie", COOKIE)
.header("Content-Type", "application/json")
.header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36")
.header("Sec-Ch-Ua-Platform", "Windows")
.header("Connection","keep-alive")
.timeout(40000)
.execute();
JSONArray rows = JSON.parseObject(execute.body()).getJSONArray("rows");
System.err.println(rows.toString());
List<Map<String, Object>> list = Lists.newArrayList();
for (Object row : rows) {
try {
JSONArray jsonArray = JSON.parseArray(row.toString());
String title = jsonArray.get(0).toString();
Integer count = Integer.valueOf(jsonArray.get(1).toString());
String content = jsonArray.get(2).toString();
if (count >= maxCount) {
continue;
}
if (filterContent.stream().anyMatch(k -> content.contains(k))) {
continue;
}
int subContentLen = content.length() <= SUB_CONTENT_LENGTH ? content.length() : SUB_CONTENT_LENGTH;
String subContent = content.substring(0, subContentLen);
int subTitleLen = title.length() <= SUB_TITLE_LENGTH ? title.length() : SUB_TITLE_LENGTH;
String subTitle = content.substring(0, subTitleLen);
Map<String, Object> map = new HashMap<>();
map.put("title", subTitle);
map.put("count", count);
map.put("content", subContent);
list.add(map);
} catch (Exception e) {
e.printStackTrace();
}
}
if (CollectionUtils.isEmpty(list)) {
System.err.println(systemName + ":未获取到数据，请求链接：" + url + " token:" + X_Csrf_Token + " cookie:" + COOKIE);
}
return list;
}
public static void sendWxMessage(String systemName, String id, List<Map<String, Object>> contentList) {
if (CollectionUtils.isEmpty(contentList)) {
System.err.println(systemName + "：发送内容为空，不发送机器人微信消息,id:" + id);
return;
}
String periodDesc = getPeriodDesc();
final int[] topNum = {1};
String StringContent = contentList.stream().map(k -> {
String title = (String) k.get("title");
Integer count = (Integer) k.get("count");
String content = (String) k.get("content");
//return String.format("> top-%d：%s\n> 出现次数：%d\n> 错误详情描述: %s", topNum[0]++, title, count, content.trim());
return String.format("> top-%d：出现次数：%d\n> 错误详情描述: %s", topNum[0]++, count, content.trim());
}).collect(Collectors.joining("\n\n"));
String content = String.format("%s近%s内错误日志Top10\n%s", systemName, periodDesc, StringContent);
String sendContent = String.format(TEMPLATE, content);//
//Map<String, String> contentMap = new HashMap<>();
//contentMap.put("content", content);
//Map<String, String> markdownMap = new HashMap<>();
//markdownMap.put("markdown", JSON.toJSONString(contentMap));
//markdownMap.put("msgtype", "markdown");
//String sendText = JSON.toJSONString(markdownMap);//
//System.err.println(sendText);
Arrays.stream(WX_ROBOT_URL.split(",")).forEach(wx -> {
//String post = HttpUtil.post(wx + "&debug=1", sendText);
//String post2 = HttpUtil.post(wx + "&debug=1", sendContent);
String post2 = HttpUtil.post(wx, sendContent);
System.err.println(systemName + "发送微信情况2" + post2);
//System.err.println(systemName + "发送微信情况" + post);
});
}
private static String getPeriodDesc() {
if (PERIOD_HOUR <= 24) {
return PERIOD_HOUR + "小时";
}
BigDecimal dayBigDecimal = BigDecimal.valueOf(PERIOD_HOUR).divide(BigDecimal.valueOf(24), 2, BigDecimal.ROUND_HALF_UP);
String s = StrUtil.removeSuffix(StrUtil.removeSuffix(String.valueOf(dayBigDecimal), "00"), "0");
String[] split = s.split("\\.");
if (split.length == 1) {
return split[0] + "天";
}
return s + "天";
}
/**
* Jsoup 模拟登录 访问个人中心
* 先构造登录请求参数，成功后获取到cookies
* 设置request cookies，再次请求
* @throws IOException
*/
public static void jsoupLogin() throws IOException {
//Jsoup加这个，避免请求https报证书问题
trustEveryone();
// 构造登陆参数
Map<String,String> data = new HashMap<>();
data.put("username", USER_NAME);
data.put("password", PASSWORD);
Connection.Response response = Jsoup.connect(LOG_CENTER_BASE_URL + "account/ldap/login")
.ignoreContentType(true) // 忽略类型验证
.ignoreHttpErrors(true)
.followRedirects(false) // 禁止重定向
.postDataCharset("utf-8")
.header("Upgrade-Insecure-Requests","1")
.header("Accept","application/json")
.header("Content-Type","application/json")
.header("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36")
.requestBody(JSON.toJSONString(data))
.method(Connection.Method.POST)
.execute();
response.charset("UTF-8");
// login 中已经获取到登录成功之后的cookies
// 构造访问个人中心的请求
Map<String, String> cookies = response.cookies();
cookies.forEach((k, v) -> COOKIE = k + "=" + v);
String body = response.body();
X_Csrf_Token = JSON.parseObject(body).getString("X-Csrf-Token");
System.err.println("COOKIE:" + COOKIE);
System.err.println("X_Csrf_Token:" + X_Csrf_Token);
}
public static void jsoupHandle(String id) throws IOException {
String url = LOG_CENTER_BASE_URL + "jobs/" + id + "/results";
Document document = Jsoup.connect(url)
.header("X-Csrf-Token", X_Csrf_Token)
.header("Cookie", COOKIE)
.ignoreContentType(true) // 忽略类型验证
.ignoreHttpErrors(true)
.method(Connection.Method.GET)
.get();
System.err.println(url);
String s = JSON.toJSONString(document);
System.err.println(s);
}
public static void trustEveryone() {
try {
HttpsURLConnection.setDefaultHostnameVerifier(new HostnameVerifier() {
public boolean verify(String hostname, SSLSession session) {
return true;
}
});
SSLContext context = SSLContext.getInstance("TLS");
context.init(null, new X509TrustManager[] { new X509TrustManager() {
public void checkClientTrusted(X509Certificate[] chain, String authType) throws CertificateException {
}
public void checkServerTrusted(X509Certificate[] chain, String authType) throws CertificateException {
}
public X509Certificate[] getAcceptedIssuers() {
return new X509Certificate[0];
}
} }, new SecureRandom());
HttpsURLConnection.setDefaultSSLSocketFactory(context.getSocketFactory());
} catch (Exception e) {
// e.printStackTrace();
}
}
}三、实现效果
企业微信机器狗开发者文档：群机器人配置说明 - 接口文档 - 企业微信开发者中心
