使用Prometheus对微服务性能自定义指标监控

背景

随着云计算和容器化技术的不断发展，微服务架构逐渐成为现代软件开发的主流趋势。微服务架构将大型应用程序拆分成多个小型、独立的服务，每个服务都可以独立开发、部署和扩展。这种架构模式提高了系统的可伸缩性、灵活性和可靠性，但同时也带来了服务监控和管理的挑战。

在微服务架构中，服务之间的依赖关系变得复杂，服务数量众多，因此需要一种有效的监控和管理工具来确保系统的稳定性和可靠性。监控工具可以帮助开发人员实时了解服务的运行状态、性能指标和异常情况，从而及时发现问题并进行处理。同时，管理工具还可以提供自动化的部署、配置和扩展功能，提高开发效率和运维质量。

Prometheus的优势

请求、数据库查询、消息队列等应用指标。
高效数据存储：Prometheus采用时间序列数据库（TSDB）来存储监控数据，具有高效的数据压缩和查询性能。
丰富查询语言：Prometheus提供了强大的数据查询语言PromQL，可以方便地对监控数据进行过滤、聚合和计算。
灵活告警机制：Prometheus支持基于规则的告警机制，可以根据监控数据的阈值触发告警通知，支持多种告警方式，如邮件、短信、Slack等。

springBoot集成Prometheus

导入Pom依赖

<dependency>
        <groupId>org.mybatis.spring.boot</groupId>
        <artifactId>spring-boot-starter-actuator</artifactId>
        <version>2.2.1.RELEASE</version>
</dependency>
<dependency>
        <groupId>io.micrometer</groupId>
        <artifactId>micrometer-registry-prometheus</artifactId>
        <version>1.3.1</version>
</dependency>

修改springBoot配置文件

开启prometheus监控配置

management:
  endpoint:
    prometheus:
        enabled: true
  endpoints:
      web:
          exposure:
              include: 'prometheus'

修改默认的Prometheus监控度量名称

prometheus默认指标中有个http.server.requests的度量名称，记录了http请求调用情况；现在以这个为例，修改名称

新建一个@Configuration 类 PrometheusConfig

import io.micrometer.core.instrument.Meter;
import io.micrometer.core.instrument.MeterRegistry;
import io.micrometer.core.instrument.config.NamingConvention;
import org.springframework.boot.actuate.autoconfigure.metrics.MeterRegistryCustomizer;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;

import java.util.Arrays;
import java.util.Objects;
import java.util.stream.Collectors;

@Configuration
public class PrometheusConfig {

    /** 用于替换 Prometheus中 公共的 http.server.requests度量名替换
     * @return
     */
    @Bean
    MeterRegistryCustomizer<MeterRegistry> metricsConfig() {
        return registry -> registry.config().namingConvention(new NamingConvention() {
            @Override
            public String name(String name, Meter.Type type, String baseUnit) {
                String collect = "";
                if(name.contains("http.server.requests")){
                    collect = Arrays.stream(name.replaceAll("http.server.requests", "jiang.xiao.yu.http").split("\\.")).filter(Objects::nonNull).collect(Collectors.joining("_"));
                }else {
                    collect = Arrays.stream(name.split("\\.")).filter(Objects::nonNull).collect(Collectors.joining("_"));
                }
                return collect;
            }
        });
    }
}

自定义Prometheus监控指标

使用拦截器监控指标

利用拦截器实现所有HTTP接口的监控

利用HTTP的拦截器添加Prometheus的监控指标，首先创建一个拦截器CustomInterceptor 实现HandlerInterceptor接口，然后重写里面的前置处理、后置处理；

import io.micrometer.core.instrument.*;
import org.springframework.web.servlet.HandlerInterceptor;
import org.springframework.web.servlet.ModelAndView;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.util.*;
import java.util.concurrent.TimeUnit;
import java.util.function.ToDoubleFunction;

public class CustomInterceptor implements HandlerInterceptor {

    private static final String CUSTOM_KPI_NAME_TIMER = "custom.kpi.timer"; //耗时
    private static final String CUSTOM_KPI_NAME_COUNTER = "custom.kpi.counter"; //api调用次数。
    private static final String CUSTOM_KPI_NAME_SUMMARY = "custom.kpi.summary"; //汇总率
    private static MeterRegistry registry;
    private long startTime;
    private GaugeNumber gaugeNumber = new GaugeNumber();

    void getRegistry(){
        if(registry == null){
            //这里使用的时SpringUtil获取Bean，没有用@Autowired注解，Autowired会因为加载时机问题导致拿不到；SpringUtil.getBean网上实现有很多，可以自行搜索；
            registry = SpringUtil.getBean(MeterRegistry.class);
        }
    }

    @Override
    public boolean preHandle(HttpServletRequest request, HttpServletResponse response, Object handler) throws Exception {
        getRegistry();
        //记录接口开始调用的时间
        startTime = System.currentTimeMillis();
        return HandlerInterceptor.super.preHandle(request, response, handler);
    }

    @Override
    public void postHandle(HttpServletRequest request, HttpServletResponse response, Object handler, ModelAndView modelAndView) throws Exception {
        HandlerInterceptor.super.postHandle(request, response, handler, modelAndView);
    }

    @Override
    public void afterCompletion(HttpServletRequest request, HttpServletResponse response, Object handler, Exception ex) throws Exception {
        //统计调用次数
        registry.counter(CUSTOM_KPI_NAME_COUNTER,"uri", request.getRequestURI(), "method", request.getMethod(),
                "status", response.getStatus() + "", "exception", ex == null ? "" : ex.getMessage(), "outcome", response.getStatus() == 200 ? "SUCCESS" : "CLIENT_ERROR").increment();
        //统计单次耗时
        registry.timer(CUSTOM_KPI_NAME_TIMER,"uri", request.getRequestURI(), "method", request.getMethod(),
                "status", response.getStatus() + "", "exception", ex == null ? "" : ex.getMessage(), "outcome", response.getStatus() == 200 ? "SUCCESS" : "CLIENT_ERROR").record(System.currentTimeMillis() - startTime, TimeUnit.MILLISECONDS);
        //统计调用成功率，根据过滤Counter对象，获取计数
        Collection<Meter> meters = registry.get(CUSTOM_KPI_NAME_COUNTER).tag("uri", request.getRequestURI()).tag("method", request.getMethod()).meters();
        double total = 0;
        double success = 0;
        for (Meter meter : meters) {
            Counter counter = (Counter) meter;
            total += counter.count();
            String status = meter.getId().getTag("status");
            if (status.equals("200")){
                success+= counter.count();
            }
        }
        //保存对应的成功率到Map中
        String key = request.getMethod() + request.getRequestURI();
        gaugeNumber.setPercent(key, Double.valueOf(success / total * 100L));
        registry.gauge(CUSTOM_KPI_NAME_SUMMARY, Tags.of("uri", request.getRequestURI(), "method", request.getMethod()), gaugeNumber, new ToDoubleFunction<GaugeNumber>() {
            @Override
            public double applyAsDouble(GaugeNumber value) {
                return value.getPercent(key);
            }
        });
        HandlerInterceptor.super.afterCompletion(request, response, handler, ex);
    }
    // gauge监控某个对象，所以用内部类替代，然后根据tag标签区分对应的成功率；key 为 method + uri
    class GaugeNumber {
        Map<String,Double> map = new HashMap<>();

        public Double getPercent(String key) {
            return map.get(key);
        }

        public void setPercent(String key, Double percent) {
            map.put(key, percent);
        }
    }
}

注册自定义拦截器给Spring

import org.springframework.context.annotation.Configuration;
import org.springframework.web.servlet.config.annotation.InterceptorRegistry;
import org.springframework.web.servlet.config.annotation.WebMvcConfigurer;

@Configuration
public class CustomInterceptors implements WebMvcConfigurer {

    @Override
    public void addInterceptors(InterceptorRegistry registry) {
        registry.addInterceptor(new CustomInterceptor()).addPathPatterns("/**");
    }
}

大功告成，启动程序测试吧

使用AOP记录监控指标

自定义指标注解

@Documented
@Retention(RetentionPolicy.RUNTIME)
@Target(ElementType.METHOD)
public @interface MethodMetrics {
    String name() default "";
    String desc() default "";
    String[] tags() default {};
    //是否记录时间间隔
    boolean withoutDuration() default false;
}

切面实现

@Aspect
public class PrometheusAnnotationAspect {

    @Autowired
    private MeterRegistry meterRegistry;

    @Pointcut("@annotation(com.smac.prometheus.annotation.MethodMetrics)")
    public void pointcut() {}

    @Around(value = "pointcut()")
    public Object process(ProceedingJoinPoint joinPoint) throws Throwable {
        Method targetMethod = ((MethodSignature) joinPoint.getSignature()).getMethod();
        Method currentMethod = ClassUtils.getUserClass(joinPoint.getTarget().getClass()).getDeclaredMethod(targetMethod.getName(), targetMethod.getParameterTypes());
        if (currentMethod.isAnnotationPresent(MethodMetrics.class)) {
            MethodMetrics methodMetrics = currentMethod.getAnnotation(MethodMetrics.class);
            return processMetric(joinPoint, currentMethod, methodMetrics);
        } else {
            return joinPoint.proceed();
        }
    }

    private Object processMetric(ProceedingJoinPoint joinPoint, Method currentMethod, MethodMetrics methodMetrics) {
        String name = methodMetrics.name();
        if (!StringUtils.hasText(name)) {
            name = currentMethod.getName();
        }
        String desc = methodMetrics.desc();
        if (!StringUtils.hasText(desc)) {
            desc = currentMethod.getName();
        }
        //不需要记录时间
        if (methodMetrics.withoutDuration()) {
            Counter counter = Counter.builder(name).tags(methodMetrics.tags()).description(desc).register(meterRegistry);
            try {
                return joinPoint.proceed();
            } catch (Throwable e) {
                throw new IllegalStateException(e);
            } finally {
                counter.increment();
            }
        }
        //需要记录时间（默认）
        Timer timer = Timer.builder(name).tags(methodMetrics.tags()).description(desc).register(meterRegistry);
        return timer.record(() -> {
            try {
                return joinPoint.proceed();
            } catch (Throwable e) {
                throw new IllegalStateException(e);
            }
        });
    }
}

在需要记监控的地方加上这个注解

@MethodMetrics(name="sms_send",tags = {"vendor"})
public void send(String mobile, SendMessage message) throws Exception {
    //do something
}