正则采集器之二—

本人是java程序员，所以使用java界流行的架构来搭建后台。后台使用java作为后端语言，mysql作为数据库，redis作为缓存中间件。采用SpringBoot作为java端依赖管理、bean生命周期管理的容器，mybatis作为数据库持久化框架，shiro作为鉴权框架。

1、新建maven项目

命名为reptile，接着加入依赖:

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
  <parent>
	    <groupId>org.springframework.boot</groupId>
	    <artifactId>spring-boot-starter-parent</artifactId>
	    <version>2.4.2</version>
	</parent>
  
  <modelVersion>4.0.0</modelVersion>
  <groupId>com.learn.reptile</groupId>
  <artifactId>reptile</artifactId>
  <version>0.0.1-SNAPSHOT</version>
  
  <properties>
    <java.version>1.8</java.version>
    <mybatis.plus.version>3.4.2</mybatis.plus.version>
    <druid.version>1.1.18</druid.version>
    <mysql.version>8.0.13</mysql.version>
    <!--<log4j.version>1.2.17</log4j.version>-->

    <hutool.version>5.5.8</hutool.version>
    <guava.version>28.1-jre</guava.version>
    
    <shiro.version>1.13.0</shiro.version>
	<httpclient.version>4.5.3</httpclient.version>
  </properties>
  
  <dependencies>
        <!-- Druid引入 -->
        <dependency>
            <groupId>com.alibaba</groupId>
            <artifactId>druid-spring-boot-starter</artifactId>
            <version>${druid.version}</version>
        </dependency>

        <!-- 数据源驱动 -->
        <dependency>
            <groupId>mysql</groupId>
            <artifactId>mysql-connector-java</artifactId>
            <version>${mysql.version}</version>
        </dependency>
        
        <!-- mybatis-plus依赖 -->
        <dependency>
            <groupId>com.baomidou</groupId>
            <artifactId>mybatis-plus-boot-starter</artifactId>
            <version>${mybatis.plus.version}</version>
        </dependency>
        
        <dependency>
	        <groupId>org.springframework.boot</groupId>
	        <artifactId>spring-boot-starter-web</artifactId>
	        <exclusions>
                <exclusion>
                    <groupId>org.springframework.boot</groupId>
                    <artifactId>spring-boot-starter-logging</artifactId>
                </exclusion>
            </exclusions>
	    </dependency>
	    
	    <dependency>
			<groupId>org.springframework.boot</groupId>
			<artifactId>spring-boot-starter-log4j2</artifactId>
			<version>2.4.2</version>
		</dependency>
		
		  <!-- spring-boot-starter-data-redis -->
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-data-redis</artifactId>
        </dependency>

        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-aop</artifactId>
        </dependency>

        <dependency>
            <groupId>redis.clients</groupId>
            <artifactId>jedis</artifactId>
        </dependency>
        
        <dependency>
            <groupId>org.projectlombok</groupId>
            <artifactId>lombok</artifactId>
            <!--            <optional>true</optional>-->
        </dependency>
        
         <!-- guava支持 -->
        <dependency>
            <groupId>com.google.guava</groupId>
            <artifactId>guava</artifactId>
            <version>${guava.version}</version>
        </dependency>

        <dependency>
            <groupId>com.alibaba.fastjson2</groupId>
            <artifactId>fastjson2</artifactId>
            <version>2.0.13</version>
        </dependency>
        
        
	    <dependency>
	        <groupId>org.springframework.boot</groupId>
	        <artifactId>spring-boot-devtools</artifactId>
	        <scope>runtime</scope>
	        <optional>true</optional>
	    </dependency>
	    
	     <!--        shiro 核心类库-->
	    <dependency>
		    <groupId>org.apache.shiro</groupId>
		    <artifactId>shiro-core</artifactId>
		    <version>${shiro.version}</version>
	    </dependency>
	    <!-- shiro 整合sping -->
	    <dependency>
	        <groupId>org.apache.shiro</groupId>
	        <artifactId>shiro-spring</artifactId>
	        <version>${shiro.version}</version>
	    </dependency>
	
	    <!-- 日志框架依赖 -->
	    <!--<dependency>
	        <groupId>log4j</groupId>
	        <artifactId>log4j</artifactId>
	        <version>${log4j.version}</version>
	    </dependency>-->
	
	    <dependency>
	        <groupId>org.apache.httpcomponents</groupId>
	        <artifactId>httpclient</artifactId>
	        <version>${httpclient.version}</version>
	    </dependency>
	    
	    <dependency>
			<groupId>cn.hutool</groupId>
			<artifactId>hutool-all</artifactId>
			<version>${hutool.version}</version>
		</dependency>
        
         <!--        okhttp依赖-->
        <dependency>
            <groupId>com.squareup.okhttp3</groupId>
            <artifactId>okhttp</artifactId>
            <version>4.9.1</version>
        </dependency>
	</dependencies>
	
	 <build>
	    <resources>
	        <resource>
	            <directory>src/main/resources</directory>
	            <filtering>true</filtering>
	        </resource>
	        <resource>
	            <directory>src/main/java</directory>
	            <includes>
	                <include>**/*.xml</include>
	            </includes>
	        </resource>
	    </resources>
	    
	    <plugins>
	        <plugin>
	            <groupId>org.apache.maven.plugins</groupId>
	            <artifactId>maven-compiler-plugin</artifactId>
	            <configuration>
	                <source>8</source>
	                <target>8</target>
	            </configuration>
	        </plugin>
			 
			 <plugin>
	            <groupId>org.springframework.boot</groupId>
	            <artifactId>spring-boot-maven-plugin</artifactId>
	            <configuration>
	                <includeSystemScope>true</includeSystemScope>
	            </configuration>
	            <executions>
	                <execution>
	                    <goals>
	                        <goal>repackage</goal>
	                    </goals>
	                </execution>
	            </executions>
	        </plugin>
	   </plugins>
	</build>
</project>

接着新建包

com.learn.reptile.config 配置文件
com.learn.reptile.entity 实体类
com.learn.reptile.mapper 数据库操作接口
com.learn.reptile.utils 工具类
com.learn.reptile.web web层类

新建启动类 ReptileApplication

package com.learn.reptile;

import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;

@SpringBootApplication(scanBasePackages = "com.learn.reptile")
public class ReptileApplication {
	
	public static void main(String[] args) {
		SpringApplication.run(ReptileApplication.class, args);
	}

}

2、配置文件

在src/main/resources下新建配置文件application.yml，内容如下:

server:
  port: 8080
  
spring:
  application:
    name: reptile
  main:
    allow-bean-definition-overriding: true
  jackson:
    date-format: yyyy-MM-dd HH:mm
    time-zone: GMT+8
  servlet:
    multipart:
      max-file-size: 100MB
      max-request-size: 100MB
  redis:
    database: 1
    host: sincerely-host
    password: 85861367LiGzy
  datasource:
    username: test
    password: test
    url: jdbc:mysql://localhost:3306/reptile?useUnicode=true&useSSL=false&characterEncoding=utf-8&serverTimezone=Asia/Shanghai&allowMultiQueries=true&allowPublicKeyRetrieval=true
    driver-class-name: com.mysql.cj.jdbc.Driver
    initial-size: 2
    min-idle: 1
    max-active: 20
    max-wait: 60000
    timeBetweenEvictionRunsMillis: 60000
    minEvictableIdleTimeMillis: 300000
    validationQuery: "select 'x'"
    testWhileIdle: true
    testOnBorrow: false
    testOnReturn: false
    poolPreparedStatements: true
    maxPoolPreparedStatementPerConnectionSize: 20
    filters: stat
    
mybatis-plus:
  global-config:
    id-type: 0
    db-column-underline: false
    refresh-mapper: true
  type-aliases-package: com.learn.reptile.entity.po
  mapper-locations: classpath:/mapper/*.xml
  configuration:
    log-impl: org.apache.ibatis.logging.stdout.StdOutImpl
    
    
logging:
  config: classpath:log4j2.xml

shiro:
  session-timeout: 1000000

log4j2.xml定义好日志级别、记录位置和格式

<?xml version="1.0" encoding="UTF-8"?>
<Configuration status="WARN" monitorInterval="30">
    <Properties>
        <Property name="LOG_PATTERN">
            %d{yyyy-MM-dd HH:mm:ss.SSS} %5p %logger{36} --- [%15.15t] %l : %m%n
        </Property>
    </Properties>
    <Appenders>
        <Console name="Console" target="SYSTEM_OUT" follow="true">
            <PatternLayout pattern="${LOG_PATTERN}"/>
        </Console>
        <RollingRandomAccessFile name="DebugFile"
            fileName="logs/debug.log"
            filePattern="logs/debug-%d{yyyy-MM-dd}.log">
            <PatternLayout pattern="${LOG_PATTERN}"/>
            <Policies>
                <TimeBasedTriggeringPolicy interval="1" modulate="true" />
                <SizeBasedTriggeringPolicy size="10MB" />
            </Policies>
            <!-- Max 10 files will be created everyday -->
            <DefaultRolloverStrategy max="10">
                <Delete basePath="logs" maxDepth="10">
                    <!-- Delete all files older than 30 days -->
                    <IfLastModified age="30d" />
                </Delete>
            </DefaultRolloverStrategy>
        </RollingRandomAccessFile>
        <Async name="AsyncDebugFile">
            <AppenderRef ref="DebugFile" />
        </Async>
        
        <RollingRandomAccessFile name="InfoFile"
            fileName="logs/info.log"
            filePattern="logs/info-%d{yyyy-MM-dd}.log">
            <PatternLayout pattern="${LOG_PATTERN}"/>
            <Policies>
                <TimeBasedTriggeringPolicy interval="1" modulate="true" />
                <SizeBasedTriggeringPolicy size="10MB" />
            </Policies>
            <!-- Max 10 files will be created everyday -->
            <DefaultRolloverStrategy max="10">
                <Delete basePath="logs" maxDepth="10">
                    <!-- Delete all files older than 30 days -->
                    <IfLastModified age="30d" />
                </Delete>
            </DefaultRolloverStrategy>
        </RollingRandomAccessFile>
        <Async name="AsyncInfoFile">
            <AppenderRef ref="InfoFile" />
        </Async>
        
         <RollingRandomAccessFile name="WarnFile"
            fileName="logs/warn.log"
            filePattern="logs/warn-%d{yyyy-MM-dd}.log">
            <PatternLayout pattern="${LOG_PATTERN}"/>
            <Policies>
                <TimeBasedTriggeringPolicy interval="1" modulate="true" />
                <SizeBasedTriggeringPolicy size="10MB" />
            </Policies>
            <!-- Max 10 files will be created everyday -->
            <DefaultRolloverStrategy max="10">
                <Delete basePath="logs" maxDepth="10">
                    <!-- Delete all files older than 30 days -->
                    <IfLastModified age="30d" />
                </Delete>
            </DefaultRolloverStrategy>
        </RollingRandomAccessFile>
        <Async name="AsyncWarnFile">
            <AppenderRef ref="WarnFile" />
        </Async>
        
         <RollingRandomAccessFile name="ErrorFile"
            fileName="logs/error.log"
            filePattern="logs/error-%d{yyyy-MM-dd}.log">
            <PatternLayout pattern="${LOG_PATTERN}"/>
            <Policies>
                <TimeBasedTriggeringPolicy interval="1" modulate="true" />
                <SizeBasedTriggeringPolicy size="10MB" />
            </Policies>
            <!-- Max 10 files will be created everyday -->
            <DefaultRolloverStrategy max="10">
                <Delete basePath="logs" maxDepth="10">
                    <!-- Delete all files older than 30 days -->
                    <IfLastModified age="30d" />
                </Delete>
            </DefaultRolloverStrategy>
        </RollingRandomAccessFile>
        <Async name="AsyncErrorFile">
            <AppenderRef ref="ErrorFile" />
        </Async>
    </Appenders>
    
    <Loggers>
        <Logger name="com.learn.reptile" level="debug" additivity="false">
            <AppenderRef ref="Console" />
            <AppenderRef ref="AsyncDebugFile" />
        </Logger>
        <Root level="info">
            <AppenderRef ref="Console" />
            <AppenderRef ref="AsyncDebugFile" />
            <AppenderRef ref="AsyncInfoFile" />
            <AppenderRef ref="AsyncWarnFile" />
            <AppenderRef ref="AsyncErrorFile" />
        </Root>
    </Loggers>
</Configuration>

github代码：GitHub - guzhangyu/reptile-api: 商品正则采集器