目录

使用RestTemplate获取国内大盘股票数据的基本信息并存入数据库中

第一步：导入RestTemplate依赖，并配置RestTemplate让其加入到SpringIoC容器中

第二步：在yml文件定义股票的相关参数

第三步：向新浪网发送请求，获取国内大盘股票的JS格式数据

第四步：使用正则表达式解析JS数据格式，并把解析后的数据封装到entity对象中

第五步：使用mybatis把entities集合批量入库

 mapper

mapperXml文件

第六步：查看数据库是否插入成功

---

使用RestTemplate获取国内大盘股票数据的基本信息并存入数据库中

基本流程：

使用RestTemplate向sina网发送请求，并获取返回的JS数据格式

var hq_str_sh000001="上证指数,3358.9338,3361.5177,3398.6161,3417.0085,3358.9338,0,0,381243178,510307202948,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2022-06-30,15:30:39,00,";
参数说明：
0：指数名称
1：开盘点
2：前收盘点
3：当前点
4：最高点
5：最低点
8：成交量
9：成交金额
30：当前日期
31：当前时间

获取到JS数据后，使用正则表达式解析JS数据，并把解析后的数据封装到entity对象中，在把entity对象插入到数据库中 

第一步：导入RestTemplate依赖，并配置RestTemplate让其加入到SpringIoC容器中

 <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-web</artifactId>
        </dependency>

/**
 * 定义http客户端工具bean
 */
@Configuration
public class HttpClientConfig {
    /**
     * 定义http客户端工具bean
     */
    @Bean
    public RestTemplate restTemplate(){
        return new RestTemplate();
    }
}

第二步：在yml文件定义股票的相关参数

# 配置股票相关的参数
stock:
  inner:   #A股
    - sh000001 # 上证ID
    - sz399001 #  深证ID
  outer: # 外盘
    - int_dji # 道琼斯
    - int_nasdaq # 纳斯达克
    - int_hangseng # 恒生
    - int_nikkei # 日经指数
    - b_FSSTI # 新加坡
  marketUrl: https://hq.sinajs.cn/list=
  blockUrl: http://vip.stock.finance.sina.com.cn/q/view/newSinaHy.php

在value object 包下写一个与yml文件相互映射的类

/**
 * 大盘编码信息
 */
@Data
@ConfigurationProperties(prefix = "stock")//映射yml文件中stock属性的值
//@Component//直接让其交给spring管理，TODO:我们不使用（不让它自己开启），我们让其他要使用该类的子模块来开启
public class StockInfoConfig {
    private List<String>inner;//国内大盘编码
    private List<String>outer;//国外大盘编码
    private String marketUrl;//大盘 外盘 个股的公共Url请求路径
    private String blockUrl;//板块采集url请求地址
}

开启这个与yml文件相互映射的类，并交给spring管理

@Configuration
//开启StockInfoConfig这个类，开始让这个类映射加载yml文件的属性，并交给spring管理
@EnableConfigurationProperties(StockInfoConfig.class)
public class CommonConfig {
    //配置雪花算法的工具类bean，交给spring管理
    @Bean
    public IdWorker idWorker(){
        /**
         * 参数一：机器ID
         * 参数二：机房ID
         */
        return new IdWorker(1L,2L);
    }
}

第三步：向新浪网发送请求，获取国内大盘股票的JS格式数据

@Service
@Slf4j
public class StockTimerTaskServiceImpl implements StockTimerTaskService {
    @Autowired
    private StockInfoConfig stockInfoConfig;
    @Autowired
    private RestTemplate restTemplate;
    @Autowired
    private IdWorker idWorker;
    @Autowired
    private StockMarketIndexInfoMapper stockMarketIndexInfoMapper;
    @Override
    public void getInnerMarketInfo() {
        //TODO:1.从新浪网获取国内大盘的JS数据格式
        //设置请求路径
        //https://hq.sinajs.cn/list=sh000001,sz399001
        String url=stockInfoConfig.getMarketUrl()+String.join(",",stockInfoConfig.getInner());
        //join方法可以把集合内的数据装换成String数据类型，并使用,分隔
        //设置请求头
        HttpHeaders headers = new HttpHeaders();
        //必须填写，否则数据采集不到
        headers.add("Referer","https://finance.sina.com.cn/stock/");
        headers.add("User-Agent","Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Safari/537.36");
        //组装请求对象
        HttpEntity<Object> httpEntity = new HttpEntity<>(headers);
        //使用restTemplate发送请求数据
        ResponseEntity<String> responseEntity = restTemplate.exchange(url, HttpMethod.GET, httpEntity, String.class);
        //获取响应码
        int statusCode= responseEntity.getStatusCodeValue();
        if(statusCode!=200){
            log.error("时间:{}，采集数据出错，响应码状态为：{}", DateTime.now().toString(DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss")),statusCode);
            return;
        }
        //获取js响应数据
        String resData = responseEntity.getBody();
        log.info("时间:{},采集数据成功，数据为:{}",DateTime.now().toString(DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss")),resData);
        /**
         * 结果：
         * 时间:2024-09-06 20:05:04,采集数据成功，数据为:
         * var hq_str_sh000001="上证指数,2791.7645,2788.3141,2765.8066,2804.0932,2765.6394,0,0,253645375,228570135027,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2024-09-06,15:30:39,00,";
         * var hq_str_sz399001="深证成指,8252.006,8249.659,8130.770,8260.690,8128.767,0.000,0.000,36378777179,314062276972.566,0,0.000,0,0.000,0,0.000,0,0.000,0,0.000,0,0.000,0,0.000,0,0.000,0,0.000,0,0.000,2024-09-06,15:00:03,00";
         */
    }
}

第四步：使用正则表达式解析JS数据格式，并把解析后的数据封装到entity对象中

entity类对象（直接与数据库字段值进行交互）

/**
 * 国内大盘数据详情表
 * @TableName stock_market_index_info
 */
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class StockMarketIndexInfo implements Serializable {
    /**
     * 主键字段（无业务意义）
     */
    private Long id;

    /**
     * 大盘编码
     */
    private String marketCode;

    /**
     * 指数名称
     */
    private String marketName;

    /**
     * 前收盘点数
     */
    private BigDecimal preClosePoint;

    /**
     * 开盘点数
     */
    private BigDecimal openPoint;

    /**
     * 当前点数
     */
    private BigDecimal curPoint;

    /**
     * 最低点数
     */
    private BigDecimal minPoint;

    /**
     * 最高点数
     */
    private BigDecimal maxPoint;

    /**
     * 成交量(手)
     */
    private Long tradeAmount;

    /**
     * 成交金额（元）
     */
    private BigDecimal tradeVolume;

    /**
     * 当前时间
     */
    private Date curTime;

    private static final long serialVersionUID = 1L;
}

@Service
@Slf4j
public class StockTimerTaskServiceImpl implements StockTimerTaskService {
    @Autowired
    private StockInfoConfig stockInfoConfig;
    @Autowired
    private RestTemplate restTemplate;
    @Autowired
    private IdWorker idWorker;
    @Autowired
    private StockMarketIndexInfoMapper stockMarketIndexInfoMapper;
    @Override
    public void getInnerMarketInfo() {
        //TODO:1.从新浪网获取国内大盘的JS数据格式
        //设置请求路径
        //https://hq.sinajs.cn/list=sh000001,sz399001
        String url=stockInfoConfig.getMarketUrl()+String.join(",",stockInfoConfig.getInner());
        //join方法可以把集合内的数据装换成String数据类型，并使用,分隔
        //设置请求头
        HttpHeaders headers = new HttpHeaders();
        //必须填写，否则数据采集不到
        headers.add("Referer","https://finance.sina.com.cn/stock/");
        headers.add("User-Agent","Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Safari/537.36");
        //组装请求对象
        HttpEntity<Object> httpEntity = new HttpEntity<>(headers);
        //使用restTemplate发送请求数据
        ResponseEntity<String> responseEntity = restTemplate.exchange(url, HttpMethod.GET, httpEntity, String.class);
        //获取响应码
        int statusCode= responseEntity.getStatusCodeValue();
        if(statusCode!=200){
            log.error("时间:{}，采集数据出错，响应码状态为：{}", DateTime.now().toString(DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss")),statusCode);
            return;
        }
        //获取js响应数据
        String resData = responseEntity.getBody();
        log.info("时间:{},采集数据成功，数据为:{}",DateTime.now().toString(DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss")),resData);
        /**
         * 结果：
         * 时间:2024-09-06 20:05:04,采集数据成功，数据为:
         * var hq_str_sh000001="上证指数,2791.7645,2788.3141,2765.8066,2804.0932,2765.6394,0,0,253645375,228570135027,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2024-09-06,15:30:39,00,";
         * var hq_str_sz399001="深证成指,8252.006,8249.659,8130.770,8260.690,8128.767,0.000,0.000,36378777179,314062276972.566,0,0.000,0,0.000,0,0.000,0,0.000,0,0.000,0,0.000,0,0.000,0,0.000,0,0.000,0,0.000,2024-09-06,15:00:03,00";
         */

        //2.TODO:使用正则解析JS数据
        //定义正则表达式
        String regex="hq_str_(.+)=\"(.+)\";";
        //表达式编译
        Pattern pattern = Pattern.compile(regex);
        //匹配字符串
        Matcher matcher = pattern.matcher(resData);
        List<StockMarketIndexInfo>entities=new ArrayList<>();
        while (matcher.find()){//有多行数据（以 ; 结尾），要使用while循环
            //如果使用的是group(0),则获取匹配的全部数据
            //获取大盘的编码
            String marketCode = matcher.group(1);//获取第一个括号匹配的数据
            //获取大盘的其他信息
            String otherInfo=matcher.group(2);//获取第二个括号匹配的数据
            //分割其他信息成String数据
            String[] splitArr = otherInfo.split(",");
            //大盘名称
            String marketName=splitArr[0];
            //获取当前大盘的开盘点数
            BigDecimal openPoint=new BigDecimal(splitArr[1]);//直接把String类型的数据变成BigDecimal数据类型
            //前收盘点
            BigDecimal preClosePoint=new BigDecimal(splitArr[2]);
            //获取大盘的当前点数
            BigDecimal curPoint=new BigDecimal(splitArr[3]);
            //获取大盘最高点
            BigDecimal maxPoint=new BigDecimal(splitArr[4]);
            //获取大盘的最低点
            BigDecimal minPoint=new BigDecimal(splitArr[5]);
            //获取成交量
            Long tradeAmt=Long.valueOf(splitArr[8]);
            //获取成交金额
            BigDecimal tradeVol=new BigDecimal(splitArr[9]);
            //获取交易时间
            DateTime dateTime = DateTime.parse(splitArr[30] + " " + splitArr[31], DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss"));
            Date date = dateTime.withSecondOfMinute(0).withMillisOfSecond(0).toDate();

            //TODO:3.把获取的数据封装成entity对象，用来插入数据库
            StockMarketIndexInfo entity = StockMarketIndexInfo.builder()
                    .id(idWorker.nextId())//使用雪花算法生成一个Long类型的id
                    .marketCode(marketCode)
                    .marketName(marketName)
                    .openPoint(openPoint)
                    .preClosePoint(preClosePoint)
                    .curPoint(curPoint)
                    .maxPoint(maxPoint)
                    .minPoint(minPoint)
                    .tradeAmount(tradeAmt)
                    .tradeVolume(tradeVol)
                    .curTime(date).build();
            entities.add(entity);
        }
        log.info("当前时间:{},获取的数据为:{}",DateTime.now().toString(DateTimeFormat.forPattern("yyyy-MM-ss HH:mm:ss")),entities);
        /**
         * 当前时间:2024-09-14 20:52:14,获取的数据为:
         * [
         * StockMarketIndexInfo(id=1832039060873678848, marketCode=sh000001, marketName=上证指数, preClosePoint=2788.3141, openPoint=2791.7645, curPoint=2765.8066, minPoint=2765.6394, maxPoint=2804.0932, tradeAmount=253645375, tradeVolume=228570135027, curTime=Fri Sep 06 15:30:00 CST 2024),
         * StockMarketIndexInfo(id=1832039103492001792, marketCode=sz399001, marketName=深证成指, preClosePoint=8249.659, openPoint=8252.006, curPoint=8130.770, minPoint=8128.767, maxPoint=8260.690, tradeAmount=36378777179, tradeVolume=314062276972.566, curTime=Fri Sep 06 15:00:00 CST 2024)
         * ]
     
    }
}

第五步：使用mybatis把entities集合批量入库

import com.hhh.stock.mapper.StockMarketIndexInfoMapper;
import com.hhh.stock.pojo.entity.StockMarketIndexInfo;
import com.hhh.stock.pojo.vo.StockInfoConfig;
import com.hhh.stock.service.StockTimerTaskService;
import com.hhh.stock.utils.IdWorker;
import lombok.extern.slf4j.Slf4j;
import org.joda.time.DateTime;
import org.joda.time.format.DateTimeFormat;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.*;
import org.springframework.stereotype.Service;
import org.springframework.web.client.RestTemplate;

import java.math.BigDecimal;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

@Service
@Slf4j
public class StockTimerTaskServiceImpl implements StockTimerTaskService {
    @Autowired
    private StockInfoConfig stockInfoConfig;
    @Autowired
    private RestTemplate restTemplate;
    @Autowired
    private IdWorker idWorker;
    @Autowired
    private StockMarketIndexInfoMapper stockMarketIndexInfoMapper;
    @Override
    public void getInnerMarketInfo() {
        //TODO:1.从新浪网获取国内大盘的JS数据格式
        //设置请求路径
        //https://hq.sinajs.cn/list=sh000001,sz399001
        String url=stockInfoConfig.getMarketUrl()+String.join(",",stockInfoConfig.getInner());
        //join方法可以把集合内的数据装换成String数据类型，并使用,分隔
        //设置请求头
        HttpHeaders headers = new HttpHeaders();
        //必须填写，否则数据采集不到
        headers.add("Referer","https://finance.sina.com.cn/stock/");
        headers.add("User-Agent","Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Safari/537.36");
        //组装请求对象
        HttpEntity<Object> httpEntity = new HttpEntity<>(headers);
        //使用restTemplate发送请求数据
        ResponseEntity<String> responseEntity = restTemplate.exchange(url, HttpMethod.GET, httpEntity, String.class);
        //获取响应码
        int statusCode= responseEntity.getStatusCodeValue();
        if(statusCode!=200){
            log.error("时间:{}，采集数据出错，响应码状态为：{}", DateTime.now().toString(DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss")),statusCode);
            return;
        }
        //获取js响应数据
        String resData = responseEntity.getBody();
        log.info("时间:{},采集数据成功，数据为:{}",DateTime.now().toString(DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss")),resData);
        /**
         * 结果：
         * 时间:2024-09-06 20:05:04,采集数据成功，数据为:
         * var hq_str_sh000001="上证指数,2791.7645,2788.3141,2765.8066,2804.0932,2765.6394,0,0,253645375,228570135027,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2024-09-06,15:30:39,00,";
         * var hq_str_sz399001="深证成指,8252.006,8249.659,8130.770,8260.690,8128.767,0.000,0.000,36378777179,314062276972.566,0,0.000,0,0.000,0,0.000,0,0.000,0,0.000,0,0.000,0,0.000,0,0.000,0,0.000,0,0.000,2024-09-06,15:00:03,00";
         */

        //2.TODO:使用正则解析JS数据
        //定义正则表达式
        String regex="hq_str_(.+)=\"(.+)\";";
        //表达式编译
        Pattern pattern = Pattern.compile(regex);
        //匹配字符串
        Matcher matcher = pattern.matcher(resData);
        List<StockMarketIndexInfo>entities=new ArrayList<>();
        while (matcher.find()){//有多行数据（以 ; 结尾），要使用while循环
            //如果使用的是group(0),则获取匹配的全部数据
            //获取大盘的编码
            String marketCode = matcher.group(1);//获取第一个括号匹配的数据
            //获取大盘的其他信息
            String otherInfo=matcher.group(2);//获取第二个括号匹配的数据
            //分割其他信息成String数据
            String[] splitArr = otherInfo.split(",");
            //大盘名称
            String marketName=splitArr[0];
            //获取当前大盘的开盘点数
            BigDecimal openPoint=new BigDecimal(splitArr[1]);//直接把String类型的数据变成BigDecimal数据类型
            //前收盘点
            BigDecimal preClosePoint=new BigDecimal(splitArr[2]);
            //获取大盘的当前点数
            BigDecimal curPoint=new BigDecimal(splitArr[3]);
            //获取大盘最高点
            BigDecimal maxPoint=new BigDecimal(splitArr[4]);
            //获取大盘的最低点
            BigDecimal minPoint=new BigDecimal(splitArr[5]);
            //获取成交量
            Long tradeAmt=Long.valueOf(splitArr[8]);
            //获取成交金额
            BigDecimal tradeVol=new BigDecimal(splitArr[9]);
            //获取交易时间
            DateTime dateTime = DateTime.parse(splitArr[30] + " " + splitArr[31], DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss"));
            Date date = dateTime.withSecondOfMinute(0).withMillisOfSecond(0).toDate();

            //TODO:3.把获取的数据封装成entity对象，用来插入数据库
            StockMarketIndexInfo entity = StockMarketIndexInfo.builder()
                    .id(idWorker.nextId())//使用雪花算法生成一个Long类型的id
                    .marketCode(marketCode)
                    .marketName(marketName)
                    .openPoint(openPoint)
                    .preClosePoint(preClosePoint)
                    .curPoint(curPoint)
                    .maxPoint(maxPoint)
                    .minPoint(minPoint)
                    .tradeAmount(tradeAmt)
                    .tradeVolume(tradeVol)
                    .curTime(date).build();
            entities.add(entity);
        }
        log.info("当前时间:{},获取的数据为:{}",DateTime.now().toString(DateTimeFormat.forPattern("yyyy-MM-ss HH:mm:ss")),entities);
        /**
         * 当前时间:2024-09-14 20:52:14,获取的数据为:
         * [
         * StockMarketIndexInfo(id=1832039060873678848, marketCode=sh000001, marketName=上证指数, preClosePoint=2788.3141, openPoint=2791.7645, curPoint=2765.8066, minPoint=2765.6394, maxPoint=2804.0932, tradeAmount=253645375, tradeVolume=228570135027, curTime=Fri Sep 06 15:30:00 CST 2024),
         * StockMarketIndexInfo(id=1832039103492001792, marketCode=sz399001, marketName=深证成指, preClosePoint=8249.659, openPoint=8252.006, curPoint=8130.770, minPoint=8128.767, maxPoint=8260.690, tradeAmount=36378777179, tradeVolume=314062276972.566, curTime=Fri Sep 06 15:00:00 CST 2024)
         * ]
         */
        //TODO:4.使用mybatis把entities集合批量入库
        int count=stockMarketIndexInfoMapper.insertBatch(entities);
        if(count>0){
            log.info("当前时间:{},{}数据插入成功",DateTime.now().toString(DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss")),entities);
        }else{
            log.error("当前时间:{},{}数据插入失败",DateTime.now().toString(DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss")),entities);
        }
    }
}

 mapper

    /**
     * 批量插入数据到数据库
     * @return
     */
    int insertBatch(@Param("entities") List<StockMarketIndexInfo>entities);

mapperXml文件

<insert id="insertBatch">
        insert into stock_market_index_info
        ( id,market_code,market_name
        ,pre_close_point,open_point,cur_point
        ,min_point,max_point,trade_amount
        ,trade_volume,cur_time)
        values
            <foreach collection="entities" item="entity" separator=",">
                (#{entity.id,jdbcType=BIGINT},#{entity.marketCode,jdbcType=CHAR},#{entity.marketName,jdbcType=VARCHAR}
                ,#{entity.preClosePoint,jdbcType=DECIMAL},#{entity.openPoint,jdbcType=DECIMAL},#{entity.curPoint,jdbcType=DECIMAL}
                ,#{entity.minPoint,jdbcType=DECIMAL},#{entity.maxPoint,jdbcType=DECIMAL},#{entity.tradeAmount,jdbcType=BIGINT}
                ,#{entity.tradeVolume,jdbcType=DECIMAL},#{entity.curTime,jdbcType=TIMESTAMP})
            </foreach>
    </insert>

第六步：查看数据库是否插入成功

 注意：在这个表中把cur_time 和market_code设置为联合唯一键，让同一时间内的国内大盘股票不可重复，即一个国内大盘股票在同一时间的数据只能有一个