mysql 分表实战

news2024/12/28 21:06:04

本文主要介绍基于range分区的相关

1、业务需求，每日160w数据，每月2000w;解决大表数据读写性能问题。

2、数据库mysql 8.0.34，默认innerDB;mysql自带的逻辑分表

3、分表的目的:解决大表性能差，小表缩小查询单位的特点(其实优化的精髓就是缩小范围)

4、创建分区表-range

4.1 创建表及组合主键

CREATE TABLE `dept_test_t` (
  `dept_test_id` bigint unsigned NOT NULL COMMENT '产品二维码ID',
  `partition_field` int NOT NULL COMMENT '分区字段：yyyyMMdd',
  `created_time` datetime NOT NULL COMMENT '创建时间',
  `param` varchar(100) COLLATE utf8mb4_bin DEFAULT NULL COMMENT '参数',
  PRIMARY KEY (`dept_test_id`,`partition_field`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin COMMENT='部门测试表'

4.2 手动创建表分区

ALTER TABLE dept_test_t PARTITION BY RANGE (partition_field) (
	PARTITION p20240222 VALUES LESS THAN (20240223),
	PARTITION p20240223 VALUES LESS THAN (20240224),
	PARTITION p20240224 VALUES LESS THAN (20240225),
	PARTITION p20240225 VALUES LESS THAN (20240226),
	PARTITION p20240226 VALUES LESS THAN (20240227),
	PARTITION p20240227 VALUES LESS THAN (20240228)	
);

查看表分区

SELECT PARTITION_NAME,PARTITION_METHOD,PARTITION_EXPRESSION,PARTITION_DESCRIPTION,
TABLE_ROWS,SUBPARTITION_NAME,SUBPARTITION_METHOD
FROM information_schema.PARTITIONS WHERE TABLE_SCHEMA=SCHEMA() AND TABLE_NAME='dept_test_t';

4.3、插入数据，再观察表分区

INSERT INTO `dept_test_t`(`dept_test_id`, `partition_field`, `created_time`, `param`) VALUES (1, 20240223, SYSDATE(), '6$1155ea$p1it$20240223');
INSERT INTO `dept_test_t`(`dept_test_id`, `partition_field`, `created_time`, `param`) VALUES (11, 20240223, SYSDATE(), '6$1155ea$p1it$20240223');
INSERT INTO `dept_test_t`(`dept_test_id`, `partition_field`, `created_time`, `param`) VALUES (12, 20240223, SYSDATE(), '6$1155ea$p1it$20240223');
INSERT INTO `dept_test_t`(`dept_test_id`, `partition_field`, `created_time`, `param`) VALUES (13, 20240224, SYSDATE(), '6$1155ea$p1it$20240224');
INSERT INTO `dept_test_t`(`dept_test_id`, `partition_field`, `created_time`, `param`) VALUES (14, 20240224, SYSDATE(), '6$1155ea$p1it$20240224');
INSERT INTO `dept_test_t`(`dept_test_id`, `partition_field`, `created_time`, `param`) VALUES (15, 20240225, SYSDATE(), '6$1155ea$p1it$20240225');
INSERT INTO `dept_test_t`(`dept_test_id`, `partition_field`, `created_time`, `param`) VALUES (16, 20240225, SYSDATE(), '6$1155ea$p1it$20240225');
INSERT INTO `dept_test_t`(`dept_test_id`, `partition_field`, `created_time`, `param`) VALUES (17, 20240226, SYSDATE(), '6$1155ea$p1it$20240226');
INSERT INTO `dept_test_t`(`dept_test_id`, `partition_field`, `created_time`, `param`) VALUES (18, 20240227, SYSDATE(), '6$1155ea$p1it$20240227');
INSERT INTO `dept_test_t`(`dept_test_id`, `partition_field`, `created_time`, `param`) VALUES (19, 20240226, SYSDATE(), '6$1155ea$p1it$20240226');
INSERT INTO `dept_test_t`(`dept_test_id`, `partition_field`, `created_time`, `param`) VALUES (20, 20240228, SYSDATE(), '6$1155ea$p1it$20240228');

SELECT PARTITION_NAME,PARTITION_METHOD,PARTITION_EXPRESSION,PARTITION_DESCRIPTION,
TABLE_ROWS,SUBPARTITION_NAME,SUBPARTITION_METHOD
FROM information_schema.PARTITIONS WHERE TABLE_SCHEMA=SCHEMA() AND TABLE_NAME='dept_test_t';

注意：数据只能插入到已存在的分区，否则报错，存入失败

INSERT INTO `dept_test_t`(`dept_test_id`, `partition_field`, `created_time`, `param`) VALUES (20, 20240228, SYSDATE(), '6$1155ea$p1it$20240228')
> 1526 - Table has no partition for value 20240228
> 时间: 0.001s

4.4、查询效率验证

EXPLAIN select * from dept_test_t where partition_field = 20240224 and param='6$1155ea$p1it$20240224';

添加索引：

ALTER TABLE `dept_test_t` ADD INDEX `dept_n1`(`param`) USING BTREE;

走了具体的分区：p20240224 ，也走了索引：dept_n1

5、创建事件-定时创建分区

 -- 事件跟踪日志
 CREATE TABLE IF NOT EXISTS partition_event_log (
    id INT AUTO_INCREMENT PRIMARY KEY,
    event_name VARCHAR(100) NOT NULL,
    remark VARCHAR(200) NOT NULL,
    created_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP
);

drop EVENT `auto_add_dept_partition`;

DELIMITER ;;
CREATE EVENT IF NOT EXISTS `auto_add_dept_partition`
ON SCHEDULE EVERY 1 DAY STARTS '2023-02-22 15:04:00'
DO
BEGIN
    DECLARE cur_date DATE;
    SET cur_date = CURDATE();
    -- 根据当前日期动态生成分区名和边界值
    SET @partition_name = CONCAT('p', LPAD(DATE_FORMAT(cur_date, '%Y%m%d'), 8, '0'));
    SET @boundary_value = DATE_FORMAT(DATE_ADD(cur_date, INTERVAL 1 DAY), '%Y%m%d');
     -- 记录成功添加新分区的日志
    INSERT INTO partition_event_log (event_name, remark) VALUES ('auto_add_partition', CONCAT('Preparing to add partition for date: ', current_date));
    -- 添加新的分区
    SET @sql = CONCAT('ALTER TABLE dept_test_t ADD PARTITION (PARTITION ', @partition_name, ' VALUES LESS THAN (', @boundary_value, '))');
  
		-- 记录成功添加新分区的日志
    INSERT INTO partition_event_log (event_name, remark) VALUES ('auto_add_dept_partition', @sql);
		
		PREPARE stmt FROM @sql;
    EXECUTE stmt;
    DEALLOCATE PREPARE stmt;		
		 
END;;
DELIMITER ;

查看event执行情况

SELECT EVENT_NAME,LAST_EXECUTED, STATUS FROM information_schema.EVENTS;