java性能优化之批量处理数据!_java批量处理大量数据

一、常见场景
场景一：对于百万级别excel数据导入，如果优化？
场景二：高并发下写数据，如何快速响应，如何提升数据处理效率，尽可能减轻数据库的压力？
场景三、大批量消费消息队列消息的时候，如何提升数据处理效率，提高消费效率减少消息挤压？

二、解决方案
1）场景一和场景三解决方案：多线程+批量保存数据库
2）场景二解决方案：异步处理+批量保存数据到数据库

三、异步处理（这里简单聊下，后面有时间再详细讲解）
1、发送数据到消息队列（比较推荐，基本不丢失，实现稍微麻烦）
2、基于guava eventbus实现异步事件（实现简单，性能比较好，重启服务有丢失的风险）
3、基于线程池实现异步处理（实现简单，并发量不大，可以用，重启服务有丢失的风险）

三、批量处理数据的几种方案对比

1、通过for循环逐条导入100w数据，例如：
for(int i=0;i<1000000;i++){
   save(entity);
}
结果分析：：耗时900s
2024-03-20 17:35:10.592 INFO [dancechar-base-service,XNIO-1 task-1,4be7f2e99e65a1,1546061704931049472] c.l.d.base.biz.student.service.StudentService.saveStuListWithOne:66- 通过for完成百万数据插入！！！总耗时：900s
2024-03-20 17:35:11.052 INFO [dancechar-base-service,XNIO-1 task-1,4be7f2e99e65a1,1546065486100299776] c.l.dancechar.base.framework.ResponseLogAdvice.beforeBodyWrite:53- 当前请求返回结果：true
2024-03-20 17:35:11.113 INFO [dancechar-base-service,XNIO-1 task-1,4be7f2e99e65a1,1546065486356152320] c.l.d.framework.common.trace.TraceWebFilter.doFilter:52- 当前请求总耗时：901772ms
 
2、通过batch循环批量导入100w数据
saveBatch(entityList);
结果分析：耗时165s，性能相比1，提供5倍以上
2024-03-20 18:19:06.704 INFO [dancechar-base-service,XNIO-1 task-2,cad1de9712a53f,1546076540821831680] c.l.d.framework.common.trace.TraceWebFilter.printAccessLog:77- 开始当前请求-/sys/student/saveStuListWithBatch,方法-POST，body参数：{}
2024-03-20 18:19:06.719 INFO [dancechar-base-service,XNIO-1 task-2,cad1de9712a53f,1546076540842803200] c.l.d.b.biz.student.controller.StudentController.saveStuListWithBatch:51- 批量插入百万数据....
2024-03-20 18:21:52.258 INFO [dancechar-base-service,XNIO-1 task-2,cad1de9712a53f,1546076540842803200] c.l.d.base.biz.student.service.StudentService.saveStuListWithBatch:77- 批量插入百万数据！！！总耗时：165s
2024-03-20 18:21:52.664 INFO [dancechar-base-service,XNIO-1 task-2,cad1de9712a53f,1546077236912717824] c.l.dancechar.base.framework.ResponseLogAdvice.beforeBodyWrite:53- 当前请求返回结果：true
2024-03-20 18:21:52.678 INFO [dancechar-base-service,XNIO-1 task-2,cad1de9712a53f,1546077236975632384] c.l.d.framework.common.trace.TraceWebFilter.doFilter:52- 当前请求总耗时：165981ms
 
3、通过多线程batch循环批量导入100w数据
threadPoolTaskExecutor.execute(()->{
   saveBatch(entityList);
})
结果分析：耗时70s，性能相比2，提升2倍以上
2024-03-20 18:26:48.625 INFO [dancechar-base-service,XNIO-1 task-3,4bca21714d53a8,1546078181767774208] c.l.d.base.biz.student.service.StudentService.saveStuListWithThreadPoolBatch:112- 通过线程池批量插入百万数据！！！总耗时：70s
2024-03-20 18:26:48.636 INFO [dancechar-base-service,XNIO-1 task-3,4bca21714d53a8,1546078478309261312] c.l.dancechar.base.framework.ResponseLogAdvice.beforeBodyWrite:53- 当前请求返回结果：true
2024-03-20 18:26:48.644 INFO [dancechar-base-service,XNIO-1 task-3,4bca21714d53a8,1546078478342815744] c.l.d.framework.common.trace.TraceWebFilter.doFilter:52- 当前请求总耗时：70716ms

四、代码演示

controller层

package com.litian.dancechar.base.biz.student.controller;
 
import com.litian.dancechar.base.biz.student.dto.StudentReqDTO;
import com.litian.dancechar.base.biz.student.dto.StudentRespDTO;
import com.litian.dancechar.base.biz.student.service.StudentService;
import com.litian.dancechar.framework.common.base.PageWrapperDTO;
import com.litian.dancechar.framework.common.base.RespResult;
import com.litian.dancechar.framework.common.util.DCBeanUtil;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
import lombok.extern.slf4j.Slf4j;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
 
import javax.annotation.Resource;
import java.util.List;
 
/**
 * 学生业务处理
 *
 * @author kyle0432
 * @date 2024/03/20 16:26
 */
@Api(tags = "学生相关api")
@RestController
@Slf4j
@RequestMapping(value = "/sys/student/")
public class StudentController {
 
    @Resource
    private StudentService studentService;
 
 
    @ApiOperation(value = "单条插入百万数据", notes = "单条插入百万数据")
    @PostMapping("saveStuListWithOne")
    public RespResult<Boolean> saveStuListWithOne() {
        log.info("单条插入百万数据....");
        return studentService.saveStuListWithOne();
    }
 
    @ApiOperation(value = "批量插入百万数据", notes = "批量插入百万数据")
    @PostMapping("saveStuListWithBatch")
    public RespResult<Boolean> saveStuListWithBatch() {
        log.info("批量插入百万数据....");
        return studentService.saveStuListWithBatch();
    }
 
    @ApiOperation(value = "通过线程池批量插入数据", notes = "通过线程池批量插入数据")
    @PostMapping("saveStuListWithThreadPoolBatch")
    public RespResult<Boolean> saveStuListWithThreadPoolBatch() {
        log.info("通过线程池批量插入百万数据....");
        return studentService.saveStuListWithThreadPoolBatch();
    }
}

service层

package com.litian.dancechar.base.biz.student.service;
 
import cn.hutool.core.util.ObjectUtil;
import cn.hutool.core.util.RandomUtil;
import cn.hutool.core.util.StrUtil;
import cn.hutool.json.JSONUtil;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import com.github.pagehelper.PageHelper;
import com.google.common.collect.Lists;
import com.litian.dancechar.base.biz.student.dao.entity.StudentDO;
import com.litian.dancechar.base.biz.student.dao.inf.StudentDao;
import com.litian.dancechar.base.biz.student.dto.StudentReqDTO;
import com.litian.dancechar.base.biz.student.dto.StudentRespDTO;
import com.litian.dancechar.base.common.constants.RedisKeyConstants;
import com.litian.dancechar.base.framework.redislistener.common.RedisChannelEnum;
import com.litian.dancechar.base.init.ThreadPoolInit;
import com.litian.dancechar.framework.cache.publish.util.MessagePublishUtil;
import com.litian.dancechar.framework.cache.redis.util.RedisHelper;
import com.litian.dancechar.framework.common.base.PageWrapperDTO;
import com.litian.dancechar.framework.common.base.RespResult;
import com.litian.dancechar.framework.common.util.DCBeanUtil;
import com.litian.dancechar.framework.common.util.PageResultUtil;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.time.StopWatch;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
 
import javax.annotation.Resource;
import java.util.List;
import java.util.concurrent.CountDownLatch;
 
/**
 * 学生服務
 *
 * @author kyle0432
 * @date 2024/03/20 19:30
 */
@Service
@Slf4j
@Transactional(rollbackFor = Exception.class)
public class StudentService extends ServiceImpl<StudentDao,StudentDO> {
 
    @Resource
    private StudentDao studentDao;
 
    @Resource
    private RedisHelper redisHelper;
 
    @Resource
    private MessagePublishUtil messagePublishUtil;
 
    /**
     * 功能：单条插入百万数据
     */
    public RespResult<Boolean> saveStuListWithOne() {
        StopWatch stopWatch = StopWatch.createStarted();
        List<StudentDO> studentList = buildStudentList(1000000);
        for(StudentDO studentDO : studentList){
            save(studentDO);
        }
        log.info("通过for完成百万数据插入！！！总耗时：{}s", stopWatch.getTime()/1000);
        return RespResult.success(true);
    }
 
    /**
     * 功能：批量插入数据
     */
    public RespResult<Boolean> saveStuListWithBatch() {
        StopWatch stopWatch = StopWatch.createStarted();
        //构建100w条数据
        List<StudentDO> studentList = buildStudentList(1000000);
        //使用mybatis-plus分批+批量方式插入【每一批1000条数据】
        this.saveBatch(studentList);
        log.info("批量插入百万数据！！！总耗时：{}s", stopWatch.getTime()/1000);
        return RespResult.success(true);
    }
 
 
    /**
     * 功能：通过线程池批量插入数据
     */
    public RespResult<Boolean> saveStuListWithThreadPoolBatch(){
        StopWatch stopWatch = StopWatch.createStarted();
        // 每次插入1000条数据
        List<StudentDO> studentList = this.buildStudentList(1000000);
        int dataSize = studentList.size();
        int step = 1000;
        int totalTasks = (dataSize % step == 0 ? dataSize/step : (dataSize/step + 1));
        final CountDownLatch countDownLatch = new CountDownLatch(totalTasks);
        for(int j = 0; j < dataSize; j=j+step){
            final int start = j;
            final int perCount = (dataSize - start) < step ? (dataSize - start) : step;
            ThreadPoolInit.getStudentThreadPoolTaskExecutor().execute(()->{
                try {
                    log.info("多线程开始: start == " + start + " , 多线程个数count" + perCount);
                    saveBatch(studentList.subList(start,perCount+start));
                    countDownLatch.countDown();
                } catch (Exception e) {
                    log.error(e.getMessage(), e);
                }
            });
        }
        try {
            countDownLatch.await();
        } catch (InterruptedException e) {
            log.error(e.getMessage(), e);
        }
        log.info("通过线程池批量插入百万数据！！！总耗时：{}s", stopWatch.getTime()/1000);
        return RespResult.success(true);
    }
 
    private List<StudentDO> buildStudentList(int dataSize){
        List<StudentDO> studentList = Lists.newArrayList();
        String noPrefix = RandomUtil.randomNumbers(3);
        for (int i=0; i<dataSize;i++){
            StudentDO studentDO = new StudentDO();
            studentDO.setNo(noPrefix+ i);
            studentList.add(studentDO);
        }
        return studentList;
    }
}

线程池初始化(不同的业务使用不同的线程池)

package com.litian.dancechar.base.init;
 
import com.litian.dancechar.framework.common.thread.CustomThreadPoolFactory;
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
import org.springframework.stereotype.Component;
 
/**
 * 线程池初始化(不同的业务使用不同的线程池)
 *
 * @author kyle0432
 * @date 2024/03/20 23:25
 */
@Component
public class ThreadPoolInit {
    /**
     * 公共的线程池
     */
    private static ThreadPoolTaskExecutor threadPoolTaskExecutor =
            CustomThreadPoolFactory.newFixedThreadPool(16);
 
    /**
     * 学生业务线程池
     */
    private static ThreadPoolTaskExecutor studentThreadPoolTaskExecutor =
            CustomThreadPoolFactory.newFixedThreadPool(8);
 
    public  static ThreadPoolTaskExecutor getThreadPoolTaskExecutor() {
        return threadPoolTaskExecutor;
    }
 
    public static ThreadPoolTaskExecutor getStudentThreadPoolTaskExecutor() {
        return studentThreadPoolTaskExecutor;
    }
}