无锡工程建设招标网站电商网站入口

1. 业务场景

• 这个是一个java 性能优化的实例，优化前需要2小时以上，甚至会卡死终止，优化后只需要2分钟；
• 这里有一个定时任务，每周一需要将上周生成的数据，从几张表读出来，然后生成新的数据，然后写入指定的一张表；
• 每周会生成3万条数据；一年就是300万；如今已经是上千万了，不管是读，还是写，性能是相当的慢。
  

2. 优化前

最初的设计逻辑是：

1. 先查询主数据：List workbenchEntities = commonSicWorkbenchMapper.findTotalSicByDate(dateStrList.get(0));
2. 再根据主数据生成新数据：List itemList = getBySingleItem(workbenchEntity);
3. 再删除老数据：deleteOldDataByWorkBench(itemList.get(0));
4. 最后插入新数据：batchSave(itemList);

代码如下：

    private void generateAndSave() {List<SicWorkbenchEntity> workbenchEntities = commonSicWorkbenchMapper.findTotalSicByDate(dateStrList.get(0));generateAndSaveByOneVersion(workbenchEntities);}private void generateAndSaveByOneVersion(List<SicWorkbenchEntity> workbenchEntities) {for (SicWorkbenchEntity workbenchEntity : workbenchEntities) {List<SicProjectionPDPEntity> itemList = getBySingleItem(workbenchEntity);deleteOldDataByWorkBench(itemList.get(0));batchSave(itemList);}}private List<SicProjectionPDPEntity> getBySingleItem(SicWorkbenchEntity workbenchEntity) {List<PdpInProjectionEntity> projectionEntities = commonPdpInProjectionMapper.findByWorkBench(workbenchEntity);List<ActivationProjectionCalculationEntity> calculationEntities=commonActivationProjectionCalculationMapper.findByWorkBench(workbenchEntity);SicProjectionPdpGenerator generator = new SicProjectionPdpGenerator(projectionEntities,calculationEntities,dateStrList);return generator.getBySingleItem(workbenchEntity);}private void deleteOldDataByWorkBench(SicProjectionPDPEntity sicProjectionPDPEntity) {commonPdpInProjectionMapper.deleteOldDataByWorkBench(sicProjectionPDPEntity);}private void batchSave(List<SicProjectionPDPEntity> itemList) {if (CollectionUtils.isNotEmpty(itemList)) {commonPdpInProjectionMapper.insertBatch(itemList);}}

3. 优化后

3.1 优化1：数据分组生成和写入

见下面generateAndSave（）方法，需要选取一个数据量均匀的字段用于分组，这里选用 subgeo字段。

3.2 优化2：数据分批次insert

这里采用int numberBatch = 1000;防止一次性插入数据量太大，卡死数据库；

3.3 优化3：insert 采用整体session提交

引入:

@Resourceprivate SqlSessionTemplate sqlSessionTemplate;SqlSession session = sqlSessionTemplate.getSqlSessionFactory().openSession(ExecutorType.BATCH, false);)

这里采用一组数据一次 session.commit()，减少了数据库连接的创建，大大提高了数据插入性能；

代码如下：

    private void generateAndSave() {List<String> subgeos = getWorkbenchSubgeos();subgeos.forEach(subgeo -> {generateAndSaveActivationBySubgeo(subgeo);});}private Map<String, Map<String, Double>> getActivationProjectionCalculationKeyBucketValueMap(String subgeo) {List<ActivationProjectionCalculationEntity> activationEntities = commonActivationProjectionCalculationMapper.getByVersionAndSubgeo(version, subgeo);Map<String, List<ActivationProjectionCalculationEntity>> dataMap = activationEntities.stream().collect(Collectors.groupingBy(ActivationProjectionCalculationEntity::getPdpUnionKey));Map<String, Map<String, Double>> result = new HashMap<>(dataMap.size());for (Map.Entry<String, List<ActivationProjectionCalculationEntity>> entry : dataMap.entrySet()) {String key = entry.getKey();Map<String, Double> value = getCollect(entry.getValue());result.put(key, value);}return result;}private Map<String, Double> getCollect(List<ActivationProjectionCalculationEntity> datas) {List<ComplexVO> complexVOTaskList = datas.stream().flatMap(o -> o.getComplexVOList().stream()).collect(Collectors.toList());List<ComplexVO> complexVOList = new ArrayList<>();for (ComplexVO complexVO : complexVOTaskList) {complexVOList.add(complexVO);}return complexVOList.stream().collect(Collectors.groupingBy(ComplexVO::getKeyName, Collectors.summingDouble(ComplexVO::getValue)));}private Map<String, Map<String, Double>> getPdpInProjectionKeyBucketValueMap(String subgeo) {List<PdpInProjectionEntity> pdpEntities = commonPdpInProjectionMapper.findByVersionAndSubgeo(pdpLastVersion, subgeo);Map<String, List<PdpInProjectionEntity>> dataMap = pdpEntities.stream().collect(Collectors.groupingBy(PdpInProjectionEntity::getPdpUnionKey));Map<String, Map<String, Double>> result = new HashMap<>(dataMap.size());for (Map.Entry<String, List<PdpInProjectionEntity>> entry : dataMap.entrySet()) {String key = entry.getKey();Map<String, Double> value = entry.getValue().stream().collect(Collectors.groupingBy(PdpInProjectionEntity::getBucket, Collectors.summingDouble(PdpInProjectionEntity::getQty)));result.put(key, value);}return result;}private String getLastVersion(String pdpVersion) {if (StringUtil.isNotEmpty(pdpVersion)) {return pdpVersion.replace("-", "");}return commonPdpInProjectionMapper.getLastVersion();}private List<String> getWorkbenchSubgeos() {return commonSicWorkbenchMapper.getSubgeoGroup(queryDate);}private List<SicWorkbenchEntity> getByQueryDateAndSubgeo(String queryDate, String subgeo) {return commonSicWorkbenchMapper.findFullSicByDateAndSubgeo(queryDate, subgeo, pdpLastVersion, true);}private void generateAndSaveActivationBySubgeo(String subgeo) {List<SicWorkbenchEntity> sicWorkbenchEntities = getByQueryDateAndSubgeo(queryDate, subgeo);List<SicProjectionPDPEntity> batchList = generateBySicWorkBench(sicWorkbenchEntities, subgeo);batchInsert(batchList);}private void batchInsert(List<SicProjectionPDPEntity> batchList) {if (CollectionUtils.isEmpty(batchList)) {return;}try (SqlSession session = sqlSessionTemplate.getSqlSessionFactory().openSession(ExecutorType.BATCH, false);) {CommonSicProjectionPDPMapper mapper = session.getMapper(CommonSicProjectionPDPMapper.class);int number = batchList.size() / numberBatch;int totalSliceNumber = batchList.size() % numberBatch == 0 ? number : number + 1;for (int i = 0; i < totalSliceNumber; i++) {int end = numberBatch * (i + 1);if (end > batchList.size()) {end = batchList.size();}List<SicProjectionPDPEntity> currentList = batchList.subList(numberBatch * i, end);mapper.insertBatch(currentList);}session.commit();batchList.clear();}}/*** 1. 生成当前数据* 2. 生成历史数据；* 3. 生成 pdp 数据；* 4.** @param sicWorkbenchEntities* @param subgeo* @return*/private List<SicProjectionPDPEntity> generateBySicWorkBench(List<SicWorkbenchEntity> sicWorkbenchEntities, String subgeo) {Map<String, List<ComplexVO>> historyData = new HashMap<>(CommonConstant.EIGHT);// pdp 数据Map<String, Map<String, Double>> pdpInProjectionKeyBucketValueMap = getPdpInProjectionKeyBucketValueMap(subgeo);// activation 数据Map<String, Map<String, Double>> activationProjectionCalculationKeyBucketValueMap = getActivationProjectionCalculationKeyBucketValueMap(subgeo);SicProjectionPdpGeneratorBack generatorBack = new SicProjectionPdpGeneratorBack(bucketStrList, sicWorkbenchEntities, historyData, pdpInProjectionKeyBucketValueMap, activationProjectionCalculationKeyBucketValueMap);return generatorBack.generate();}