java性能优化实例
1. 业务场景
- 这个是一个java 性能优化的实例，优化前需要2小时以上，甚至会卡死终止，优化后只需要2分钟；
- 这里有一个定时任务，每周一需要将上周生成的数据，从几张表读出来，然后生成新的数据，然后写入指定的一张表；
- 每周会生成3万条数据；一年就是300万；如今已经是上千万了，不管是读，还是写，性能是相当的慢。
2. 优化前
最初的设计逻辑是：
- 先查询主数据：List workbenchEntities = commonSicWorkbenchMapper.findTotalSicByDate(dateStrList.get(0));
- 再根据主数据生成新数据：List itemList = getBySingleItem(workbenchEntity);
- 再删除老数据：deleteOldDataByWorkBench(itemList.get(0));
- 最后插入新数据：batchSave(itemList);
代码如下：
private void generateAndSave() {
List<SicWorkbenchEntity> workbenchEntities = commonSicWorkbenchMapper.findTotalSicByDate(dateStrList.get(0));
generateAndSaveByOneVersion(workbenchEntities);
}
private void generateAndSaveByOneVersion(List<SicWorkbenchEntity> workbenchEntities) {
for (SicWorkbenchEntity workbenchEntity : workbenchEntities) {
List<SicProjectionPDPEntity> itemList = getBySingleItem(workbenchEntity);
deleteOldDataByWorkBench(itemList.get(0));
batchSave(itemList);
}
}
private List<SicProjectionPDPEntity> getBySingleItem(SicWorkbenchEntity workbenchEntity) {
List<PdpInProjectionEntity> projectionEntities = commonPdpInProjectionMapper.findByWorkBench(workbenchEntity);
List<ActivationProjectionCalculationEntity> calculationEntities=commonActivationProjectionCalculationMapper.findByWorkBench(workbenchEntity);
SicProjectionPdpGenerator generator = new SicProjectionPdpGenerator(projectionEntities,calculationEntities,dateStrList);
return generator.getBySingleItem(workbenchEntity);
}
private void deleteOldDataByWorkBench(SicProjectionPDPEntity sicProjectionPDPEntity) {
commonPdpInProjectionMapper.deleteOldDataByWorkBench(sicProjectionPDPEntity);
}
private void batchSave(List<SicProjectionPDPEntity> itemList) {
if (CollectionUtils.isNotEmpty(itemList)) {
commonPdpInProjectionMapper.insertBatch(itemList);
}
}
3. 优化后
3.1 优化1：数据分组生成和写入
见下面generateAndSave（）方法，需要选取一个数据量均匀的字段用于分组，这里选用 subgeo字段。
3.2 优化2：数据分批次insert
这里采用int numberBatch = 1000;防止一次性插入数据量太大，卡死数据库；
3.3 优化3：insert 采用整体session提交
引入:
@Resource
private SqlSessionTemplate sqlSessionTemplate;
SqlSession session = sqlSessionTemplate.getSqlSessionFactory().openSession(ExecutorType.BATCH, false);)
这里采用一组数据一次 session.commit()，减少了数据库连接的创建，大大提高了数据插入性能；
代码如下：
private void generateAndSave() {
List<String> subgeos = getWorkbenchSubgeos();
subgeos.forEach(subgeo -> {
generateAndSaveActivationBySubgeo(subgeo);
});
}
private Map<String, Map<String, Double>> getActivationProjectionCalculationKeyBucketValueMap(String subgeo) {
List<ActivationProjectionCalculationEntity> activationEntities = commonActivationProjectionCalculationMapper.getByVersionAndSubgeo(version, subgeo);
Map<String, List<ActivationProjectionCalculationEntity>> dataMap = activationEntities.stream().collect(Collectors.groupingBy(ActivationProjectionCalculationEntity::getPdpUnionKey));
Map<String, Map<String, Double>> result = new HashMap<>(dataMap.size());
for (Map.Entry<String, List<ActivationProjectionCalculationEntity>> entry : dataMap.entrySet()) {
String key = entry.getKey();
Map<String, Double> value = getCollect(entry.getValue());
result.put(key, value);
}
return result;
}
private Map<String, Double> getCollect(List<ActivationProjectionCalculationEntity> datas) {
List<ComplexVO> complexVOTaskList = datas.stream().flatMap(o -> o.getComplexVOList().stream()).collect(Collectors.toList());
List<ComplexVO> complexVOList = new ArrayList<>();
for (ComplexVO complexVO : complexVOTaskList) {
complexVOList.add(complexVO);
}
return complexVOList.stream().collect(Collectors.groupingBy(ComplexVO::getKeyName, Collectors.summingDouble(ComplexVO::getValue)));
}
private Map<String, Map<String, Double>> getPdpInProjectionKeyBucketValueMap(String subgeo) {
List<PdpInProjectionEntity> pdpEntities = commonPdpInProjectionMapper.findByVersionAndSubgeo(pdpLastVersion, subgeo);
Map<String, List<PdpInProjectionEntity>> dataMap = pdpEntities.stream().collect(Collectors.groupingBy(PdpInProjectionEntity::getPdpUnionKey));
Map<String, Map<String, Double>> result = new HashMap<>(dataMap.size());
for (Map.Entry<String, List<PdpInProjectionEntity>> entry : dataMap.entrySet()) {
String key = entry.getKey();
Map<String, Double> value = entry.getValue().stream().collect(Collectors.groupingBy(PdpInProjectionEntity::getBucket, Collectors.summingDouble(PdpInProjectionEntity::getQty)));
result.put(key, value);
}
return result;
}
private String getLastVersion(String pdpVersion) {
if (StringUtil.isNotEmpty(pdpVersion)) {
return pdpVersion.replace("-", "");
}
return commonPdpInProjectionMapper.getLastVersion();
}
private List<String> getWorkbenchSubgeos() {
return commonSicWorkbenchMapper.getSubgeoGroup(queryDate);
}
private List<SicWorkbenchEntity> getByQueryDateAndSubgeo(String queryDate, String subgeo) {
return commonSicWorkbenchMapper.findFullSicByDateAndSubgeo(queryDate, subgeo, pdpLastVersion, true);
}
private void generateAndSaveActivationBySubgeo(String subgeo) {
List<SicWorkbenchEntity> sicWorkbenchEntities = getByQueryDateAndSubgeo(queryDate, subgeo);
List<SicProjectionPDPEntity> batchList = generateBySicWorkBench(sicWorkbenchEntities, subgeo);
batchInsert(batchList);
}
private void batchInsert(List<SicProjectionPDPEntity> batchList) {
if (CollectionUtils.isEmpty(batchList)) {
return;
}
try (SqlSession session = sqlSessionTemplate.getSqlSessionFactory().openSession(ExecutorType.BATCH, false);) {
CommonSicProjectionPDPMapper mapper = session.getMapper(CommonSicProjectionPDPMapper.class);
int number = batchList.size() / numberBatch;
int totalSliceNumber = batchList.size() % numberBatch == 0 ? number : number + 1;
for (int i = 0; i < totalSliceNumber; i++) {
int end = numberBatch * (i + 1);
if (end > batchList.size()) {
end = batchList.size();
}
List<SicProjectionPDPEntity> currentList = batchList.subList(numberBatch * i, end);
mapper.insertBatch(currentList);
}
session.commit();
batchList.clear();
}
}
/**
* 1. 生成当前数据
* 2. 生成历史数据；
* 3. 生成 pdp 数据；
* 4.
*
* @param sicWorkbenchEntities
* @param subgeo
* @return
*/
private List<SicProjectionPDPEntity> generateBySicWorkBench(List<SicWorkbenchEntity> sicWorkbenchEntities, String subgeo) {
Map<String, List<ComplexVO>> historyData = new HashMap<>(CommonConstant.EIGHT);
// pdp 数据
Map<String, Map<String, Double>> pdpInProjectionKeyBucketValueMap = getPdpInProjectionKeyBucketValueMap(subgeo);
// activation 数据
Map<String, Map<String, Double>> activationProjectionCalculationKeyBucketValueMap = getActivationProjectionCalculationKeyBucketValueMap(subgeo);
SicProjectionPdpGeneratorBack generatorBack = new SicProjectionPdpGeneratorBack(bucketStrList, sicWorkbenchEntities, historyData, pdpInProjectionKeyBucketValueMap, activationProjectionCalculationKeyBucketValueMap);
return generatorBack.generate();
}