在有限的内存中计算超限数据的重复值

在有限内存的前提下，想要实现读取超限数据并完成统计功能的方案有很多。基本思路都是把大数据分片成小数据，然后对小数据进行求值，如果内存不足的情况下可能需要借助磁盘来进行分片数据的保存。这里使用滑动时间窗口来解决。

• 第一步：一次性读取所有数字
• 第二步：计算窗口大小
• 第三步：遍历每个窗口，统计频率

import java.io.*;
import java.nio.file.*;
import java.util.*;public class TopFrequentNumberFinder {private static final long MAX_NUMBER = 1_000_000_000; // 数字最大值（支持到十亿）private static final int MEMORY_LIMIT_MB = 100;       // 内存限制（MB）private static final int INT_SIZE_BYTES = 4;          // 每个 int 占用字节数private static final int HASHMAP_THRESHOLD = 5_000_000; // 稀疏窗口 HashMap 最大容量private static final int SAMPLE_SIZE = 1000;          // 抽样大小public static void main(String[] args) throws IOException {String filePath = "large_data.txt"; // 输入文件路径// 第一步：一次性读取所有数字List<Long> allNumbers = readAllNumbersFromFile(filePath);// 第二步：计算窗口大小int windowSize = calculateWindowSize(MEMORY_LIMIT_MB, MAX_NUMBER, INT_SIZE_BYTES);// 第三步：遍历每个窗口，统计频率FrequencyResult globalResult = new FrequencyResult(-1, -1);for (long windowStart = 0; windowStart < MAX_NUMBER; windowStart += windowSize) {long windowEnd = Math.min(windowStart + windowSize, MAX_NUMBER);// 判断当前窗口是否稀疏boolean isSparse = isSparseWindow(allNumbers, windowStart, windowEnd);// 初始化存储结构Map<Long, Integer> sparseMap = null;int[] denseArray = null;if (isSparse) {sparseMap = new HashMap<>();} else {denseArray = new int[(int) (windowEnd - windowStart)];}// 统计当前窗口中的数字频率FrequencyStats stats = processWindow(allNumbers, windowStart, windowEnd, sparseMap, denseArray);// 合并临时文件数据if (stats.sparseMap != null && !stats.sparseMap.isEmpty()) {mergeWithTempFile(stats.sparseMap, windowStart, windowEnd);}// 获取当前窗口最大频率项FrequencyResult currentMax = stats.getMax();// 更新全局最大值if (currentMax.frequency > globalResult.frequency) {globalResult = currentMax;}}// 输出结果System.out.println("出现次数最多的数字是: " + globalResult.number);System.out.println("出现次数: " + globalResult.frequency);}/*** 一次性读取所有数字到内存*/private static List<Long> readAllNumbersFromFile(String filePath) throws IOException {List<Long> numbers = new ArrayList<>();try (BufferedReader reader = Files.newBufferedReader(Paths.get(filePath))) {String line;while ((line = reader.readLine()) != null) {try {long number = Long.parseLong(line.trim());numbers.add(number);} catch (NumberFormatException ignored) {// 忽略非法行}}}return numbers;}/*** 动态计算窗口大小*/private static int calculateWindowSize(int memoryLimitMB, long maxNumber, int intSizeBytes) {int maxWindowSize = (memoryLimitMB * 1024 * 1024) / intSizeBytes;return (int) Math.max(1, Math.min(maxWindowSize, maxNumber));}/*** 判断窗口是否稀疏（基于抽样）*/private static boolean isSparseWindow(List<Long> allNumbers, long windowStart, long windowEnd) {int matchCount = 0;int sampleCount = 0;Random rand = new Random();while (sampleCount < SAMPLE_SIZE) {int index = rand.nextInt(allNumbers.size());long number = allNumbers.get(index);if (number >= windowStart && number < windowEnd) {matchCount++;}sampleCount++;}double density = (double) matchCount / SAMPLE_SIZE;return density < 0.01;}/*** 处理一个窗口的数据*/private static FrequencyStats processWindow(List<Long> allNumbers,long windowStart,long windowEnd,Map<Long, Integer> sparseMap,int[] denseArray) throws IOException {FrequencyStats stats = new FrequencyStats(sparseMap, denseArray);for (Long number : allNumbers) {if (number >= windowStart && number < windowEnd) {if (sparseMap != null) {sparseMap.put(number, sparseMap.getOrDefault(number, 0) + 1);if (sparseMap.size() > HASHMAP_THRESHOLD) {dumpToDisk(sparseMap, windowStart, windowEnd);sparseMap.clear();}} else {int index = (int) (number - windowStart);denseArray[index]++;}}}return stats;}/*** 将稀疏窗口数据写入磁盘*/private static void dumpToDisk(Map<Long, Integer> data, long windowStart, long windowEnd) throws IOException {String fileName = "tmp/window_" + windowStart + "_" + windowEnd + ".tmp";Path path = Paths.get(fileName);if (!Files.exists(path)) {Files.createDirectories(path.getParent());Files.createFile(path);}try (BufferedWriter writer = Files.newBufferedWriter(path, StandardOpenOption.APPEND)) {for (Map.Entry<Long, Integer> entry : data.entrySet()) {writer.write(entry.getKey() + "," + entry.getValue() + "\n");}}}/*** 合并临时文件数据*/private static void mergeWithTempFile(Map<Long, Integer> sparseMap, long windowStart, long windowEnd) throws IOException {String fileName = "tmp/window_" + windowStart + "_" + windowEnd + ".tmp";Path path = Paths.get(fileName);if (!Files.exists(path)) return;try (BufferedReader reader = Files.newBufferedReader(path)) {String line;while ((line = reader.readLine()) != null) {String[] parts = line.split(",");long number = Long.parseLong(parts[0]);int freq = Integer.parseInt(parts[1]);sparseMap.put(number, sparseMap.getOrDefault(number, 0) + freq);}}Files.delete(path); // 删除临时文件}/*** 频率统计对象*/static class FrequencyStats {Map<Long, Integer> sparseMap;int[] denseArray;public FrequencyStats(Map<Long, Integer> sparseMap, int[] denseArray) {this.sparseMap = sparseMap;this.denseArray = denseArray;}public FrequencyResult getMax() {FrequencyResult result = new FrequencyResult(-1, -1);if (sparseMap != null) {for (Map.Entry<Long, Integer> entry : sparseMap.entrySet()) {if (entry.getValue() > result.frequency) {result = new FrequencyResult(entry.getKey(), entry.getValue());}}}if (denseArray != null) {for (int i = 0; i < denseArray.length; i++) {if (denseArray[i] > result.frequency) {result = new FrequencyResult(i + result.number, denseArray[i]);}}}return result;}}/*** 结果封装类*/static class FrequencyResult {long number;int frequency;public FrequencyResult(long number, int frequency) {this.number = number;this.frequency = frequency;}}
}