实验- 分片上传 VS 直接上传

分片上传和直接上传是两种常见的文件上传方式。分片上传将文件分成多个小块，每次上传一个小块，可以并行处理多个分片，适用于大文件上传，减少了单个请求的大小，能有效避免因网络波动或上传中断导致的失败，并支持断点续传。相比之下，直接上传是将文件作为一个整体上传，通常适用于较小的文件，简单快捷，但对于大文件来说，容易受到网络环境的影响，上传中断时需要重新上传整个文件。因此，分片上传在大文件上传中具有更高的稳定性和可靠性。

FileUploadController

package per.mjn.controller;

import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.multipart.MultipartFile;
import per.mjn.service.FileUploadService;

import java.io.File;
import java.io.IOException;
import java.util.List;

@RestController
@RequestMapping("/upload")
public class FileUploadController {

    private static final String UPLOAD_DIR = "F:/uploads/";

    @Autowired
    private FileUploadService fileUploadService;

    // 启动文件分片上传
    @PostMapping("/start")
    public ResponseEntity<String> startUpload(@RequestParam("file") MultipartFile file,
                                              @RequestParam("totalParts") int totalParts,
                                              @RequestParam("partIndex") int partIndex) {
        try {
            String fileName = file.getOriginalFilename();
            fileUploadService.saveChunk(file, partIndex);  // 存储单个分片
            return ResponseEntity.ok("Chunk " + partIndex + " uploaded successfully.");
        } catch (IOException e) {
            return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR)
                    .body("Error uploading chunk: " + e.getMessage());
        }
    }

    // 多线程并行上传所有分片
    @PostMapping("/uploadAll")
    public ResponseEntity<String> uploadAllChunks(@RequestParam("file") List<MultipartFile> files,
                                                  @RequestParam("fileName") String fileName) {
        try {
            fileUploadService.uploadChunksInParallel(files, fileName);  // 调用并行上传方法
            return ResponseEntity.ok("All chunks uploaded successfully.");
        } catch (Exception e) {
            return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR)
                    .body("Error uploading chunks: " + e.getMessage());
        }
    }

    // 合并文件分片
    @PostMapping("/merge")
    public ResponseEntity<String> mergeChunks(@RequestParam("fileName") String fileName,
                                              @RequestParam("totalParts") int totalParts) {
        try {
            System.out.println(fileName);
            System.out.println(totalParts);
            fileUploadService.mergeChunks(fileName, totalParts);
            return ResponseEntity.ok("File uploaded and merged successfully.");
        } catch (IOException e) {
            return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR)
                    .body("Error merging chunks: " + e.getMessage());
        }
    }

    // 直接上传整个文件
    @PostMapping("/file")
    public ResponseEntity<String> handleFileUpload(@RequestParam("file") MultipartFile file) {
        try {
            String fileName = file.getOriginalFilename();
            File targetFile = new File(UPLOAD_DIR + fileName);
            File dir = new File(UPLOAD_DIR);
            if (!dir.exists()) {
                dir.mkdirs();
            }
            file.transferTo(targetFile);
            return ResponseEntity.ok("File uploaded successfully: " + fileName);
        } catch (IOException e) {
            return ResponseEntity.status(500).body("Error uploading file: " + e.getMessage());
        }
    }
}

FileUploadService

package per.mjn.service;

import org.springframework.web.multipart.MultipartFile;

import java.io.IOException;
import java.util.List;

public interface FileUploadService {
    public void saveChunk(MultipartFile chunk, int partIndex) throws IOException;
    public void uploadChunksInParallel(List<MultipartFile> chunks, String fileName);
    public void mergeChunks(String fileName, int totalParts) throws IOException;
}

FileUploadServiceImpl

package per.mjn.service.impl;

import org.springframework.stereotype.Service;
import org.springframework.web.multipart.MultipartFile;
import per.mjn.service.FileUploadService;

import java.io.*;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;

@Service
public class FileUploadServiceImpl implements FileUploadService {
    private static final String UPLOAD_DIR = "F:/uploads/";
    private final ExecutorService executorService;

    public FileUploadServiceImpl() {
        // 使用一个线程池来并发处理上传
        this.executorService = Executors.newFixedThreadPool(4); // 4个线程用于并行上传
    }

    // 保存文件分片
    public void saveChunk(MultipartFile chunk, int partIndex) throws IOException {
        File dir = new File(UPLOAD_DIR);
        if (!dir.exists()) {
            dir.mkdirs();
        }
        File chunkFile = new File(UPLOAD_DIR + chunk.getOriginalFilename() + ".part" + partIndex);
        chunk.transferTo(chunkFile);
    }

    // 处理所有分片的上传
    public void uploadChunksInParallel(List<MultipartFile> chunks, String fileName) {
        List<Callable<Void>> tasks = new ArrayList<>();
        for (int i = 0; i < chunks.size(); i++) {
            final int index = i;
            final MultipartFile chunk = chunks.get(i);
            tasks.add(() -> {
                try {
                    saveChunk(chunk, index);
                    System.out.println("Uploaded chunk " + index + " of " + fileName);
                } catch (IOException e) {
                    e.printStackTrace();
                }
                return null;
            });
        }

        try {
            // 执行所有上传任务
            executorService.invokeAll(tasks);
        } catch (InterruptedException e) {
            e.printStackTrace();
        }
    }

    // 合并文件分片
    public void mergeChunks(String fileName, int totalParts) throws IOException {
        File mergedFile = new File(UPLOAD_DIR + fileName);
        try (BufferedOutputStream outputStream = new BufferedOutputStream(new FileOutputStream(mergedFile))) {
            for (int i = 0; i < totalParts; i++) {
                File chunkFile = new File(UPLOAD_DIR + fileName + ".part" + i);
                try (BufferedInputStream inputStream = new BufferedInputStream(new FileInputStream(chunkFile))) {
                    byte[] buffer = new byte[8192];
                    int bytesRead;
                    while ((bytesRead = inputStream.read(buffer)) != -1) {
                        outputStream.write(buffer, 0, bytesRead);
                    }
                }
                chunkFile.delete(); // 删除临时分片文件
            }
        }
    }
}

前端测试界面

<!DOCTYPE html>
<html lang="en">
<head>
    <meta charset="UTF-8">
    <meta name="viewport" content="width=device-width, initial-scale=1.0">
    <title>File Upload</title>
</head>
<body>
    <h1>File Upload</h1>
    <input type="file" id="fileInput">
    <button onclick="startUpload()">Upload File</button>
    <button onclick="directUpload()">Upload File Directly</button>

    <script>
        const CHUNK_SIZE = 5 * 1024 * 1024; // 5MB per chunk
        const fileInput = document.querySelector('#fileInput');
        let totalChunks;

        function startUpload() {
            const file = fileInput.files[0];
            totalChunks = Math.ceil(file.size / CHUNK_SIZE);

            let currentChunk = 0;
            let files = [];
            while (currentChunk < totalChunks) {
                const chunk = file.slice(currentChunk * CHUNK_SIZE, (currentChunk + 1) * CHUNK_SIZE);
                files.push(chunk);
                currentChunk++;
            }

            // 并行上传所有分片
            uploadAllChunks(files, file.name);
        }

        function uploadAllChunks(chunks, fileName) {
            const promises = chunks.map((chunk, index) => {
                const formData = new FormData();
                formData.append('file', chunk, fileName);
                formData.append('partIndex', index);
                formData.append('totalParts', totalChunks);
                formData.append('fileName', fileName);

                return fetch('http://172.20.10.2:8080/upload/start', {
                    method: 'POST',
                    body: formData
                }).then(response => response.text())
                  .then(data => console.log(`Chunk ${index} uploaded successfully.`))
                  .catch(error => console.error(`Error uploading chunk ${index}`, error));
            });

            // 等待所有分片上传完成
            Promise.all(promises)
                .then(() => {
                    console.log('All chunks uploaded, now merging.');
                    mergeChunks(fileName);
                })
                .catch(error => console.error('Error during uploading chunks', error));
        }

        function mergeChunks(fileName) {
            fetch(`http://172.20.10.2:8080/upload/merge?fileName=${fileName}&totalParts=${totalChunks}`, {
                method: 'POST'
            }).then(response => response.text())
              .then(data => console.log('File uploaded and merged successfully.'))
              .catch(error => console.error('Error merging chunks', error));
        }

        // 直接上传整个文件
        function directUpload() {
            const file = fileInput.files[0];
            if (!file) {
                alert('Please select a file to upload.');
                return;
            }

            const formData = new FormData();
            formData.append('file', file); // 将整个文件添加到 FormData

            fetch('http://172.20.10.2:8080/upload/file', {
                method: 'POST',
                body: formData
            })
            .then(response => response.text())
            .then(data => console.log('File uploaded directly: ', data))
            .catch(error => console.error('Error uploading file directly', error));
        }
    </script>
</body>
</html>

测试分片上传与直接上传耗时

我们上传一个310MB的文件，分片上传每个分片在前端设置为10MB，在后端开5个线程并发执行上传操作。
直接上传没有分片大小也没有开多线程，下面是两种方式的测试结果。

分片上传，耗时2.419s

直接上传，耗时4.572s