大文件分片上传-CSDN博客

本文链接：https://blog.csdn.net/darkendon/article/details/146255088

实现思路

如标题所说，大文件分片上传，大文件上传可以使用分片的解决办法。

具体操作如下：

1.文件分片

2.创建分片上传任务

3.分片上传

4.合并分片

前端代码示例如下

<!DOCTYPE html>
<html lang="en">

<head>
    <meta charset="UTF-8">
    <meta name="viewport" content="width=device-width, initial-scale=1.0">
    <title>Document</title>

    <style>
        .container {
            width: 300px;
            margin: 100px auto;
        }
    </style>
</head>

<body>
    <div class="container">
        <input type="file" id="fileInput">
        <button id="uploadButton">Upload</button>
    </div>
</body>
<script>

    const uploadButton = document.getElementById('uploadButton').addEventListener('click', async () => {
        const fileInput = document.getElementById('fileInput');
        const file = fileInput.files[0];
        if (!file) {
            alert('Please select a file first.');
            return;
        }
        // 分割文件
        const chunkSize = 1024 * 1024 * 10; // 10MB
        const chunks = splitFile(file, chunkSize);
        console.log('chunks', chunks)

        // 预处理文件
        let preHandleResponse = await fetch("/largeFile/preHandle", {
            method: 'POST',
            headers: {
                'Content-Type': 'application/json'
            },
            body: JSON.stringify({
                fileName: file.name,
                chunkSize: chunks.length
            })
        }
        )
        //
        if (!preHandleResponse.ok) {
            alert('文件预处理失败');
            return;
        }
        let preHandleResult = await preHandleResponse.json()

        const doChunks = chunks.map( (chunk, i) => {
            const formData = new FormData();
            formData.append('file', chunks[i]);
            formData.append('taskId', preHandleResult.id);
            formData.append('taskTag', preHandleResult.taskTag);
            formData.append('chunkNum', i);

            // 发送分片到服务器
            return fetch('/largeFile/uploadChunk', {
                method: 'POST',
                body: formData
            })
        })
        Promise.all(doChunks)
            .then(async () => {
                //发送合并请求
                let mergeResult = await fetch("/largeFile/merge", {
                    method: 'POST',
                    headers: {
                        'Content-Type': 'application/json'
                    },
                    body: JSON.stringify({
                        taskId: preHandleResult.id,
                        taskTag: preHandleResult.taskTag
                    })
                }
                )
                if (mergeResult.ok) {
                    alert('合并成功')
                    return;
                }
            })
    })

    //分割文件
    function splitFile(file, chunkSize) {
        const chunks = [];
        let start = 0;
        while (start < file.size) {
            const end = Math.min(file.size, start + chunkSize);
            const chunk = file.slice(start, end);
            chunks.push(chunk);
            start = end;
        }
        return chunks;
    }

</script>

</html>

上面代码提供了一个文件上传的按钮，具体操作过程如下

1.在上传前将文件按10m每个切割，切割成多个

2.上传之前调用预上传接口(/largeFile/preHandle)，获取本次文件处理的taskid(taskId)，以上ID用以标记同一个大文件的不同小文件，和追踪整个上传任务

3.并行上传各个文件分片，调用上传文件接口(/largeFile/uploadChunk)

4.分片文件全部上传完成之后，调用合并分片接口(/largeFile/merge)，完成操作

预处理接口

//DTO
@Data
public class LargeFilePreHandleDto {

    private Integer chunkSize;

    private String fileName;

    public LargeFileMergeTask toSave(){
      return  LargeFileMergeTask.builder()
                .taskTag(UUID.randomUUID().toString())
                .chunkSize(this.chunkSize)
              .fileName(this.fileName)
                .createTime(LocalDateTime.now())
                .done(false)
                .build();
    }
}

//Controller
  @PostMapping("/preHandle")
    public LargeFileMergeTask preHandle(@RequestBody LargeFilePreHandleDto preHandleDto) {
        if (preHandleDto.getChunkSize() == null || preHandleDto.getChunkSize() <= 0) {
            throw new RuntimeException("参数错误");
        }

        return largeFileHandleService.preHandle(preHandleDto);
    }

//Service
    @Transactional
    public LargeFileMergeTask preHandle(LargeFilePreHandleDto preHandleDto) {

        LargeFileMergeTask toSave = preHandleDto.toSave();
        largeFileMergeTaskService.save(toSave);


        return toSave;
    }

上传分片接口

//DTO
@Data
public class LargeFileUploadChunkDto {

    private Long taskId;

    private String taskTag;

    private int chunkNum;

    public LargeFileMergeSubFile toSave(){
        return LargeFileMergeSubFile
                .builder()
                .taskId(taskId)
                .taskTag(taskTag)
                .chunkNum(chunkNum)
                .createTime(LocalDateTime.now())
                .build();
    }
}


//Controller
    @PostMapping("/uploadChunk")
    public LargeFileMergeSubFile uploadChunk(LargeFileUploadChunkDto dto, @RequestPart("file") MultipartFile file) throws IOException {

        if(dto.getTaskId()==null|| dto.getTaskTag()==null|| StringUtils.isEmpty(dto.getTaskTag())|| dto.getChunkNum()<0){
            throw new RuntimeException("参数错误");
        }

        LargeFileMergeSubFile largeFileMergeSubFile = largeFileHandleService.uploadChunk(dto, file);
        return largeFileMergeSubFile;
    }

//Service
@Transactional
    public LargeFileMergeSubFile uploadChunk(LargeFileUploadChunkDto dto, MultipartFile file) throws IOException {
        InputStream inputStream = file.getInputStream();

        //TODO 保存磁盘，根据实际存储方式替换
        String filePath = dto.getTaskId() + dto.getTaskTag() + dto.getChunkNum();

        File tempFile = new File(fileConfig.getTempPath()+File.separator+filePath);
        OutputStream os = new FileOutputStream(tempFile);

        byte[] buffer = new byte[1024*8];
        int len = 0;
        while ((len = inputStream.read(buffer)) != -1) {
            os.write(buffer, 0, len);
        }
        os.close();
        //
        LargeFileMergeSubFile save = dto.toSave();
        save.setFilePath(filePath);
        largeFileMergeSubFileService.save(save);
        return save;
    }

合并分片逻辑

//DTO
@Data
public class LargeFileMergeDto {

    private Long taskId;

    private String taskTag;
}

//Controller
    @PostMapping("/merge")
    public String merge(@RequestBody LargeFileMergeDto largeFileMergeDto) throws IOException {

        if(largeFileMergeDto.getTaskId()==null|| StringUtils.isEmpty(largeFileMergeDto.getTaskTag())){
            throw new RuntimeException("参数错误");
        }

        largeFileHandleService.merge(largeFileMergeDto);
        return "merge success";
    }


//Service
@Transactional
    public void merge(LargeFileMergeDto dto) throws IOException {
        // TODO merge可能耗时较长，需要考虑和前端进行异步 交互

        LargeFileMergeTask task = largeFileMergeTaskService.checkExist(dto.getTaskId(), dto.getTaskTag());

        List<LargeFileMergeSubFile> subFiles = largeFileMergeSubFileService.listByTaskId(dto.getTaskId());
        logger.info("task chunk size={},subFilesSize={}",task.getChunkSize(),subFiles.size());
        if(task.getChunkSize()!=subFiles.size()){
            throw new RuntimeException("文件分片数量不匹配");
        }
        //TODO 从磁盘读取，根据实际存储方式替换

        //此次为内存处理，文件极大容易导致内存溢出，可以考虑利用磁盘处理
        ByteArrayOutputStream os=new ByteArrayOutputStream();
        for (LargeFileMergeSubFile subFile : subFiles) {

            File tempFile = new File(fileConfig.getTempPath()+File.separator+subFile.getFilePath());

            try (InputStream inputStream = new FileInputStream(tempFile)) {
                byte[] buffer = new byte[1024*8];
                int len = 0;
                while ((len = inputStream.read(buffer)) != -1) {
                    os.write(buffer, 0, len);

                }
            } catch (IOException e) {
                throw new RuntimeException(e);
            }
        }
        //
        String filePath =fileConfig.getUploadPath()+File.separator+ UUID.randomUUID().toString()+task.getFileName();
        OutputStream outputStream = new FileOutputStream(filePath);
        os.writeTo(outputStream);
        outputStream.close();

        FileInfo fileInfo = FileInfo
                .builder()
                .fileName(task.getFileName())
                .filePath(filePath)
                .createTime(LocalDateTime.now())
                .build();
        fileService.save(fileInfo);

        largeFileMergeTaskService.done(task.getId());
        //TODO 可考虑定时清理
        largeFileMergeSubFileService.deleteByTaskId(task.getId());
        logger.info("merge success");
    }


    //TODO 定期处理无效任务记录。
}