赞
踩
1、前端以分片的形式计算出整个文件的 md5 值以及文件大小 size;
2、使用 md5、 size 去请求后台判断文件是否已经存在;
3、如果根据 md5 查询有数据,且数据大小与 size 一致,则文件已存在,此时跳到步骤 10
4、如果根据 md5 查询有数据,且数据大小与 size 不一致,则文件上传了一部分,此时返回主键 id 以及已经上传的文件分片索引,并跳到步骤 6
5、如果根据 md5 查询无数据,则未上传过此文件。此时向数据库中插入一条数据,仅保存 name、md5,并返回主键 id 作为分片文件的父 id;
6、前端将文件分片,遍历这些分片,进行文件分片上传。分片文件上传的同时,根据步骤 4 返回的文件分片索引,用以判断分片是否已经上传,若已上传则跳过,否则进行上传分片文件;
7、若上传失败,则保存失败的文件分片索引,最后进行重试;
8、文件分片上传全部结束,通知服务器进行合并;
9、合并结束,删除保存的分片临时文件以及数据库分片数据
10、结束!
mysql> show create table file; +-------+------------------------------------------------------- | Table | Create Table +-------+------------------------------------------------------- | file | CREATE TABLE `file` ( `id` int(11) NOT NULL AUTO_INCREMENT, `patch_index` int(11) DEFAULT NULL, `parent` int(11) DEFAULT NULL, `name` varchar(64) NOT NULL, `path` varchar(255) DEFAULT NULL, `md5` varchar(255) NOT NULL, `size` bigint(11) NOT NULL, `create_time` datetime DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY (`id`) ) ENGINE=InnoDB AUTO_INCREMENT=1179 DEFAULT CHARSET=utf8 | +-------+------------------------------------------------------- 1 row in set (0.00 sec)
其中 patch_index 为分片上传顺序,合并文件时需要以此数据为准,否则文件可能打不开(我刚开始就想着直接用自增 id 作为合并文件的顺序。但是我上传了一个 1.37 GB 视频后,文件无法播放,因为在并发下,自增 id 的顺序和分片文件的顺序已经不一定一致了)
1、html 代码
<!DOCTYPE html> <html lang="en"> <head> <meta charset="UTF-8"> <title>分片上传</title> <script type="text/javascript" src="http://cdn.bootcss.com/jquery/1.12.4/jquery.min.js"></script> <script type="text/javascript" src="/js/spark-md5.min.js"></script> <script type="text/javascript" src="/js/patchUpload.js"></script> </head> <body> <input id="file" type="file"/> <br/><br/> <button id="upload">分片上传</button> <p id="process">0<span>%</span></p> <button id="try" style="display: none">重试</button> </body> </html>
2、js 代码
var patchUpload = { /** * 分片上传成功索引 */ succeed: [], /** * 分片上传失败索引 */ failed: [], /** * 失败重试次数 */ try: 3, /** * 分片大小,这里是 5MB */ shardSize: 5 * 1024 * 1024, /** * 初始化 */ init: function () { this.setEvent(); }, /** * 设置页面事件监听 */ setEvent: function () { var me = this; $("#upload").click(function (e) { var files = $("#file")[0].files; if(files.length < 1) { alert("请选择文件!"); return; } me.succeed = []; me.failed = []; me.try = 3; me.loadProcess(0); me.md5checkUpload(files[0]); }); $("#try").click(function (e) { var files = $("#file")[0].files; me.try = 3; me.md5checkUpload(files[0]); }); }, /** * 检查文件是否已存在 * @param file * @param md5 */ checkUpload: function (file, md5) { var me = this; $.ajax({ url: "/file/exists", type: "get", data: {md5: md5, size: file.size}, dataType: "json", success: function(data) { if (data.status === 1) { me.loadProcess(100); alert("急速秒传!"); return ; } if(data.id && data.status === 0) { me.succeed = data.patchIndex; me.upload(data.id, file); return ; } me.upload(me.prepareUpload(md5, file), file); }, error: function(XMLHttpRequest, textStatus, errorThrown) { alert("服务器错误!"); } }); }, /** * 文件不存在时,插入文件的基本信息,为上传文件分片做准备 * @param md5 * @param file * @returns {*} */ prepareUpload: function (md5, file) { var id; $.ajax({ url: "/file/new", type: "post", async: false, data: JSON.stringify({name: file.name, md5: md5, size: 0}), contentType: "application/json;charset=utf-8", dataType: "json", success: function(data) { if(data && data.id) { id = data.id; return; } alert("上传文件失败!"); }, error: function(XMLHttpRequest, textStatus, errorThrown) { alert("服务器错误!"); } }); return id; }, /** * 上传文件 * @param id * @param file */ upload: function (id, file) { var me = this; if(!id) return; var shardCount = Math.ceil(file.size / this.shardSize); for (var i = 0; i < shardCount; i++) { if(me.succeed.length !== 0 && me.succeed.indexOf(i) > -1 && me.failed.indexOf(i) === -1) { continue; } this.uploadPatch(id, file, i, shardCount); } }, /** * 上传分片文件 * @param parent * @param file * @param index * @param shardCount */ uploadPatch: function (parent, file, index, shardCount) { var me = this; var start = index * this.shardSize; var end = Math.min(file.size, start + this.shardSize); var patch = file.slice(start, end); var spark = new SparkMD5(); var reader = new FileReader(); reader.readAsBinaryString(patch); $(reader).load(function (e) { spark.appendBinary(e.target.result); var md5 = spark.end(); var form = new FormData(); form.append("index", index); form.append("parent", parent); form.append("md5", md5); form.append("size", patch.size); form.append("patch", patch); form.append("name", file.name + "-patch-" + index); $.ajax({ url: "/file/patch/upload", type: "post", data: form, processData: false, contentType: false, dataType: "json", success: function(data) { if(!data || !data.ok) { me.failed.push(index); console.log("上传分片" + index + "失败!"); return ; } me.succeed.push(index); console.log("上传分片" + index + "成功!"); me.loadProcess(((me.succeed.length - 1) * me.shardSize + patch.size) / file.size * 100); me.mergePatch(parent, file, shardCount); }, error: function(XMLHttpRequest, textStatus, errorThrown) { me.failed.push(index); console.log("服务器错误,上传分片" + index + "失败!"); me.tryAgain(parent, file, shardCount); } }); }); }, /** * 请求合并文件 * @param parent * @param file * @param shardCount */ mergePatch: function (parent, file, shardCount) { var me = this; if(me.succeed.length + me.failed.length !== shardCount) return; if(me.failed.length !== 0) { me.tryAgain(parent, file, shardCount); return ; } $.ajax({ url: "/file/patch/merge", type: "post", data: {parent: parent, size: file.size}, dataType: "json", success: function(data) { if (data && data.ok) { me.loadProcess(100); alert("上传文件成功!"); return ; } alert("上传文件失败!"); }, error: function(XMLHttpRequest, textStatus, errorThrown) { alert("服务器错误!"); } }); }, /** * 重试 */ tryAgain: function (parent, file, shardCount) { var me = this; if(me.succeed.length + me.failed.length !== shardCount) return; if(me.failed.length === 0) { me.mergePatch(parent, file, shardCount); return ; } if(me.try === 0) { $("#try").css("display", "block"); return ; } me.try--; console.log("重试..."); while(me.failed.length !== 0) { me.uploadPatch(parent, file, me.failed.pop(), shardCount); } }, /** * 读取进度条(见笑了) * @param process */ loadProcess: function (process) { process = Math.min(100, process); if(process === 100) { $("#try").css("display", "none"); } $("#process").html(process.toFixed(2) + '<span>%</span>'); }, /** * 获取文件的 md5 值 * @param file * @returns {*|number} */ md5checkUpload: function (file) { var me = this; var index = 0; var shardCount = Math.ceil(file.size / this.shardSize); var spark = new SparkMD5.ArrayBuffer(); var fileReader = new FileReader(); var blobSlice = File.prototype.slice || File.prototype.mozSlice || File.prototype.webkitSlice; fileReader.onload = function (e) { index++; spark.append(e.target.result); if(index < shardCount) { loadNext(); return; } me.checkUpload(file, spark.end()); }; function loadNext() { var start = index * me.shardSize; var end = Math.min(start + me.shardSize, file.size); fileReader.readAsArrayBuffer(blobSlice.call(file, start, end)); } loadNext(); } }; $(function () { patchUpload.init(); });
普通的 springboot 项目,pom.xml 代码就不贴了
1、application.yml
spring: datasource: druid: driver-class-name: com.mysql.jdbc.Driver url: jdbc:mysql://localhost:3306/test?useSSL=false username: root password: password initial-size: 5 min-idle: 5 max-active: 100 keep-alive: true min-evictable-idle-time-millis: 28740000 max-evictable-idle-time-millis: 28800000 servlet: multipart: max-file-size: 10MB max-request-size: 100MB mybatis: configuration: lazy-loading-enabled: true aggressive-lazy-loading: false cache-enabled: true map-underscore-to-camel-case: true logging: level: com.kfyty.mybatis.auto.mapper.handle: info com.kfyty.upload.mapper: debug
2、实体类(自己写的代码生成器生成的)
package com.kfyty.upload.pojo; import java.util.Date; import lombok.Data; /** * TABLE_NAME: file * TABLE_COMMENT: * * By kfyty */ @Data public class FilePojo { /** * */ private Integer id; /** * */ private Integer patchIndex; /** * */ private Integer parent; /** * */ private String name; /** * */ private String path; /** * */ private String md5; /** * */ private Long size; /** * */ private Date createTime; public FilePojo() { } public FilePojo(Integer patchIndex, Integer parent, String name, String path, String md5, Long size) { this.patchIndex = patchIndex; this.parent = parent; this.name = name; this.path = path; this.md5 = md5; this.size = size; } }
3、一个 vo(帮助理解 js)
package com.kfyty.upload.vo; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; import java.util.List; @Data @NoArgsConstructor @AllArgsConstructor public class FileExists { /** * 文件 id */ private Integer id; /** * 文件状态 * -1: 不存在 * 1: 已存在 * 0: 部分存在 */ private Integer status; /** * 已上传的文件分片索引 */ private List<Integer> patchIndex; public static FileExists nonExistent() { return new FileExists(null, -1, null); } public static FileExists exists(Integer id) { return new FileExists(id, 1, null); } public static FileExists partExistent(Integer id, List<Integer> patchIndex) { return new FileExists(id, 0, patchIndex); } }
4、Mapper 接口(使用了自己写的一个扩展包,使用一个注解就能查询,所以没有 Mapper.xml 文件)
package com.kfyty.upload.mapper; import com.kfyty.mybatis.auto.mapper.BaseMapper; import com.kfyty.mybatis.auto.mapper.annotation.AutoMapper; import com.kfyty.mybatis.auto.mapper.annotation.SelectKey; import com.kfyty.upload.pojo.FilePojo; import org.apache.ibatis.annotations.Mapper; import org.apache.ibatis.annotations.Param; import java.util.List; @Mapper @SelectKey @AutoMapper(entity = FilePojo.class) public interface FileMapper extends BaseMapper<Integer, FilePojo> { @AutoMapper int updateByIdSetPathAndSize(@Param("id") Integer id, @Param("path") String path, @Param("size") Long size); @AutoMapper int deleteByParent(@Param("parent") Integer parent); @AutoMapper List<Integer> findPatchIndexByParent(@Param("parent") Integer parent); @AutoMapper FilePojo findByMd5(@Param("md5") String md5); @AutoMapper FilePojo findByParentAndMd5(@Param("parent") Integer parent, @Param("md5") String md5); @AutoMapper List<FilePojo> findByParentOrderByPatchIndexAsc(@Param("parent") Integer parent); }
5、上传文件的工具类
package com.kfyty.upload.utils; import org.springframework.web.multipart.MultipartFile; import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; import java.nio.channels.FileChannel; import java.text.SimpleDateFormat; import java.util.Date; import java.util.List; import java.util.UUID; public abstract class UploadUtil { private static final String SAVE_PATH = "D:/temp/patch/"; public static String initPath() { String date = new SimpleDateFormat("yyyy-MM-dd").format(new Date()); return SAVE_PATH + date + "/"; } public static String parseFileType(String fileName) { return !fileName.contains(".") ? "" : fileName.substring(fileName.lastIndexOf(".")); } public static String saveFile(MultipartFile source, Long size) throws IOException { if(source.getSize() != size) { throw new RuntimeException("上传字节数与接收字节数不符!"); } File file = new File(initPath()); if(!file.exists() && !file.mkdirs()) { throw new RuntimeException("创建文件夹失败!"); } String fileType = parseFileType(source.getOriginalFilename()); while(true) { String saveFileName = UUID.randomUUID() + fileType.toLowerCase(); File saveFilePath = new File(file.getPath(), saveFileName); if(saveFilePath.exists()) { continue; } source.transferTo(saveFilePath); return saveFilePath.getAbsolutePath(); } } public static String mergeFile(String fileType, List<String> fileNames) throws IOException { File file = new File(initPath()); if(!file.exists() && !file.mkdirs()) { throw new RuntimeException("创建文件夹失败!"); } File saveFilePath = null; do { String saveFileName = UUID.randomUUID() + fileType.toLowerCase(); saveFilePath = new File(file.getPath(), saveFileName); } while (saveFilePath.exists()); FileChannel out = new FileOutputStream(saveFilePath).getChannel(); for (String fileName : fileNames) { File patch = new File(fileName); FileChannel in = new FileInputStream(patch).getChannel(); in.transferTo(0, in.size(), out); in.close(); patch.delete(); } out.close(); return saveFilePath.getAbsolutePath(); } }
6、启动类和控制器
package com.kfyty.upload; import com.kfyty.upload.mapper.FileMapper; import com.kfyty.upload.pojo.FilePojo; import com.kfyty.upload.utils.UploadUtil; import com.kfyty.upload.vo.FileExists; import com.kfyty.upload.vo.Result; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.context.ConfigurableApplicationContext; import org.springframework.transaction.annotation.Transactional; import org.springframework.util.CollectionUtils; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RestController; import org.springframework.web.multipart.MultipartFile; import java.io.IOException; import java.util.List; import java.util.Optional; import java.util.stream.Collectors; @Slf4j @RestController @SpringBootApplication public class UploadApplication { @Autowired private FileMapper fileMapper; @PostMapping("/file/new") public FilePojo uploadFile(@RequestBody FilePojo file) { fileMapper.insert(file); return file; } @GetMapping("/file/exists") public FileExists fileExists(String md5, Long size) { FilePojo file = fileMapper.findByMd5(md5); if(file == null) { return FileExists.nonExistent(); } if(file.getSize().equals(size)) { return FileExists.exists(file.getId()); } return FileExists.partExistent(file.getId(), fileMapper.findPatchIndexByParent(file.getId())); } @PostMapping("/file/patch/upload") public Result filePatchExists(String name, Integer index, Integer parent, String md5, Long size, MultipartFile patch) throws IOException { FilePojo file = fileMapper.findByParentAndMd5(parent, md5); if(file == null || !file.getSize().equals(size)) { Optional.ofNullable(file).ifPresent(e -> fileMapper.deleteByPk(e.getId())); fileMapper.insert(new FilePojo(index, parent, name, UploadUtil.saveFile(patch, size), md5, size)); return Result.OK(); } return file.getSize().equals(size) ? Result.OK() : Result.FAIL(); } @Transactional @PostMapping("/file/patch/merge") public Result filePatchMerge(Integer parent, Long size) throws IOException { FilePojo fileInfo = fileMapper.findByPk(parent); List<FilePojo> patchs = fileMapper.findByParentOrderByPatchIndexAsc(parent); Long total = patchs.stream().mapToLong(FilePojo::getSize).sum(); if(fileInfo == null || CollectionUtils.isEmpty(patchs) || !total.equals(size)) { fileMapper.deleteByParent(parent); log.warn("total: {}, require size: {}, and delete file to retry !", total, size); return Result.FAIL(); } String fileType = UploadUtil.parseFileType(fileInfo.getName()); String path = UploadUtil.mergeFile(fileType, patchs.stream().map(FilePojo::getPath).collect(Collectors.toList())); fileMapper.updateByIdSetPathAndSize(parent, path, total); fileMapper.deleteByParent(parent); return Result.OK(); } public static void main(String[] args) { ConfigurableApplicationContext run = SpringApplication.run(UploadApplication.class, args); } }
7、控制器中用到的一个辅助类
package com.kfyty.upload.vo; import lombok.Data; import lombok.AllArgsConstructor; import java.util.function.Predicate; @Data @AllArgsConstructor public class Result<T> { private boolean ok; private T data; public Result() { } public Result(boolean ok) { this(ok, null); } public Result(T data) { this(true, data); } public static Result OK() { return new Result(true); } public static Result FAIL() { return new Result(false); } public static Result test(Object o, Predicate<Object> p) { return p.test(o) ? new Result<>(o) : FAIL(); } }
。。。。。。。。。。。。。。。。。。。。。。。。。。。
PS:Mapper 接口中用到的扩展包可以去这里下载安装:
https://github.com/kfyty/mybatis-auto-mapper
补充一个下载的断点续传例子:
package com.kfyty.upload.download; import lombok.Data; import org.springframework.util.StringUtils; import java.io.File; import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; /** * 描述: 解析请求头的范围参数 * * @author kfyty * @date 2021/2/20 15:50 * @email kfyty725@hotmail.com */ @Data public class DownloadRange { private long pos; private long last; private long length; public DownloadRange(long pos, long last) { this.pos = pos; this.last = last; this.length = last - pos + 1; } public static DownloadRange resolve(String range, File file) { if(StringUtils.isEmpty(range)) { return new DownloadRange(0, file.length() - 1); } if(range.contains(",")) { throw new IllegalArgumentException("暂不支持的规范:" + range); } List<String> split = Arrays.stream(range.replace("bytes=", "").split("-")).filter(s -> !StringUtils.isEmpty(s)).collect(Collectors.toList()); if(split.size() == 2) { return new DownloadRange(Long.parseLong(split.get(0)), Long.parseLong(split.get(1))); } long num = Long.parseLong(split.get(0)); return new DownloadRange(range.charAt(0) == '-' ? file.length() - num : num, file.length() - 1); } }
package com.kfyty.upload.download; import lombok.extern.slf4j.Slf4j; import org.springframework.http.MediaType; import org.springframework.util.StringUtils; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.RestController; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import java.io.File; import java.io.IOException; import java.io.OutputStream; import java.io.RandomAccessFile; import java.net.URLEncoder; /** * 描述: 下载控制器 * * @author kfyty * @date 2021/2/20 14:31 * @email kfyty725@hotmail.com */ @Slf4j @RestController public class BreakpointDownload { /** * 下载,支持断点续传 */ @GetMapping("breakpoint/download") public void breakpointDownload(String path, HttpServletRequest request, HttpServletResponse response) throws Exception { File file = new File(path); try (OutputStream os = response.getOutputStream(); RandomAccessFile raf = new RandomAccessFile(file, "r")) { /** * 设置响应头,以支持断点续传,必须 */ response.setContentType(MediaType.APPLICATION_OCTET_STREAM_VALUE); response.setHeader("Accept-Ranges", "bytes"); response.setHeader("Content-Disposition", "attachment; filename=\"" + URLEncoder.encode(file.getName(), "UTF-8") + "\""); response.setDateHeader("Last-Modified", file.lastModified()); /** * 发起断点续传时,设置为 206 */ String requestRange = request.getHeader("Range"); if (!StringUtils.isEmpty(requestRange)) { response.setStatus(HttpServletResponse.SC_PARTIAL_CONTENT); } DownloadRange range = DownloadRange.resolve(requestRange, file); raf.seek(range.getPos()); String contentRange = String.format("bytes %s-%s/%s", range.getPos(), range.getLast(), file.length()); response.setHeader("Content-Range", contentRange); response.addHeader("Content-Length", String.valueOf(range.getLength())); int read = 0; long total = 0L; byte[] bytes = new byte[1024 * 1024]; while (total < range.getLength() && read != -1) { read = raf.read(bytes, 0, (int) Math.min(bytes.length, range.getLength() - total)); os.write(bytes, 0, read); total += read; } os.flush(); } catch (IOException e) { log.error(e.getMessage(), e); } } }
Copyright © 2003-2013 www.wpsshop.cn 版权所有,并保留所有权利。