赞
踩
最近公司在做分布式的文件存储系统,但问题出现了,文件系统使用哪一款?
最后决定用fastdfs在做,但是在springboot集成的时候出现了问题,前端使用vue组件在上传时候,因为并发比较高,直接将服务器给搞死了.原本打算将文件一块块的分片上传后再合并,然后在到fastdfs中,但集群的时候,文件上传到不同的服务器中.则没有办法在一个服务其中合并.所以最后决定将文件块直接向fastdfs中写,但是问题又来了,fastdfs创建一个文件,不能创建一块空间,只能在后边进行追加.这样的话,我们在服务器端,就不能多线程向里面跑数据了.因为必须要有顺序的上传.故此,前端只能先传第一块在传第二块,这样的传递.后端使用redis将文件信息保存,左锁机制.
最后其他服务调用我们oss服务需要将本地的一个文件保存到fastdfs中,openfeign 基本不支持,需要引入一些扩展包,进行处理,模拟表单.然后在上传.
一波三折啊.最后实现了.
下面是我的代码,高手别喷哦!!!^^
OSS系统代码
这是系统目录结构
import com.github.tobato.fastdfs.FdfsClientConfig;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.EnableMBeanExport;
import org.springframework.context.annotation.Import;
import org.springframework.jmx.support.RegistrationPolicy;
@Configuration
@Import(FdfsClientConfig.class)
// Jmx重复注册bean的问题
@EnableMBeanExport(registration = RegistrationPolicy.IGNORE_EXISTING)
public class DfsConfig {
}
package com.techhero.component.oss.config; import cn.hutool.core.io.FileUtil; import com.github.tobato.fastdfs.domain.fdfs.StorePath; import com.github.tobato.fastdfs.domain.proto.storage.DownloadByteArray; import com.github.tobato.fastdfs.service.AppendFileStorageClient; import com.github.tobato.fastdfs.service.FastFileStorageClient; import lombok.extern.slf4j.Slf4j; import org.apache.commons.io.FilenameUtils; import org.apache.commons.lang3.StringUtils; import org.springframework.stereotype.Component; import org.springframework.web.multipart.MultipartFile; import javax.annotation.Resource; import java.io.*; import java.nio.charset.Charset; /** * 文件上传工具类 */ @Slf4j @Component public class FastDFSClient { @Resource private FastFileStorageClient storageClient; @Resource private AppendFileStorageClient appendFileStorageClient; /** * 分片文件上传 * @param group 文件上传组 * @param inputStream 上传文件流 * @param fileSize 当前块大小 * @param fileName 文件名称 * @param path 文件追加路径 * @param offset 偏移量 * @return */ public String appendUpload(String group,InputStream inputStream,Long fileSize,String fileName,String path,Long offset){ /*如果当前块是第一块则先添加文件*/ if(StringUtils.isBlank(path)){ StorePath storePath = appendFileStorageClient.uploadAppenderFile(group, inputStream, fileSize, FileUtil.extName(fileName)); path = storePath.getPath(); }else{ appendFileStorageClient.modifyFile(group, path, inputStream, fileSize,offset); } return path; } /*创建增量断点续传文件*/ public String createAppendFile(String group,InputStream inputStream,Long fileSize,String fileName){ StorePath storePath = appendFileStorageClient.uploadAppenderFile(group, inputStream, fileSize, FileUtil.extName(fileName)); return storePath.getPath(); } /** * 修改文件块 * @param group 组名 * @param path 路径 * @param inputStream 文件流 * @param fileSize 文件大小 * @param offset 偏移量 */ public void modifyAppendFile(String group,String path,InputStream inputStream,Long fileSize,Long offset){ appendFileStorageClient.modifyFile(group, path, inputStream, fileSize,offset); } /** * 上传文件 */ public String uploadImg(MultipartFile multipartFile) throws Exception { String originalFilename = multipartFile.getOriginalFilename(). substring(multipartFile.getOriginalFilename(). lastIndexOf(".") + 1); StorePath storePath = this.storageClient.uploadImageAndCrtThumbImage( multipartFile.getInputStream(), multipartFile.getSize(), originalFilename, null); return storePath.getFullPath(); } /** * 上传文件 * @param multipartFile 文件对象 * @return 文件访问地址 * @throws IOException */ public String uploadFile(MultipartFile multipartFile) throws IOException { String originalFilename = multipartFile.getOriginalFilename(). substring(multipartFile.getOriginalFilename(). lastIndexOf(".") + 1); StorePath storePath = storageClient.uploadFile(multipartFile.getInputStream(),multipartFile.getSize(), originalFilename,null); return storePath.getFullPath(); } /** * 上传文件 * @param file 文件对象 * @return 文件访问地址 * @throws IOException */ public String uploadFile(File file) throws IOException { FileInputStream inputStream = new FileInputStream (file); StorePath storePath = storageClient.uploadFile(inputStream,file.length(), FilenameUtils.getExtension(file.getName()),null); return storePath.getFullPath(); } /** * 将一段字符串生成一个文件上传 * @param content 文件内容 * @param fileExtension * @return */ public String uploadFile(String content, String fileExtension) { byte[] buff = content.getBytes(Charset.forName("UTF-8")); ByteArrayInputStream stream = new ByteArrayInputStream(buff); StorePath storePath = storageClient.uploadFile(stream,buff.length, fileExtension,null); return storePath.getFullPath(); } /** * 下载文件 * * @param fileUrl 文件URL * @return 文件字节 * @throws IOException */ public byte[] downloadFile(String fileUrl) throws IOException { if(fileUrl.startsWith("/")){ fileUrl=fileUrl.substring(fileUrl.indexOf("/")+1); } String group = fileUrl.substring(0, fileUrl.indexOf("/")); String path = fileUrl.substring(fileUrl.indexOf("/") + 1); DownloadByteArray downloadByteArray = new DownloadByteArray(); byte[] bytes = storageClient.downloadFile(group, path, downloadByteArray); return bytes; } /** * 删除文件 */ public Boolean deleteFile(String fileUrl) { if (StringUtils.isEmpty(fileUrl)) { log.info("fileUrl == >>文件路径为空..."); return Boolean.FALSE; } try { StorePath storePath = StorePath.parseFromUrl(fileUrl); storageClient.deleteFile(storePath.getGroup(), storePath.getPath()); return Boolean.TRUE; } catch (Exception e) { log.error("[文件服务-文件删除失败]", e); return Boolean.FALSE; } } }
package com.techhero.component.oss.constant; public interface UploadConstant { /*缓存过期时间*/ long REDIS_EXPIRE = 60L * 60L * 12L; /*Redis存储KEY*/ String FILE_UPLOAD = "FILE_UPLOAD:"; /*当前文件路径*/ String FILE_UPLOAD_PATH = FILE_UPLOAD + "PATH:"; /*当前文件已上传大小*/ String FILE_UPLOAD_SIZE = FILE_UPLOAD + "SIZE:"; /*当前最大块号*/ String FILE_UPLOAD_CHUNK = FILE_UPLOAD + "CHUNK:"; }
package com.techhero.component.oss.controller; import com.techhero.common.model.oss.MultipartFileChunk; import com.techhero.common.utils.req.ResBean; import com.techhero.component.oss.service.FileUploadService; import lombok.extern.slf4j.Slf4j; import org.springframework.http.MediaType; import org.springframework.web.bind.annotation.*; import org.springframework.web.multipart.MultipartFile; import javax.annotation.Resource; @Slf4j @RestController @RequestMapping("big") public class BigFileUploadController { @Resource private FileUploadService fileUploadService; /** * 分片文件上传 */ @PostMapping("trunkUpload") public ResBean trunkUpload(MultipartFileChunk multipartFileChunk){ return fileUploadService.trunkUpload(multipartFileChunk); } /** * 分片文件上传 */ @PostMapping(value = "feignUpload",consumes = MediaType.MULTIPART_FORM_DATA_VALUE) public ResBean feignUpload(MultipartFile file, String identifier, Long chunkSize, Long totalChunks, Long totalSize, Long chunkNumber, Long currentChunkSize, String fileName ){ return fileUploadService.trunkUpload(new MultipartFileChunk(identifier,chunkSize,totalChunks,totalSize,chunkNumber,currentChunkSize,fileName,file)); } /** * 获取文件路径 */ @GetMapping("getFilePath") public ResBean getFilePath(@RequestParam String identifier){ return fileUploadService.getFilePath(identifier); } }
package com.techhero.component.oss.controller; import com.techhero.common.model.oss.MultipartFileChunk; import com.techhero.common.utils.req.ResBean; import com.techhero.component.oss.service.FileUploadService; import lombok.extern.slf4j.Slf4j; import org.apache.commons.io.FileUtils; import org.apache.tomcat.util.http.fileupload.servlet.ServletFileUpload; import org.springframework.beans.factory.annotation.Value; import org.springframework.web.bind.annotation.*; import org.springframework.web.multipart.MultipartFile; import javax.annotation.Resource; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; /** * 文件上传接口 */ @Slf4j @RestController @RequestMapping("small") public class SmallFileController { /*上传文件位置*/ @Value("${fdfs.file.path}") private String filePath; /*文件临时目录*/ @Value("${fdfs.file.tempPath}") private String filePathTemp; @Resource private FileUploadService fileUploadService; /** * 文件上传 * http://xxx/group1/M00/00/00/wKhIgl0n4AKABxQEABhlMYw_3Lo825.png */ @PostMapping(value = "/uploadFile") public ResBean uploadFile(@RequestParam("file") MultipartFile file) { return fileUploadService.uploadFile(file); } /** * 文件删除 */ @GetMapping(value = "/deleteByPath") public ResBean deleteByPath(@RequestParam("filePath") String filePath) { return fileUploadService.delFileByPath(filePath); } /** * 文件下载 * @param filePath 文件路径 */ @GetMapping("/download") public void downloadFile(@RequestParam("filePath") String filePath, HttpServletResponse response) throws Exception { fileUploadService.downloadFile(filePath,response); } /** * 分片上传(大文件) */ @PostMapping("upload") public ResBean upload(MultipartFileChunk chunk, HttpServletRequest request) throws IOException { boolean isMultipart = ServletFileUpload.isMultipartContent(request); if (isMultipart) { MultipartFile file = chunk.getFile(); if (file == null) { return ResBean.failed("文件未选择"); } Long chunkNumber = chunk.getChunkNumber(); if (chunkNumber == null) { chunkNumber = 0L; } File outFile = new File(filePathTemp + File.separator + chunk.getIdentifier(), chunkNumber + ".part"); if(outFile.exists()){ return ResBean.success("附件上传成功"); } InputStream inputStream = file.getInputStream(); FileUtils.copyInputStreamToFile(inputStream, outFile); return ResBean.success("附件上传成功"); } return ResBean.failed("没有文件"); } /** * 合并所有分片 */ @GetMapping("/merge") @ResponseBody public ResBean merge(String fileName, String identifier) throws Exception { File file = new File(filePathTemp + File.separator + identifier); if (file.isDirectory()) { File[] files = file.listFiles(); if (files != null && files.length > 0) { File partFile = new File(filePath + File.separator + fileName); for (int i = 1; i <= files.length; i++) { File s = new File(filePathTemp + File.separator + identifier, i + ".part"); try (FileOutputStream destTempFos = new FileOutputStream(partFile, true)) { FileUtils.copyFile(s, destTempFos); } } //删除临时目录文件 FileUtils.deleteDirectory(file); return ResBean.success(fileName); } } return ResBean.failed("文件上传失败"); } }
package com.techhero.component.oss.service; import com.github.tobato.fastdfs.domain.conn.FdfsWebServer; import com.techhero.common.bean.config.RedisCacheService; import com.techhero.common.model.oss.MultipartFileChunk; import com.techhero.common.utils.req.ResBean; import com.techhero.component.oss.config.FastDFSClient; import com.techhero.component.oss.constant.UploadConstant; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.springframework.web.multipart.MultipartFile; import javax.annotation.Resource; import javax.servlet.ServletOutputStream; import javax.servlet.http.HttpServletResponse; import java.io.IOException; import java.io.InputStream; import java.net.URLEncoder; @Slf4j @Service public class FileUploadService { /** * 解析路径 */ private static final String SEPARATOR = "/"; @Value("${fdfs.groupName:group1}") private String groupName; /** * 文件上传客户端 */ @Resource private FastDFSClient fastDFSClient; /** * 缓存 */ @Resource private RedisCacheService redisCacheService; /** * web地址配置类 */ @Resource private FdfsWebServer fdfsWebServer; /** * 简单文件上传 * * @param file 文件 */ public ResBean uploadFile(MultipartFile file) { try { String path = fastDFSClient.uploadFile(file); if (!org.springframework.util.StringUtils.isEmpty(path)) { if (!path.startsWith("/")) { return ResBean.success("/" + path); } return ResBean.success(path); } else { log.info("[文件服务-文件上传失败]@路径为空"); } } catch (Exception e) { log.error("[文件上传失败]", e); } return ResBean.failed("网络正忙,请重试..."); } /** * 文件路径 * * @param filePath 文件路径 */ public ResBean delFileByPath(String filePath) { log.info("[文件删除]@ {}", filePath); Boolean delFlag = fastDFSClient.deleteFile(filePath); return ResBean.validCountBean(delFlag); } /** * 文件下载 * * @param filePath 文件路径 */ public void downloadFile(String filePath, HttpServletResponse response) throws Exception { String fileName = filePath.substring(filePath.lastIndexOf(".") + 1); log.debug("[文件下载]-[文件名称={}]", fileName); byte[] bytes = fastDFSClient.downloadFile(filePath); response.setHeader("Content-disposition", "attachment;filename=" + URLEncoder.encode(fileName, "UTF-8")); response.setCharacterEncoding("UTF-8"); try (ServletOutputStream outputStream = response.getOutputStream()) { outputStream.write(bytes); outputStream.flush(); } catch (Exception e) { log.error("[文件下载-异常]", e); } } /** * 分片文件上传 * * @param multipartFileChunk 分片文件对象 * @describe 返回路径 M00/00/00/rBID22Da54CEOok0AAAAAJscELM782.exe */ public ResBean trunkUpload(MultipartFileChunk multipartFileChunk) { log.info("[分片文件上传-上传数据开始]#\n{}",multipartFileChunk); MultipartFile multipartFile = multipartFileChunk.getFile(); /*总块数*/ Long totalChunks = multipartFileChunk.getTotalChunks(); /*当前块角标*/ Long chunkNumber = multipartFileChunk.getChunkNumber(); /*文件唯一凭证信息*/ String identifier = multipartFileChunk.getIdentifier(); /*当前块大小*/ Long currentChunkSize = multipartFile.getSize(); /*第一块文件路径标记*/ String filePath = redisCacheService.get(UploadConstant.FILE_UPLOAD_PATH + identifier); try (InputStream inputStream = multipartFile.getInputStream()) { if (chunkNumber.equals(1L) && filePath == null) { /*文件名称*/ String fileName = multipartFileChunk.getFileName(); String path = fastDFSClient.createAppendFile(groupName, inputStream, currentChunkSize, fileName); /*存放路径*/ redisCacheService.set(UploadConstant.FILE_UPLOAD_PATH + identifier, path, UploadConstant.REDIS_EXPIRE); /*存放当前文件大小*/ redisCacheService.set(UploadConstant.FILE_UPLOAD_SIZE + identifier, currentChunkSize.toString(), UploadConstant.REDIS_EXPIRE); /*最大块号*/ redisCacheService.set(UploadConstant.FILE_UPLOAD_CHUNK + identifier, chunkNumber.toString(), UploadConstant.REDIS_EXPIRE); return ResBean.success(path); } /*-----------下面是文件已经创建,冲第二块开始,则走下边流程------------*/ /*如果第一块没有上传,则其他块不允许上传*/ if (chunkNumber > 1L && StringUtils.isBlank(filePath)) { return ResBean.failed("请等待头文件传输完毕,文件马上上传..."); } /*文件最大块号*/ String fileUploadChunk = redisCacheService.get(UploadConstant.FILE_UPLOAD_CHUNK + identifier); Long fileUploadChunkMax = Long.parseLong(fileUploadChunk) + 1L; if (chunkNumber > 1L && StringUtils.isNotBlank(filePath) && StringUtils.isNotBlank(fileUploadChunk) && chunkNumber.equals(fileUploadChunkMax)) { /*文件已上传大小*/ String fileUploadSizeRedis = redisCacheService.get(UploadConstant.FILE_UPLOAD_SIZE + identifier); Long fileUploadSize = Long.valueOf(fileUploadSizeRedis); fastDFSClient.modifyAppendFile(groupName, filePath, inputStream, currentChunkSize, fileUploadSize); long historyFileSize = currentChunkSize + fileUploadSize; /*当前文件已经上传大小*/ redisCacheService.set(UploadConstant.FILE_UPLOAD_SIZE + identifier, Long.toString(historyFileSize), UploadConstant.REDIS_EXPIRE); /*最大块号*/ redisCacheService.set(UploadConstant.FILE_UPLOAD_CHUNK + identifier, chunkNumber.toString(), UploadConstant.REDIS_EXPIRE); /*如果当前块角标与总块数相等或者大于的情况下则,设定redis设定一下超时时间,防止用户页面跳转后一直没有清理缓存*/ if (chunkNumber >= totalChunks) { /*这里只需要设定路径缓存即可,因为路径缓存超时时间设定比较早而其他缓存则刚设置,延迟不到一秒,没有并发则忽略延迟*/ redisCacheService.expire(UploadConstant.FILE_UPLOAD_PATH, UploadConstant.REDIS_EXPIRE); } return ResBean.success(filePath); } } catch (IOException e) { log.error("[分片文件上传-产生流异常]", e); } return ResBean.failed("当前文件等待上传..."); } /** * 获取文件路径 * @param identifier 文件唯一凭证 */ public ResBean getFilePath(String identifier) { String filePath = redisCacheService.get(UploadConstant.FILE_UPLOAD_PATH + identifier); String[] keys = new String[]{ UploadConstant.FILE_UPLOAD_PATH + identifier,//路径 UploadConstant.FILE_UPLOAD_SIZE + identifier,//已上传文件大小 UploadConstant.FILE_UPLOAD_CHUNK + identifier,//当前已上传文件块 }; /*清除redis缓存*/ redisCacheService.del(keys); String fullPath = getFullPath(fdfsWebServer.getWebServerUrl(), groupName, filePath); return ResBean.success(fullPath); } /*拼接文件访问路径*/ private String getFullPath(String webUrl, String groupName, String filePath) { return (webUrl + SEPARATOR + groupName + SEPARATOR + filePath).replace(SEPARATOR.concat(SEPARATOR), SEPARATOR); } }
server: port: 19908 spring: autoconfigure: exclude: org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration,org.springframework.boot.autoconfigure.jdbc.DataSourceTransactionManagerAutoConfiguration application: name: techhero-component-oss profiles: active: local cloud: config: fail-fast: true discovery: serviceId: techhero-config-server enabled: true profile: ${spring.profiles.active} label: ${spring.profiles.active} security: basic: enabled: false --- spring: profiles: local eureka: instance: prefer-ip-address: true lease-renewal-interval-in-seconds: 5 lease-expiration-duration-in-seconds: 20 client: serviceUrl: defaultZone: http://techhero:techhero123@127.0.0.1:10421/eureka registry-fetch-interval-seconds: 10
<!-- FastDFS依赖 --> <dependency> <groupId>com.github.tobato</groupId> <artifactId>fastdfs-client</artifactId> <version>1.26.5</version> </dependency> <!--连接池--> <dependency> <groupId>com.alibaba</groupId> <artifactId>druid</artifactId> <version>${druid.version}</version> </dependency> <dependency> <groupId>com.alibaba</groupId> <artifactId>druid-spring-boot-starter</artifactId> <version>${druid.starter.version}</version> </dependency> <!--FEIGN --> <dependency> <groupId>org.springframework.cloud</groupId> <artifactId>spring-cloud-starter-feign</artifactId> </dependency> <!-- 消息总线 --> <dependency> <groupId>org.springframework.cloud</groupId> <artifactId>spring-cloud-starter-bus-amqp</artifactId> </dependency> <!--这个包下面会沾--> <dependency> <groupId>com.techhero.common</groupId> <artifactId>techhero-common</artifactId> <version>1.0-SNAPSHOT</version> </dependency>
common包依赖
<!--Redis--> <dependency> <groupId>org.springframework.boot</groupId> <artifactId>spring-boot-starter-data-redis</artifactId> </dependency> <!--添加spring对cache的支持--> <dependency> <groupId>org.springframework.boot</groupId> <artifactId>spring-boot-starter-cache</artifactId> </dependency> <!--消息总线--> <dependency> <groupId>org.springframework.cloud</groupId> <artifactId>spring-cloud-starter-bus-amqp</artifactId> </dependency> <!--JWT--> <dependency> <groupId>io.jsonwebtoken</groupId> <artifactId>jjwt</artifactId> <version>${jjwt.version}</version> </dependency> <dependency> <groupId>com.baomidou</groupId> <artifactId>mybatis-plus</artifactId> <version>${mybatis-plus.version}</version> </dependency> <dependency> <groupId>com.xiaoleilu</groupId> <artifactId>hutool-all</artifactId> <version>${hutool.version}</version> </dependency> <dependency> <groupId>cn.hutool</groupId> <artifactId>hutool-all</artifactId> <version>5.0.6</version> </dependency> <!-- TTL --> <dependency> <groupId>com.alibaba</groupId> <artifactId>transmittable-thread-local</artifactId> <version>${ttl.version}</version> </dependency> <!--切面增强AOP--> <dependency> <groupId>org.springframework.boot</groupId> <artifactId>spring-boot-starter-aop</artifactId> </dependency> <!-- 序列化工具 --> <dependency> <groupId>com.alibaba</groupId> <artifactId>fastjson</artifactId> <version>1.2.46</version> </dependency> <dependency> <groupId>org.codehaus.jackson</groupId> <artifactId>jackson-mapper-asl</artifactId> <version>1.9.13</version> </dependency> <dependency> <groupId>com.fasterxml.jackson.core</groupId> <artifactId>jackson-databind</artifactId> <version>2.3.0</version> </dependency> <!-- okhttp --> <dependency> <groupId>io.github.openfeign</groupId> <artifactId>feign-okhttp</artifactId> </dependency> <!--<dependency> <groupId>com.netflix.feign</groupId> <artifactId>feign-okhttp</artifactId> <version>8.18.0</version> </dependency>--> <!--swagger--> <dependency> <groupId>io.springfox</groupId> <artifactId>springfox-swagger2</artifactId> <version>${swagger2.version}</version> </dependency> <!--配置属性加密工具--> <dependency> <groupId>com.github.ulisesbocchio</groupId> <artifactId>jasypt-spring-boot-starter</artifactId> <version>${jasypt.version}</version> </dependency> <dependency> <groupId>org.apache.poi</groupId> <artifactId>poi</artifactId> <version>${apache.poi.version}</version> </dependency> <dependency> <groupId>org.apache.poi</groupId> <artifactId>poi-examples</artifactId> <version>${apache.poi.version}</version> </dependency> <dependency> <groupId>org.apache.poi</groupId> <artifactId>poi-excelant</artifactId> <version>${apache.poi.version}</version> </dependency> <dependency> <groupId>org.apache.poi</groupId> <artifactId>poi-ooxml</artifactId> <version>${apache.poi.version}</version> </dependency> <dependency> <groupId>org.apache.poi</groupId> <artifactId>poi-ooxml-schemas</artifactId> <version>${apache.poi.version}</version> </dependency> <dependency> <groupId>org.apache.poi</groupId> <artifactId>poi-scratchpad</artifactId> <version>${apache.poi.version}</version> </dependency> <dependency> <groupId>commons-fileupload</groupId> <artifactId>commons-fileupload</artifactId> <version>${fileupload.version}</version> </dependency> <!-- spring-boot-starter-data-mongodb --> <!--<dependency> <groupId>org.springframework.boot</groupId> <artifactId>spring-boot-starter-data-mongodb</artifactId> </dependency>--> <!-- 邮件 --> <dependency> <groupId>org.springframework.boot</groupId> <artifactId>spring-boot-starter-mail</artifactId> </dependency> <!--security --> <!--<dependency> <groupId>org.springframework.security</groupId> <artifactId>spring-security-core</artifactId> <version>4.2.5.RELEASE</version> </dependency>--> <dependency> <groupId>com.netflix.zuul</groupId> <artifactId>zuul-core</artifactId> <version>2.1.3</version> <scope>compile</scope> </dependency> <!--TechHero--> <!-- <dependency>--> <!-- <groupId>com.itechhero</groupId>--> <!-- <artifactId>module-framework-cache</artifactId>--> <!-- <version>1.0.1-RC1</version>--> <!-- </dependency>--> <!-- <dependency>--> <!-- <groupId>com.itechhero</groupId>--> <!-- <artifactId>module-framework-core</artifactId>--> <!-- <version>1.0.1-RC1</version>--> <!-- </dependency>--> <!--通用Mapper--> <dependency> <groupId>tk.mybatis</groupId> <artifactId>mapper</artifactId> <version>${tkmapper.version}</version> </dependency> <!--数据库驱动--> <dependency> <groupId>com.oracle</groupId> <artifactId>ojdbc6</artifactId> <version>${oracle.version}</version> </dependency> <dependency> <groupId>mysql</groupId> <artifactId>mysql-connector-java</artifactId> <version>5.1.46</version> <scope>runtime</scope> </dependency> <dependency> <groupId>org.mybatis.generator</groupId> <artifactId>mybatis-generator-core</artifactId> <version>1.3.6</version> </dependency> <!--<dependency> <groupId>com.techhero.common</groupId> <artifactId>techhero-common-cache</artifactId> <version>1.0-SNAPSHOT</version> </dependency>--> <!--原core包中所用到的依赖--> <dependency> <groupId>commons-beanutils</groupId> <artifactId>commons-beanutils</artifactId> </dependency> <!--原core包中所用到的依赖--> <dependency> <groupId>net.sf.ezmorph</groupId> <artifactId>ezmorph</artifactId> <version>1.0.6</version> </dependency> <!--原core包中所用到的依赖--> <dependency> <groupId>net.sf.json-lib</groupId> <artifactId>json-lib</artifactId> <version>2.4</version> <type>jar</type> <classifier>jdk15</classifier> </dependency> <dependency> <groupId>org.apache.commons</groupId> <artifactId>commons-lang3</artifactId> <version>3.7</version> </dependency> <!--原core包中所用到的依赖--> <dependency> <groupId>org.apache.velocity</groupId> <artifactId>velocity</artifactId> </dependency> <!--原core包中所用到的依赖--> <dependency> <groupId>org.springframework</groupId> <artifactId>spring-test</artifactId> </dependency> <!--原core包中所用到的依赖--> <dependency> <groupId>dom4j</groupId> <artifactId>dom4j</artifactId> <version>1.6.1</version> </dependency> <!--原core包中所用到的依赖--> <dependency> <groupId>org.springframework.security</groupId> <artifactId>spring-security-core</artifactId> </dependency> <!--原core包中所用到的依赖--> <dependency> <groupId>commons-httpclient</groupId> <artifactId>commons-httpclient</artifactId> <version>3.1</version> </dependency> <dependency> <groupId>com.github.pagehelper</groupId> <artifactId>pagehelper</artifactId> <version>4.1.1</version> <scope>compile</scope> </dependency> <!--quartz相关依赖--> <dependency> <groupId>org.quartz-scheduler</groupId> <artifactId>quartz</artifactId> <version>2.3.0</version> </dependency> <dependency> <groupId>org.quartz-scheduler</groupId> <artifactId>quartz-jobs</artifactId> <version>2.3.0</version> </dependency> <dependency> <groupId>io.github.openfeign.form</groupId> <artifactId>feign-form</artifactId> <version>3.0.3</version> </dependency> <dependency> <groupId>io.github.openfeign.form</groupId> <artifactId>feign-form-spring</artifactId> <version>3.0.3</version> </dependency>
下边是common包写的类,这个是为了其他子工程不需要做任何事儿,只需要依赖于common工程就可以直接feign调用oss系统
package com.techhero.common.bean.config.oss; import com.techhero.common.feigin.OssFeign; import com.techhero.common.utils.JSONUtil; import com.techhero.common.utils.db.IdWorker; import com.techhero.common.utils.req.ResBean; import lombok.extern.slf4j.Slf4j; import org.apache.commons.fileupload.FileItem; import org.apache.commons.fileupload.FileItemFactory; import org.apache.commons.fileupload.disk.DiskFileItemFactory; import org.springframework.stereotype.Component; import org.springframework.web.multipart.MultipartFile; import org.springframework.web.multipart.commons.CommonsMultipartFile; import javax.annotation.Resource; import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.io.OutputStream; /** * 全局提供文件上传工具类 */ @Slf4j @Component public class OssClientUtils { @Resource private OssFeign ossFeign; public static void main(String[] args) { long a = 18 / 19; System.out.println(a); } /** * 分片文件上传 * @param file 文件对象 * @param splitSize 分块上传每块多大 1M=1*1024*1024 * @param retryNum 重试次数(文件上传时候出现上传失败情况,这种情况下,重试多少次) * @return 返回响应对象 */ public ResBean upload(File file, Long splitSize, int retryNum) { String fileName = file.getName(); /*文件总大小*/ Long fileTotalSize = file.length(); long totalChunks = fileTotalSize / splitSize; if (fileTotalSize % splitSize > 0 || totalChunks == 0) { totalChunks++; } /*文件唯一标识*/ String identifier = IdWorker.getOrderIdByUUId(); FileItemFactory factory = new DiskFileItemFactory(16, null); try (FileInputStream in = new FileInputStream(file)) { for (int i = 1; i <= totalChunks; i++) { FileItem item = factory.createItem("file", "multipart/form-data", true, fileName); try(OutputStream out = item.getOutputStream()){ byte[] b = new byte[splitSize.intValue()]; int read = in.read(b); log.debug("[读取字节长度{}-byte]", read); out.write(b, 0, read); out.flush(); MultipartFile multipartFile = new CommonsMultipartFile(item); /*重试机制*/ for (int r = 0; r < retryNum; r++) { ResBean resBean = ossFeign.feignUpload(multipartFile, identifier, splitSize, totalChunks, fileTotalSize, (long) i, (long) read, fileName); log.info("[FEIGN文件上传,返回信息]#{}", JSONUtil.objectToJson(resBean)); if (resBean.isSuccess()) { log.info("[FEIGN文件上传,上传成功]#{}", resBean.getData()); break; } /*如果是最后一次,则返回上传失败*/ if (r == retryNum - 1) { return ResBean.failed("[文件上传][" + i + "块上传" + retryNum + "次不成功..]"); } } } } ResBean endResBean = ossFeign.getFilePath(identifier); if (endResBean.isSuccess()) { log.info("[FEIGN文件上传,**文件切片全部**上传成功]#{}", endResBean.getData()); return endResBean; } } catch (IOException e) { log.error("[切割文件FEIGN调用-出现IO异常]", e); return ResBean.failed("出现IO异常"); } return ResBean.failed("文件上传失败..."); } }
package com.techhero.common.feigin; import feign.codec.Encoder; import com.techhero.common.bean.config.oss.MultipartSupportConfig; import com.techhero.common.constant.ServiceNameConstant; import com.techhero.common.feigin.fallback.OssFeignFallback; import com.techhero.common.utils.req.ResBean; import feign.form.spring.SpringFormEncoder; import org.springframework.beans.factory.ObjectFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.web.HttpMessageConverters; import org.springframework.cloud.netflix.feign.FeignClient; import org.springframework.cloud.netflix.feign.support.SpringEncoder; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.http.MediaType; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RequestPart; import org.springframework.web.multipart.MultipartFile; @FeignClient(path = "big", value = ServiceNameConstant.OSS_SERVICE, fallback = OssFeignFallback.class, configuration = OssFeign.MultipartSupportConfig.class) public interface OssFeign { /*@PostMapping(value="trunkUpload",consumes = MediaType.MULTIPART_FORM_DATA_VALUE) ResBean trunkUpload(@RequestBody MultipartFileChunk multipartFileChunk);*/ /** * 分片文件上传 * * @param identifier 唯一标识 * @param chunkSize 分片大小 * @param totalChunks 总分片数量 * @param totalSize 总大小 * @param chunkNumber 往前分片号 * @param currentChunkSize 当前分片大小 * @param fileName 文件名称 * @param file 文件 */ @PostMapping(value = "feignUpload", consumes = MediaType.MULTIPART_FORM_DATA_VALUE) ResBean feignUpload( @RequestPart("file") MultipartFile file, @RequestParam("identifier") String identifier, @RequestParam("chunkSize") Long chunkSize, @RequestParam("totalChunks") Long totalChunks, @RequestParam("totalSize") Long totalSize, @RequestParam("chunkNumber") Long chunkNumber, @RequestParam("currentChunkSize") Long currentChunkSize, @RequestParam("fileName") String fileName ); /** * 获取文件路径 * * @param identifier 文件唯一标识 */ @GetMapping("getFilePath") ResBean getFilePath(@RequestParam("identifier") String identifier); @Configuration class MultipartSupportConfig { @Autowired private ObjectFactory<HttpMessageConverters> messageConverters; @Bean public Encoder feignFormEncoder() { return new SpringFormEncoder(new SpringEncoder(messageConverters)); } } }
package com.techhero.common.feigin.fallback; import com.techhero.common.feigin.OssFeign; import com.techhero.common.utils.req.ResBean; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RequestPart; import org.springframework.web.multipart.MultipartFile; @Slf4j @Service public class OssFeignFallback implements OssFeign { /** * @param identifier 唯一标识 * @param chunkSize 分片大小 * @param totalChunks 总分片数量 * @param totalSize 总大小 * @param chunkNumber 往前分片号 * @param currentChunkSize 当前分片大小 * @param fileName 文件名称 * @param file 文件 */ @Override public ResBean feignUpload( @RequestPart("file") MultipartFile file, @RequestParam("identifier") String identifier, @RequestParam("chunkSize") Long chunkSize, @RequestParam("totalChunks") Long totalChunks, @RequestParam("totalSize") Long totalSize, @RequestParam("chunkNumber") Long chunkNumber, @RequestParam("currentChunkSize") Long currentChunkSize, @RequestParam("fileName") String fileName ) { log.info("[文件上传->分片上传]#远程调用失败&\n{}", identifier); return ResBean.failed("[文件上传->分片上传]#远程调用失败,请重试.."); } /** * 获取文件路径 * * @param identifier 文件唯一标识 */ @Override public ResBean getFilePath(String identifier) { log.info("[文件上传-获取文件全路径]#远程调用失败&\n文件唯一标识={}", identifier); return ResBean.failed("[文件上传-获取文件全路径]#远程调用失败,请重试.."); } }
package com.techhero.common.model.oss; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; import lombok.ToString; import org.springframework.web.multipart.MultipartFile; import java.io.Serializable; @Data @ToString @AllArgsConstructor @NoArgsConstructor public class MultipartFileChunk implements Serializable { /*文件标识*/ private String identifier; /*分块大小*/ private Long chunkSize; /*当前文件总块数*/ private Long totalChunks; /*当前文件总大小*/ private Long totalSize; /*当前块号*/ private Long chunkNumber; /*当前块大小*/ private Long currentChunkSize; /*文件名称*/ private String fileName; /*文件*/ private MultipartFile file; }
OK 目前上面的代码就是如此,下面是子工程调用的地方顺便搞出来吧
只需要把OssClientUtils注入到业务类里面即可
package com.techhero.base.transdata.service; import com.techhero.common.bean.config.oss.OssClientUtils; import com.techhero.common.utils.JSONUtil; import com.techhero.common.utils.req.ResBean; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; import javax.annotation.Resource; import java.io.File; @Slf4j @Component("fileUpLoadService") public class FileUpLoadServiceImpl { @Resource private OssClientUtils ossClientUtils; //@PostConstruct public void load(){ File file=new File("/Users/cnnoter/Downloads/21_06_26 08_42_34.mp4"); //第一个参数文件对象,第二个参数分片大小这个是10M分片大小,最后边是重试次数 ResBean upload = ossClientUtils.upload(file, 10 * 1024 * 1024L, 5); log.info("传输完成\n{}", JSONUtil.objectToJson(upload)); } }
前端代码
<template> <div> <el-button icon="el-icon-circle-plus" type="primary" @click="start">打开上传文件框</el-button> <el-dialog title="上传" :visible.sync="dialogVisible" width="50%" class="el-dialog-global" > <div class="sec"> <uploader :options="options" :file-status-text="statusText" ref="uploader" @file-complete="fileComplete" @complete="complete"></uploader> </div> <div slot="footer" class="dialog-footer"> <el-button @click="dialogVisible = false">取 消</el-button> <el-button type="primary" @click="dialogVisible = false">确 定</el-button> </div> </el-dialog> </div> </template> <script> import axios from 'axios'; export default { data () { return { dialogVisible: false, options: { target: axios.defaults.baseURL + '/oss/big/trunkUpload', testChunks: false, maxChunkRetries: 1000, //最大自动失败重试上传次数 simultaneousUploads:1, chunkSize: 10 * 1024 * 1024,//分片大小 // checkChunkUploadedByResponse: function (chunk, message) { // let objMessage = JSON.parse(message); // console.log(objMessage); // console.log(objMessage.success==true); // return objMessage.success==true; // }, processParams(params) {//每一次分片传给后台的参数,params是该方法返回的形参,包含分片信息 console.log(params, '123'); params.fileName=params.filename; return params; // return {//返回一个对象,会添加到每一个分片的请求参数里面 // chunkSize: params.chunkSize, // totalSize: params.totalSize, // filename: params.filename, // identifier: params.identifier, // totalChunks: params.totalChunks, // chunkNumber: params.chunkNumber, // }; } }, attrs: { accept: 'image/*' }, statusText: { success: '成功了', error: '出错了', uploading: '上传中', paused: '暂停中', waiting: '等待中' } } }, methods: { start(){ this.dialogVisible=true; }, complete () { console.log('complete', arguments) }, fileComplete () { console.log('file complete', arguments) const file = arguments[0].file let url = '/big/getFilePath?identifier=' + arguments[0].uniqueIdentifier this.$httpclient.get(url, {}, res => { console.log(res); }, 'oss') } }, mounted () { this.$nextTick(() => { window.uploader = this.$refs.uploader.uploader }) } } </script>
Copyright © 2003-2013 www.wpsshop.cn 版权所有,并保留所有权利。