赞
踩
分片上传: 将一个文件按照指定大小分割成多份数据块(Part)分开上传, 上传之后再由服务端整合为原本的文件
分片上传场景:
分片上传流程:
- minio:
- minioUrl: http://ip地址:9000 # MinIO 服务地址-需要修改
- minioName: 账号 # MinIO 访问密钥-需要修改
- minioPass: 密码 # MinIO 秘钥密码-需要修改
- bucketName: 桶名 # MinIO 桶名称-需要修改
- region: ap-southeast-1 # MinIO 存储区域,可以指定为 "ap-southeast-1"
-
- spring:
- servlet:
- multipart:
- max-file-size: 10MB
- max-request-size: 10MB
- <!--minio-->
- <dependency>
- <groupId>io.minio</groupId>
- <artifactId>minio</artifactId>
- <version>8.0.3</version>
- </dependency>
- /**
- * @author xiaoyi
- */
- @Slf4j
- @AllArgsConstructor
- public class MinioTemplate {
- /**
- * MinIO 客户端
- */
- private final MinioClient minioClient;
- /**
- * MinIO 配置类
- */
- private final MinioConfig minioConfig;
-
- /**
- * 查询所有存储桶
- *
- * @return Bucket 集合
- */
- @SneakyThrows
- public List<Bucket> listBuckets() {
- return minioClient.listBuckets();
- }
-
- /**
- * 查询文件大小
- *
- * @return Bucket 集合
- */
- @SneakyThrows
- public Long getObjectSize(String bucketName, String objectName) {
- return minioClient.statObject(StatObjectArgs.builder().bucket(bucketName).object(objectName).build()).size();
- }
-
- /**
- * 桶是否存在
- *
- * @param bucketName 桶名
- * @return 是否存在
- */
- @SneakyThrows
- public boolean bucketExists(String bucketName) {
- return minioClient.bucketExists(BucketExistsArgs.builder().bucket(bucketName).build());
- }
-
- /**
- * 创建存储桶
- *
- * @param bucketName 桶名
- */
- @SneakyThrows
- public void makeBucket(String bucketName) {
- if (!bucketExists(bucketName)) {
- minioClient.makeBucket(MakeBucketArgs.builder().bucket(bucketName).build());
- }
- }
-
- /**
- * 删除一个空桶 如果存储桶存在对象不为空时,删除会报错。
- *
- * @param bucketName 桶名
- */
- @SneakyThrows
- public void removeBucket(String bucketName) {
- removeBucket(bucketName, false);
- minioClient.removeBucket(RemoveBucketArgs.builder().bucket(bucketName).build());
- }
-
- /**
- * 删除一个桶 根据桶是否存在数据进行不同的删除
- * 桶为空时直接删除
- * 桶不为空时先删除桶中的数据,然后再删除桶
- *
- * @param bucketName 桶名
- */
- @SneakyThrows
- public void removeBucket(String bucketName, boolean bucketNotNull) {
- if (bucketNotNull) {
- deleteBucketAllObject(bucketName);
- }
- minioClient.removeBucket(RemoveBucketArgs.builder().bucket(bucketName).build());
- }
-
- /**
- * 上传文件
- *
- * @param inputStream 流
- * @param originalFileName 原始文件名
- * @param bucketName 桶名
- * @return ObjectWriteResponse
- */
- @SneakyThrows
- public OssFile putObject(InputStream inputStream, String bucketName, String originalFileName) {
- String uuidFileName = generateFileInMinioName(originalFileName);
- try {
- if (ObjectUtils.isEmpty(bucketName)) {
- bucketName = minioConfig.getBucketName();
- }
- minioClient.putObject(
- PutObjectArgs.builder()
- .bucket(bucketName)
- .object(uuidFileName)
- .stream(inputStream, inputStream.available(), -1)
- .build());
-
-
- return new OssFile(uuidFileName, originalFileName);
- } finally {
- if (inputStream != null) {
- inputStream.close();
- }
- }
- }
-
-
- /**
- * 删除桶中所有的对象
- *
- * @param bucketName 桶对象
- */
- @SneakyThrows
- public void deleteBucketAllObject(String bucketName) {
- List<String> list = listObjectNames(bucketName);
- if (!list.isEmpty()) {
- for (String objectName : list) {
- deleteObject(bucketName, objectName);
- }
- }
- }
-
- /**
- * 查询桶中所有的对象名
- *
- * @param bucketName 桶名
- * @return objectNames
- */
- @SneakyThrows
- public List<String> listObjectNames(String bucketName) {
- List<String> objectNameList = new ArrayList<>();
- if (bucketExists(bucketName)) {
- Iterable<Result<Item>> results = listObjects(bucketName, true);
- for (Result<Item> result : results) {
- String objectName = result.get().objectName();
- objectNameList.add(objectName);
- }
- }
- return objectNameList;
- }
-
-
- /**
- * 删除一个对象
- *
- * @param bucketName 桶名
- * @param objectName 对象名
- */
- @SneakyThrows
- public void deleteObject(String bucketName, String objectName) {
- minioClient.removeObject(RemoveObjectArgs.builder()
- .bucket(bucketName)
- .object(objectName)
- .build());
- }
-
- /**
- * 上传分片文件
- *
- * @param inputStream 流
- * @param objectName 存入桶中的对象名
- * @param bucketName 桶名
- * @return ObjectWriteResponse
- */
- @SneakyThrows
- public OssFile putChunkObject(InputStream inputStream, String bucketName, String objectName) {
- try {
- minioClient.putObject(
- PutObjectArgs.builder()
- .bucket(bucketName)
- .object(objectName)
- .stream(inputStream, inputStream.available(), -1)
- .build());
- return new OssFile(objectName, objectName);
- } finally {
- if (inputStream != null) {
- inputStream.close();
- }
- }
- }
-
- /**
- * 返回临时带签名、Get请求方式的访问URL
- *
- * @param bucketName 桶名
- * @param filePath Oss文件路径
- * @return 临时带签名、Get请求方式的访问URL
- */
- @SneakyThrows
- public String getPresignedObjectUrl(String bucketName, String filePath) {
- return minioClient.getPresignedObjectUrl(
- GetPresignedObjectUrlArgs.builder()
- .method(Method.GET)
- .bucket(bucketName)
- .object(filePath)
- .build());
- }
-
- /**
- * 返回临时带签名、过期时间为1天的PUT请求方式的访问URL
- *
- * @param bucketName 桶名
- * @param filePath Oss文件路径
- * @param queryParams 查询参数
- * @return 临时带签名、过期时间为1天的PUT请求方式的访问URL
- */
- @SneakyThrows
- public String getPresignedObjectUrl(String bucketName, String filePath, Map<String, String> queryParams) {
- return minioClient.getPresignedObjectUrl(
- GetPresignedObjectUrlArgs.builder()
- .method(Method.PUT)
- .bucket(bucketName)
- .object(filePath)
- .expiry(1, TimeUnit.DAYS)
- .extraQueryParams(queryParams)
- .build());
- }
-
-
- /**
- * GetObject接口用于获取某个文件(Object)。此操作需要对此Object具有读权限。
- *
- * @param bucketName 桶名
- * @param objectName 文件路径
- */
- @SneakyThrows
- public InputStream getObject(String bucketName, String objectName) {
- return minioClient.getObject(
- GetObjectArgs.builder().bucket(bucketName).object(objectName).build());
- }
-
- /**
- * 查询桶的对象信息
- *
- * @param bucketName 桶名
- * @param recursive 是否递归查询
- * @return 桶的对象信息
- */
- @SneakyThrows
- public Iterable<Result<Item>> listObjects(String bucketName, boolean recursive) {
- return minioClient.listObjects(
- ListObjectsArgs.builder().bucket(bucketName).recursive(recursive).build());
- }
-
- /**
- * 获取带签名的临时上传元数据对象,前端可获取后,直接上传到Minio
- *
- * @param bucketName 桶名称
- * @param fileName 文件名
- * @return Map<String, String>
- */
- @SneakyThrows
- public Map<String, String> getPresignedPostFormData(String bucketName, String fileName) {
- // 为存储桶创建一个上传策略,过期时间为7天
- PostPolicy policy = new PostPolicy(bucketName, ZonedDateTime.now().plusDays(1));
- // 设置一个参数key,值为上传对象的名称
- policy.addEqualsCondition("key", fileName);
- // 添加Content-Type,例如以"image/"开头,表示只能上传照片,这里吃吃所有
- policy.addStartsWithCondition("Content-Type", MediaType.ALL_VALUE);
- // 设置上传文件的大小 64kiB to 10MiB.
- //policy.addContentLengthRangeCondition(64 * 1024, 10 * 1024 * 1024);
- return minioClient.getPresignedPostFormData(policy);
- }
-
-
- public String generateFileInMinioName(String originalFilename) {
- return "files" + StrUtil.SLASH + DateUtil.format(new Date(), "yyyy-MM-dd") + StrUtil.SLASH + UUID.randomUUID() + StrUtil.UNDERLINE + originalFilename;
- }
-
- /**
- * 初始化默认存储桶
- */
- @PostConstruct
- public void initDefaultBucket() {
- String defaultBucketName = minioConfig.getBucketName();
- if (bucketExists(defaultBucketName)) {
- log.info("默认存储桶:defaultBucketName已存在");
- } else {
- log.info("创建默认存储桶:defaultBucketName");
- makeBucket(minioConfig.getBucketName());
- }
- }
-
- /**
- * 文件合并,将分块文件组成一个新的文件
- *
- * @param bucketName 合并文件生成文件所在的桶
- * @param objectName 原始文件名
- * @param sourceObjectList 分块文件集合
- * @return OssFile
- */
- @SneakyThrows
- public OssFile composeObject(List<ComposeSource> sourceObjectList, String bucketName, String objectName) {
- minioClient.composeObject(ComposeObjectArgs.builder()
- .bucket(bucketName)
- .object(objectName)
- .sources(sourceObjectList)
- .build());
- String presignedObjectUrl = getPresignedObjectUrl(bucketName, objectName);
- return new OssFile(presignedObjectUrl, objectName);
- }
-
- /**
- * 文件合并,将分块文件组成一个新的文件
- *
- * @param originBucketName 分块文件所在的桶
- * @param targetBucketName 合并文件生成文件所在的桶
- * @param objectName 存储于桶中的对象名
- * @return OssFile
- */
- @SneakyThrows
- public OssFile composeObject(String originBucketName, String targetBucketName, String objectName) {
- Iterable<Result<Item>> results = listObjects(originBucketName, true);
- List<String> objectNameList = new ArrayList<>();
- for (Result<Item> result : results) {
- Item item = result.get();
- objectNameList.add(item.objectName());
- }
- if (ObjectUtils.isEmpty(objectNameList)) {
- throw new IllegalArgumentException(originBucketName + "桶中没有文件,请检查");
- }
-
- List<ComposeSource> composeSourceList = new ArrayList<>(objectNameList.size());
- // 对文件名集合进行升序排序
- objectNameList.sort((o1, o2) -> Integer.parseInt(o2) > Integer.parseInt(o1) ? -1 : 1);
- for (String object : objectNameList) {
- composeSourceList.add(ComposeSource.builder()
- .bucket(originBucketName)
- .object(object)
- .build());
- }
-
- return composeObject(composeSourceList, targetBucketName, objectName);
- }
- }
- import ai.gantong.common.constant.CommonConstant;
- import ai.gantong.common.constant.SymbolConstant;
- import ai.gantong.common.util.MinioUtil;
- import io.minio.MinioClient;
- import lombok.extern.slf4j.Slf4j;
- import org.springframework.beans.factory.annotation.Value;
- import org.springframework.context.annotation.Bean;
- import org.springframework.context.annotation.Configuration;
-
- /**
- * Minio文件上传配置文件
- *
- * @author xiaoyi
- */
- @Slf4j
- @Configuration
- public class MinioConfig {
- @Value(value = "${minio.minioUrl}")
- private String minioUrl;
- @Value(value = "${minio.minioName}")
- private String minioName;
- @Value(value = "${minio.minioPass}")
- private String minioPass;
- @Value(value = "${minio.bucketName}")
- private String bucketName;
-
- public String getBucketName() {
- return bucketName;
- }
-
- @Bean
- public void initMinio() {
- MinioUtil.setMinioUrl(minioUrl);
- MinioUtil.setMinioName(minioName);
- MinioUtil.setMinioPass(minioPass);
- MinioUtil.setBucketName(bucketName);
- }
-
- // 将 MinIOClient 注入到 Spring 上下文中
- @Bean("minioClient")
- public MinioClient minioClient() {
- return MinioClient.builder().endpoint(minioUrl).credentials(minioName, minioPass).region(region).build();
- }
-
- // 初始化MinioTemplate,封装了一些MinIOClient的基本操作
- @Bean(name = "minioTemplate")
- public MinioTemplate minioTemplate() {
- return new MinioTemplate(minioClient(), this);
- }
-
- }
- /**
- * 根据文件大小和文件的md5校验文件是否存在, 实现秒传接口
- *
- * @param md5 文件的md5
- * @return 操作是否成功
- */
- @ApiOperation(value = "极速秒传接口")
- @GetMapping(value = "/fastUpload")
- public Result<String> checkFileExists(@ApiParam(value = "文件的md5") String md5) {
- return fileService.checkFileExists(md5);
- }
-
- /**
- * 大文件分片上传
- *
- * @param md5 文件的md5
- * @param file 文件
- * @param fileName 文件名
- * @param index 分片索引
- * @return 分片执行结果
- */
- @ApiOperation(value = "上传分片的接口")
- @PostMapping(value = "/upload")
- public Result<String> upload(@ApiParam(value = "文件的md5") String md5, @ApiParam(value = "文件") MultipartFile file,
- @ApiParam(value = "文件名") String fileName, @ApiParam(value = "分片索引") Integer index) {
- return fileService.upload(md5, file, fileName, index);
- }
-
- /**
- * 大文件合并
- *
- * @param mergeInfo 合并信息
- * @return 分片合并的状态
- */
- @ApiOperation(value = "合并分片的接口")
- @PostMapping(value = "/merge")
- public Result<String> merge(@RequestBody MergeInfo mergeInfo) {
- return fileService.merge(mergeInfo);
- }
- @Slf4j
- @Service
- public class FileServiceImpl implements IFileService {
-
- private static final String MD5_KEY = "自定义前缀:minio:file:md5List";
-
- @Resource
- private MinioClient minioClient;
- @Resource
- private MinioConfig minioConfig;
- @Resource
- private MinioTemplate minioTemplate;
- @Resource
- private RedisTemplate<String, Object> redisTemplate;
-
- @Override
- public Result<String> checkFileExists(String md5) {
- Result<String> result = new Result<>();
- // 先从Redis中查询
- String url = (String) redisTemplate.boundHashOps(MD5_KEY).get(md5);
- // 文件不存在
- if (StrUtil.isEmpty(url)) {
- result.setSuccess(false);
- result.setMessage("资源不存在");
- } else {
- // 文件已经存在了
- result.setSuccess(true);
- result.setResult(url);
- result.setMessage("极速秒传成功");
- }
- return result;
- }
-
- @Override
- public Result<String> upload(String md5, MultipartFile file, String fileName, Integer index) {
- // 上传过程中出现异常
- Assert.notNull(file, "文件上传异常=>文件不能为空!");
- // 创建文件桶
- minioTemplate.makeBucket(md5);
- String objectName = String.valueOf(index);
- try {
- // 上传文件
- minioTemplate.putChunkObject(file.getInputStream(), md5, objectName);
- // 设置上传分片的状态
- return Result.ok("文件上传成功!");
- } catch (Exception e) {
- e.printStackTrace();
- return Result.error("文件上传失败!");
- }
- }
-
- @Override
- public Result<String> merge(MergeInfo mergeInfo) {
- Assert.notNull(mergeInfo, "mergeInfo不能为空!");
- String md5 = mergeInfo.getMd5();
- String fileType = mergeInfo.getFileType();
- try {
- // 开始合并请求
- String targetBucketName = minioConfig.getBucketName();
- String fileNameWithoutExtension = UUID.randomUUID().toString();
- String objectName = fileNameWithoutExtension + "." + fileType;
- // 合并文件
- minioTemplate.composeObject(md5, targetBucketName, objectName);
- log.info("桶:{} 中的分片文件,已经在桶:{},文件 {} 合并成功", md5, targetBucketName, objectName);
-
- // 合并成功之后删除对应的临时桶
- minioTemplate.removeBucket(md5, true);
- log.info("删除桶 {} 成功", md5);
-
- // 表示是同一个文件, 且文件后缀名没有被修改过
- String url = minioTemplate.getPresignedObjectUrl(targetBucketName, objectName);
-
- // 存入redis中
- redisTemplate.boundHashOps(MD5_KEY).put(md5, url);
-
- return Result.ok("文件合并成功");// 成功
- } catch (Exception e) {
- log.error("文件合并执行异常 => ", e);
- return Result.error("文件合并异常");// 失败
- }
- }
- }
- @Data
- @ApiModel(description = "大文件合并信息")
- public class MergeInfo implements Serializable {
- @ApiModelProperty(value = "文件的md5")
- public String md5;
- @ApiModelProperty(value = "文件名")
- public String fileName;
- @ApiModelProperty(value = "文件类型")
- public String fileType;
- }
- @Data
- @NoArgsConstructor
- @AllArgsConstructor
- public class OssFile {
- /**
- * OSS 存储时文件路径
- */
- private String ossFilePath;
- /**
- * 原始文件名
- */
- private String originalFileName;
- }
Copyright © 2003-2013 www.wpsshop.cn 版权所有,并保留所有权利。