JAVA 整合 AWS S3(Amazon Simple Storage Service)文件上传,分片上传,删除,下载_aws-java-sdk-s3

依赖

因为aws需要发送请求上传、下载等api，所以需要加上httpclient相关的依赖

<dependency>
 <groupId>com.amazonaws</groupId>
  <artifactId>aws-java-sdk-s3</artifactId>
  <version>1.11.628</version>
</dependency>

<dependency>
  <groupId>org.apache.httpcomponents</groupId>
  <artifactId>httpclient</artifactId>
  <version>4.5.13</version>
</dependency>

<dependency>
  <groupId>org.apache.httpcomponents</groupId>
  <artifactId>httpcore</artifactId>
  <version>4.4.13</version>
</dependency>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17

工具类

上传和下载方式都被重载了，所以可以根据不同的业务场景去使用不同的重载方式。

import com.amazonaws.AmazonServiceException;
import com.amazonaws.ClientConfiguration;
import com.amazonaws.Protocol;
import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.auth.AWSStaticCredentialsProvider;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.client.builder.AwsClientBuilder;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
import com.amazonaws.services.s3.model.*;
import com.amazonaws.services.s3.transfer.TransferManager;
import com.amazonaws.services.s3.transfer.TransferManagerBuilder;
import com.amazonaws.services.s3.transfer.Upload;
import com.amazonaws.util.StringUtils;
import com.mocha.order.enums.PropertiesEnum;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Component;

import javax.annotation.PostConstruct;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.InputStream;
import java.net.URL;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.stream.Collectors;


@Component
public class CosUtil {
    private static final Logger LOGGER = LoggerFactory.getLogger(CosUtil.class);

    //访问key
    public static String accessKey;
    //密钥key
    public static String secretKey;

    //服务区域
    public static String serviceEndpoint; // e.g., "cos.gz-tst.cos.tg.unicom.local"
    //区域
    public static String region; // e.g., "gz-tst"

    static AWSCredentials credentials ;
    static AWSStaticCredentialsProvider awsStaticCredentialsProvider ;
    static ClientConfiguration config ;
    static AwsClientBuilder.EndpointConfiguration endpointConfiguration ;
    static AmazonS3 conn ;

    //分片大小5M,一般设置5的倍数即可，分片大小最大不要超过100M
    public static final int  FIVE_PARTSIZE=5242880;

    //1M
    public static final int  ONE_PARTSIZE=1048576;


    @PostConstruct
    public void init() {
    	//这些配置的值，可以从数据库获取、也可以丛配置文件获取
        final String cosAccessKey = PropertiesEnum.getCosAccessKey();
        final String cosSecretKey = PropertiesEnum.getCosSecretKey();
        final String cosServiceEndpoint = PropertiesEnum.getCosServiceEndpoint();
        final String cosRegion = PropertiesEnum.getCosRegion();

        if (org.apache.commons.lang3.StringUtils.isBlank(cosAccessKey)){
            LOGGER.error("联通云Cos配置的AccessKey为空，请保证【COS_ACCESSKEY】值不为空，否则会影响项目相关功能的使用");
        }

        if (org.apache.commons.lang3.StringUtils.isBlank(cosSecretKey)){
            LOGGER.error("联通云Cos配置的SecretKey为空，请保证【COS_SECRETKEY】值不为空，否则会影响项目相关功能的使用");
        }

        if (org.apache.commons.lang3.StringUtils.isBlank(cosServiceEndpoint)){
            LOGGER.error("联通云Cos配置的ServiceEndpoint为空，请保证【COS_SERVICEENDPOINT】值不为空，否则会影响项目相关功能的使用");
        }

        if (org.apache.commons.lang3.StringUtils.isBlank(cosRegion)){
            LOGGER.error("联通云Cos配置的Region为空，请保证【COS_REGION】值不为空，否则会影响项目相关功能的使用");
        }

        LOGGER.info("联通云Cos的配置分别为，accessKey：{}，secretKey：{}，serviceEndpoint：{}，region：{}",cosAccessKey,cosAccessKey,cosServiceEndpoint,cosRegion);

        LOGGER.info("开始初始化联通云Cos的配置，开始初始化时间：{}",new Date());

        accessKey=cosAccessKey;
        secretKey=cosSecretKey;
        serviceEndpoint=cosServiceEndpoint;
        region=cosRegion;

        LOGGER.info("结束初始化联通云Cos的配置，结束初始化时间：{}",new Date());

        credentials = new BasicAWSCredentials(accessKey, secretKey);
        awsStaticCredentialsProvider = new AWSStaticCredentialsProvider(credentials);
        config = new ClientConfiguration();
        endpointConfiguration = new AwsClientBuilder.EndpointConfiguration(serviceEndpoint, region);
        conn = AmazonS3ClientBuilder.standard()
                .withCredentials(awsStaticCredentialsProvider)
                .withClientConfiguration(config.withProtocol(Protocol.HTTP).withSignerOverride("S3SignerType"))
                .withEndpointConfiguration(endpointConfiguration).build();
    }

    //检查桶是否存在
    public static boolean doesBucketExist(String bucketName){
        return conn.listBuckets().stream().map(Bucket::getName).collect(Collectors.toList()).contains(bucketName);
    }
    /**
     * 创建桶
     * 1、创建已经存在的桶，不会把之前存在的桶内容删除
     *
     */
    public static Bucket createBucket(String bucketName){
        LOGGER.info("创建桶：{}",bucketName);
        return conn.createBucket(bucketName);
    }

    // 列出所有桶列表
    public static List<Bucket> listBuckets(){
        return conn.listBuckets();
    }


    // 列出桶内对象
    public static ObjectListing listObjects(String bucketName){
        ObjectListing objects = conn.listObjects(bucketName);
        do {
            for (S3ObjectSummary objectSummary : objects.getObjectSummaries()) {
                LOGGER.info(objectSummary.getKey() + "\t" + objectSummary.getSize() + "\t" + StringUtils.fromDate(objectSummary.getLastModified()));
            }
            objects = conn.listNextBatchOfObjects(objects);
        } while (objects.isTruncated());
        return objects;
    }

    //  获取文件
    public static ObjectMetadata  getObjectMeta(String bucketName, String fileName){
        LOGGER.info("上传文件，桶名称：{}，文件名称：{}",bucketName,fileName);
        return conn.getObjectMetadata(bucketName, fileName);
    }

    // 上传文件-通过实体类
    public static PutObjectResult uploadFile(PutObjectRequest putObjectRequest){
        LOGGER.info("上传文件桶实体类方式，实体类是：{}",putObjectRequest);
        return conn.putObject(putObjectRequest);
    }

    // 上传文件-通过File
    public static PutObjectResult uploadFile(String bucketName, String keyName, File file){
        LOGGER.info("上传文件，桶名称：{}，文件名称：{}",bucketName,keyName);
        return conn.putObject(bucketName, keyName, file);
    }

    // 上传文件-通过InputStream
    public static PutObjectResult uploadFile(String bucketName, String keyName, InputStream inputStream){
        LOGGER.info("上传文件，桶名称：{}，文件名称：{}",bucketName,keyName);
        return conn.putObject(bucketName, keyName, inputStream, new ObjectMetadata());
    }

    // 上传文件-通过字符串
    public static PutObjectResult uploadFile(String bucketName, String keyName, String content){
        LOGGER.info("上传文件，桶名称：{}，文件名称：{}",bucketName,keyName);
        return conn.putObject(bucketName, keyName, content);
    }


    // 分片上传(不要需要指定分片的大小、还提供上传进度跟踪、断点续传、并发上传等功能)
    public static Upload uploadFileByShard(String bucketName, String keyName, String filePath) throws InterruptedException,AmazonServiceException {
        TransferManager tm = TransferManagerBuilder.standard().withS3Client(conn).build();
        try {
            LOGGER.info("开始分片上传,桶名称：{}，文件名称：{}，路径：{}",bucketName,keyName,filePath);
            Upload upload = tm.upload(bucketName, keyName, new File(filePath));
            upload.waitForCompletion();
            LOGGER.info("结束分片上传,桶名称：{}，文件名称：{}，路径：{}",bucketName,keyName,filePath);
            return upload;
        } catch (AmazonServiceException | InterruptedException e) {
            LOGGER.error("分片上传异常,桶名称：{}，文件名称：{}，路径：{}",bucketName,keyName,filePath);
            e.printStackTrace();
            throw e;
        }
    }


    /**
     * 大文件分段上传（需要指定分片的大小，自定义的线程池）
     *
     * @param bucketName  bucketName
     * @param objectName  objectName
     * @param file        MultipartFile
     * @param minPartSize 每片大小，单位：字节（eg：5242880 <- 5m）
     * @return
     */
    public static boolean uploadMultipartFileByPart(String bucketName, String objectName, File file, int minPartSize) {

        long size = file.length();

        final String fileName = file.getName();

        if (size==0) {
            LOGGER.error("分片上传的文件：{}为空",fileName);
            return false;
        }
        // 计算分片大小

        // 得到总共的段数，和 分段后，每个段的开始上传的字节位置
        List<Long> positions = Collections.synchronizedList(new ArrayList<>());
        long filePosition = 0;
        while (filePosition < size) {
            positions.add(filePosition);
            filePosition += Math.min(minPartSize, (size - filePosition));
        }
        if (LOGGER.isDebugEnabled()) {
            LOGGER.info("文件：{}，总大小：{}字节，分为{}段",fileName, size, positions.size());
        }

        // 创建一个列表保存所有分传的 PartETag, 在分段完成后会用到
        List<PartETag> partETags = Collections.synchronizedList(new ArrayList<>());

        // 第一步，初始化，声明下面将有一个 Multipart Upload
        // 设置文件类型
        ObjectMetadata metadata = new ObjectMetadata();

        String fileType = fileName.substring(fileName.lastIndexOf(".") + 1);
        metadata.setContentType(fileType);

        InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest(bucketName, objectName, metadata);
        InitiateMultipartUploadResult initResponse = conn.initiateMultipartUpload(initRequest);
        if (LOGGER.isDebugEnabled()) {
            LOGGER.info("分片上传开始，上传的文件时：{}",fileName);
        }
        //声明线程池
        ExecutorService exec = Executors.newFixedThreadPool(5);
        long begin = System.currentTimeMillis();
        try {
            for (int i = 0; i < positions.size(); i++) {
                int finalI = i;
                exec.execute(() -> {
                    long time1 = System.currentTimeMillis();
                    UploadPartRequest uploadRequest = new UploadPartRequest()
                            .withBucketName(bucketName)
                            .withKey(objectName)
                            .withUploadId(initResponse.getUploadId())
                            .withPartNumber(finalI + 1)
                            .withFileOffset(positions.get(finalI))
                            .withFile(file)
                            .withPartSize(Math.min(minPartSize, (size - positions.get(finalI))));
                    // 第二步，上传分段，并把当前段的 PartETag 放到列表中
                    partETags.add(conn.uploadPart(uploadRequest).getPartETag());
                    LOGGER.info("分片上传的文件时：{}，第{}段上传耗时：{}",fileName ,finalI + 1, (System.currentTimeMillis() - time1));
                });
            }
            //任务结束关闭线程池
            exec.shutdown();
            //判断线程池是否结束，不加会直接结束方法
            while (true) {
                if (exec.isTerminated()) {
                    break;
                }
            }

            // 第三步，完成上传，合并分段
            CompleteMultipartUploadRequest compRequest = new CompleteMultipartUploadRequest(bucketName, objectName, initResponse.getUploadId(), partETags);
            conn.completeMultipartUpload(compRequest);

        } catch (Exception e) {
            conn.abortMultipartUpload(new AbortMultipartUploadRequest(bucketName, objectName, initResponse.getUploadId()));
            LOGGER.error("分片上传文件:{}异常，异常是：{}, ",fileName, e.getLocalizedMessage());
            e.printStackTrace();
        }
        LOGGER.info("分片上传文件：{}结束，总上传耗时：{}毫秒",fileName, (System.currentTimeMillis() - begin));

        return false;
    }

    /**
     * 初始化，声明有一个Multipart Upload
     *
     * @param initRequest 初始化请求
     * @return 初始化返回
     */
    private InitiateMultipartUploadResult initiateMultipartUpload(
            InitiateMultipartUploadRequest initRequest) {
        return conn.initiateMultipartUpload(initRequest);
    }

    /**
     * 上传分段
     *
     * @param uploadRequest 上传请求
     * @return 上传分段返回
     * @see <a href="http://docs.aws.amazon.com/goto/WebAPI/s3-2006-03-01/UploadPart">AWS
     * API Documentation</a>
     */
    private UploadPartResult uploadPart(UploadPartRequest uploadRequest) {
        return conn.uploadPart(uploadRequest);
    }

    /**
     * 分段合并
     *
     * @param compRequest 合并请求
     * @see <a href="http://docs.aws.amazon.com/goto/WebAPI/s3-2006-03-01/CompleteMultipartUpload">AWS
     * API Documentation</a>
     */
    private CompleteMultipartUploadResult completeMultipartUpload(
            CompleteMultipartUploadRequest compRequest) {
        return conn.completeMultipartUpload(compRequest);
    }

    /**
     * 中止分片上传
     *
     * @param uploadRequest 中止文件上传请求
     * @see <a href="http://docs.aws.amazon.com/goto/WebAPI/s3-2006-03-01/AbortMultipartUpload">AWS
     * API Documentation</a>
     */
    private void abortMultipartUpload(AbortMultipartUploadRequest uploadRequest) {
        conn.abortMultipartUpload(uploadRequest);
    }


    // 修改对象的访问控制权限
    public static void modifyFileAccessAuthority(String bucketName, String keyName, CannedAccessControlList cannedAccessControlList){
        LOGGER.info("修改文件的访问权限,桶名称：{}，文件名称：{}，权限类型时：{}",bucketName,keyName,cannedAccessControlList.toString());
        conn.setObjectAcl(bucketName, keyName, cannedAccessControlList);
    }

    // 下载一个对象(到指定路径)
    public static ObjectMetadata downloadFileAssignPath(String bucketName, String keyName, String fileSavePath){
        return conn.getObject(new GetObjectRequest(bucketName, keyName), new File(fileSavePath));
    }

    // 生成对象下载链接（带签名）
    public static URL generatorFileUrl(String bucketName, String keyName){
        GeneratePresignedUrlRequest request = new GeneratePresignedUrlRequest(bucketName, keyName);
        return conn.generatePresignedUrl(request);
    }

    // 删除文件
    public static void deleteFie(String bucketName, String keyName){
        LOGGER.info("删除文件,桶名称：{}，文件名称：{}，删除时间：{}",bucketName,keyName,new Date());
        conn.deleteObject(bucketName, keyName);
    }

    // 删除桶
    public static void deleteFie(String bucketName){
        LOGGER.info("删除桶,桶名称：{}，删除时间：{}",bucketName,new Date());
        conn.deleteBucket(bucketName);
    }





    public static void main(String[] args) {

        String filePath = "G:\\mk\\GDrepo\\repo\\org\\apache\\httpcomponents\\httpcomponents-core\\4.0.1\\httpcomponents-core-4.0.1.pom"; //文件路径

//        ByteArrayInputStream input1 = new ByteArrayInputStream("Hello World!".getBytes());
//        CosUtil.createBucket("aaa");
//        final FileInputStream inputStream;
//        try {
//            inputStream = new FileInputStream("G:\\mk\\GDrepo\\repo\\org\\apache\\httpcomponents\\httpcomponents-core\\4.0.1\\httpcomponents-core-4.0.1.pom");
//        } catch (FileNotFoundException e) {
//            throw new RuntimeException(e);
//        }
//        CosUtil.uploadFile("aaa","a.pom",inputStream);

        CosUtil.uploadFile("test","a.txt",new File(filePath));
        CosUtil.uploadFile("test","a.txt",new File(filePath));


        //上传空文件夹
        conn.putObject("aaa", "demo" + "/","");
        //上传文件到指定的文件夹
        final PutObjectRequest putObjectRequest = new PutObjectRequest("aaa", "demo" + "/" + "a.txt", new File(filePath));
        conn.putObject(putObjectRequest);


        // 列出所有桶列表
        List<Bucket> buckets = conn.listBuckets();
        for (Bucket bucket : buckets) {
            System.out.println(bucket.getName() + "\t" + StringUtils.fromDate(bucket.getCreationDate()));
        }

        // 创建桶
        Bucket bucket = conn.createBucket("111");
        Bucket bucket1 = conn.createBucket("111");
        Bucket bucket12 = conn.createBucket("111");
        System.out.println(bucket.getName());

        // 列出桶内对象
        ObjectListing objects = conn.listObjects(bucket.getName());
        do {
            for (S3ObjectSummary objectSummary : objects.getObjectSummaries()) {
                System.out.println(objectSummary.getKey() + "\t" +
                        objectSummary.getSize() + "\t" +
                        StringUtils.fromDate(objectSummary.getLastModified()));
            }
            objects = conn.listNextBatchOfObjects(objects);
        } while (objects.isTruncated());

        // 创建对象
        ByteArrayInputStream input = new ByteArrayInputStream("Hello World!".getBytes());
        conn.putObject(bucket.getName(), "hello.txt", input, new ObjectMetadata());

        //分段上传
        String keyName = "demoya"; //文件上传成功后的文件名

        TransferManager tm = TransferManagerBuilder.standard().withS3Client(conn).build();

        try {
            // TransferManager processes all transfers asynchronously,
            // so this call returns immediately.
            System.out.println("开始分片上传");
            Upload upload = tm.upload("my-test-bucket", keyName, new File(filePath));

            //Optionally, wait for the upload to finish before continuing.
            upload.waitForCompletion();
            System.out.println("结束分片上传");
        } catch (AmazonServiceException | InterruptedException e) {
            // The call was transmitted successfully, but Amazon S3 couldn't process
            // it, so it returned an error response.
            e.printStackTrace();
        }

        // 修改对象的访问控制权限
        conn.setObjectAcl(bucket.getName(), "hello.txt", CannedAccessControlList.PublicRead);

        // 下载一个对象(到本地文件)
        conn.getObject(new GetObjectRequest(bucket.getName(), "hello.txt"), new File("G:\\mk\\hello.txt"));

        // 生成对象下载链接（带签名）
        GeneratePresignedUrlRequest request = new GeneratePresignedUrlRequest(bucket.getName(), "hello.txt");
        System.out.println(conn.generatePresignedUrl(request));

        // 删除一个对象
        conn.deleteObject(bucket.getName(), "hello.txt");

        //删除桶
        conn.deleteBucket(bucket.getName());

    }
}


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448