SpringBoot集成tobato的FastDFS测试(包含docker安装fdfs和dfs的客户端连接池使用)

• 启动tracker： docker run -d --name=tracker -v /root/data/fdfs/tracker:/data/fast_data --privileged=true --net=host morunchang/fastdfs sh tracker.sh

• 启动storage： docker run -d --name=storage -v /root/data/fdfs/storageData:/data/fast_data --privileged=true --net=host -e TRACKER_IP=192.168.15.133:22122 -e GROUP_NAME=group1 morunchang/fastdfs sh storage.sh

• pom引入包

<dependency>
    <groupId>com.github.tobato</groupId>
    <artifactId>fastdfs-client</artifactId>
    <version>1.26.5</version>
</dependency>
1
2
3
4
5
6

• 创建dfs配置

<!--引入包之后boot项目会有自动提示-->
fdfs:
    connect-timeout: 601
    so-timeout: 1501
    tracker-list[0]: 192.168.15.133:22122
1
2
3
4
5

• 创建测试controller

/**
 * @author : LiuMingyao
 * @date : 2019/8/22 14:38
 * @description : TODO
 */
@Api("fastDfs测试接口")
@RestController
public class TestDfsController {
	//tobato的包会自动注入spring.yml中关于dfs的配置，简单使用只需要注入FastFileStorageClient即可
	//连接池和tracker可以自行选择是否需要注入使用
    @Autowired
    private FastFileStorageClient storageClient;
    //自选
    @Autowired
    private TrackerClient trackerClient;
    @Autowired
    protected FdfsConnectionPool pool;

    @ApiOperation("图片文件上传测试")
    @ApiImplicitParams({@ApiImplicitParam(name = "localPath", value = "本地需要上传的文件路径", required = true, paramType = "query", dataType = "String")})
    @GetMapping("/liuUpload")
    public String liuUpload(String localPath) throws Exception {
        File file = new File(localPath);
        Set<MetaData> metaDataSet = new HashSet<>();
        metaDataSet.add(new MetaData("Author", "Author"));
        metaDataSet.add(new MetaData("date", "当前时间"));
        FileInputStream inputStream = new FileInputStream(file);
        FastImageFile fastImageFile = new FastImageFile(inputStream, file.length(), "png", metaDataSet);
        StorePath storePath = storageClient.uploadImage(fastImageFile);
        return storePath.getFullPath();
    }


    @ApiOperation("图片文件下载测试")
    @ApiImplicitParams({@ApiImplicitParam(name = "groupName", value = "需要下载的文件所属组", required = true, paramType = "query", dataType = "String")
            , @ApiImplicitParam(name = "path", value = "需要下载的文件path", required = true, paramType = "query", dataType = "String")})
    @GetMapping("/liuDownload")//group1/M00/00/00/wKgPhV1eTUuAYfcmAAARSoVDk74492.png
    public String liuDownload(String groupName, String path) throws Exception {
        Object o = storageClient.downloadFile(groupName, path, new DownloadCallback<Object>() {
            @Override
            public Object recv(InputStream inputStream) throws IOException {
                System.out.println("接受到文件");
                byte[] bytes = inputStream.readAllBytes();
                FileImageOutputStream outputStream=new FileImageOutputStream(new File("d:/aaa.png"));
                outputStream.write(bytes);
                return "d:/aaa.png";
            }
        });
        return o.toString();
    }

    @ApiOperation("文件上传测试")
    @ApiImplicitParams({
            })
    @PostMapping("/upload")
    public String uploadFile(@RequestParam("file") MultipartFile file) throws IOException {
        Set<MetaData> metaDataSet = new HashSet<>();
        metaDataSet.add(new MetaData("a", "b"));
        metaDataSet.add(new MetaData("date", "时间"));
        StorePath path = storageClient.uploadFile(file.getInputStream(), file.getSize(), FilenameUtils.getExtension(file.getOriginalFilename()),
                metaDataSet);
        return path.getFullPath();
    }

    @ApiOperation("缩略图")
    @PostMapping("img")
    public String uploadImageAndCrtThumbImage(@RequestParam("file") MultipartFile file) throws IOException {
        Set<MetaData> metaDataSet = new HashSet<>();
        metaDataSet.add(new MetaData("a", "b"));
        metaDataSet.add(new MetaData("date", "时间"));
        StorePath path = storageClient.uploadImageAndCrtThumbImage(file.getInputStream(), file.getSize(), FilenameUtils.getExtension(file.getOriginalFilename()),
                metaDataSet);
        return path.getFullPath();
    }
}
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75

---

注意：需要使用swagger测试的需要导入swagger的包:

<!--api测试包-->
<dependency>
    <groupId>io.springfox</groupId>
    <artifactId>springfox-swagger2</artifactId>
    <version>2.6.1</version>
</dependency>

<dependency>
    <groupId>io.springfox</groupId>
    <artifactId>springfox-swagger-ui</artifactId>
    <version>2.6.1</version>
</dependency>

还要自行配置swagger：

@Configuration
@EnableSwagger2
public class SwaggerConfig {

@Bean
public Docket createRestApi() {
    return new Docket(DocumentationType.SWAGGER_2)
            .apiInfo(apiInfo())
            .select()
            .apis(RequestHandlerSelectors.basePackage("com.example.study"))
            .paths(PathSelectors.any())
            .build();
    }
    
    private ApiInfo apiInfo() {
    return new ApiInfoBuilder()
            .title("Spring Boot中使用Swagger2构建RESTful APIs")
            .description("我的CSDN:https://mp.csdn.net/postedit/95599118")
            .termsOfServiceUrl("GitHub:https://github.com/MrLawrenc")
            .contact("指是弹琴")
            .version("1.0")
            .build();
    }
}
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39

• fastdfs的连接池配置:

fdfs:
  connect-timeout: 601
  so-timeout: 1501
  tracker-list[0]: 192.168.15.133:22122
#  tracker-list[1]: 192.168.15.132:22122
  thumb-image: # 缩略图
    width: 60
    height: 60
  #    连接池配置
  pool:
    #从池中借出的对象的最大数目
    max-total: 5
    #获取连接时的最大等待毫秒数100
    max-wait-millis: 2000
    jmx-name-prefix: liumingyao
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15

• 此时并发获取clien连接的时候是从线程池拿的，下面验证

    @Autowired
    private FastFileStorageClient storageClient;
    //在controller里面注入池对象
    @Autowired
    protected FdfsConnectionPool pool;
    
    
     @ApiOperation("300m的文件上传压力测试")
    @GetMapping("/pullTest")
    public void testPoll() throws Exception {
        File file = new File("d:/aaa.png");
        final AtomicInteger failCount = new AtomicInteger(0);
        final AtomicInteger count = new AtomicInteger(0);
        int totalCount = 20;
        ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor();
        executor.setCorePoolSize(5);
        executor.afterPropertiesSet();
        log.info("pool.getMaxTotal()" + pool.getMaxTotal());
        log.info("pool.getMaxWaitMillis()" + pool.getMaxWaitMillis());
        for (int i = 0; i < totalCount; i++) {
            executor.execute(() -> {
                try {
                    byte[] bytes = FileUtils.readFileToByteArray(new File("e:/ggg.7z"));//300m+

                    log.info("活动连接{}", pool.getNumActive());
                    log.info("空闲连接{}", pool.getNumIdle());
                    log.info("连接获取总数统计{}", pool.getBorrowedCount());
                    log.info("连接返回总数统计{}", pool.getReturnedCount());
                    log.info("连接销毁总数统计{}", pool.getDestroyedCount());

                    //第一个是参数group
                    StorePath storePath = storageClient.uploadFile(null, new ByteArrayInputStream(bytes),
                            bytes.length, "7z");
                    log.info("{}  storePath  {}", Thread.currentThread().getName(), storePath);
                } catch (Exception e) {
                    e.printStackTrace();
                    failCount.incrementAndGet();
                } finally {
                    count.incrementAndGet();
                }
            });

        }
        //等待线程执行完主线程再放行销毁连接池
        while (count.get() < totalCount) {
            try {
                Thread.sleep(100);
            } catch (InterruptedException e) {
                return;
            }
        }
        executor.destroy();
        log.info("total count: {}", count.get());
        log.info("fail count: {}", failCount.get());
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54

进入http://localhost/swagger-ui.htm（端口自行配置的）访问该请求,观察日志输出，可以看见连接获取各种的情况。

注意：如果出现从报错->从池中获取对象异常失败，可以适当增大等待时间或是增大连接池对象的数量。

• 另外如何需要自行设置连接池的对象数量，可以自动注入pool之后设置值，如果需要tracker，也可以自动注入

   @Autowired
    private TrackerClient trackerClient;

Tracker对象有如下方法:
    StorageNode getStoreStorage();

    StorageNode getStoreStorage(String var1);

    StorageNodeInfo getFetchStorage(String var1, String var2);

    StorageNodeInfo getUpdateStorage(String var1, String var2);

    List<GroupState> listGroups();

    List<StorageState> listStorages(String var1);

    List<StorageState> listStorages(String var1, String var2);

    void deleteStorage(String var1, String var2);
    
池对象设置值:
        @Autowired
        protected FdfsConnectionPool pool;
        
        pool.setMaxTotal(100);
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25

更多方法参见github原作者的单元测试: 原作者的github

出现报错情况可以自行看看，其实作者封装的fastDFS大部分代码都能看懂，而且单元测试基本涵盖了大部分用法