Hadoop HDFS 客户端测试

依赖

<dependencies>
    <dependency>
        <groupId>junit</groupId>
        <artifactId>junit</artifactId>
        <version>RELEASE</version>
    </dependency>
    <dependency>
        <groupId>org.apache.logging.log4j</groupId>
        <artifactId>log4j-core</artifactId>
        <version>2.8.2</version>
    </dependency>
    <dependency>
        <groupId>org.apache.hadoop</groupId>
        <artifactId>hadoop-common</artifactId>
        <version>2.7.2</version>
    </dependency>
    <dependency>
        <groupId>org.apache.hadoop</groupId>
        <artifactId>hadoop-client</artifactId>
        <version>2.7.2</version>
    </dependency>
    <dependency>
        <groupId>org.apache.hadoop</groupId>
        <artifactId>hadoop-hdfs</artifactId>
        <version>2.7.2</version>
    </dependency>
</dependencies>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27

测试demo1

代码

package com.yljphp.hdfs;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;

import java.io.IOException;

public class Demo {

    public static void main(String[] args) throws IOException {

        Configuration configuration = new Configuration();

        configuration.set("fs.defaultFS", "hdfs://hadoop102:9000");

        //获取HDFS客户端对象
        FileSystem fs = FileSystem.get(configuration);

        //HDFS上创建文件夹
        fs.mkdirs(new Path("/demo/0331"));

        //关闭资源
        fs.close();

        System.out.println("ok~~~");
    }
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29

配置VM options

客户端去操作 hdfs 时，是有一个用户身份的。默认情况下，hdfs 客户端 api 会从 jvm 中 获取一个参数来作为自己的用户身份:-DHADOOP_USER_NAME=root

如果不添加上面的配置，会报如下错误。

查看

测试demo2

代码

package com.yljphp.hdfs;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;

import java.io.IOException;
import java.net.URI;

public class Demo {

    public static void main(String[] args) throws Exception {

        Configuration configuration = new Configuration();

        configuration.set("fs.defaultFS", "hdfs://hadoop102:9000");

        //获取HDFS客户端对象
        //FileSystem fs = FileSystem.get(configuration);

        FileSystem fs = FileSystem.get(new URI("hdfs://hadoop102:9000"), configuration, "root");

        //HDFS上创建文件夹
        fs.mkdirs(new Path("/demo/0332"));

        //关闭资源
        fs.close();

        System.out.println("ok~~~");
    }
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32

查看