【简单易懂版】使用IDEA操作Hadoop（增删改查）_idea上编写hadoop程序

---

前提：服务器中已经配置好了hadoop
本人亲测，以下代码已经跑通，基础功能都可以完成！！！希望对大家有用！！！

一、引入hdfs依赖

1. 创建一个maven项目cn.et
2. 本地maven配置阿里镜像，用于快速下载依赖（重启加载）
3. pow文件中引入hadoop依赖

		<!--	hdfs依赖	-->
		<dependency>
			<groupId>org.apache.hadoop</groupId>
			<artifactId>hadoop-client</artifactId>
			<version>3.1.3</version>
		</dependency>
1
2
3
4
5
6

二、创建hdfs工具类

创建一个HdfsApiUtils 类，用于实现hdfs的增删改查：

1. 获取hdfs的代码需要封装到静态代码块(先执行而且只执行一次)
2. 创建文件或目录（mkdirs，建议写绝对路径hdfs://地址:9000/新目录）
3. 删除文件或目录（delete）
4. 修改或移动文件或目录（rename）
5. 查询当前路径下所有的文件或目录（显示时间和MB）

package com.example.springbootonline.utils;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.stereotype.Component;

import java.io.IOException;
import java.net.URI;
import java.util.List;


/**
 * Created with IntelliJ IDEA.
 *
 * @Author: Print
 * @Date: 2023/07/17/10:07
 * @Description:idea集成hdfs；地址hdfs://地址:9000
 */

@Component
public class HdfsApiUtils {
    private static String hdfsUrl = "hdfs://地址:9000";


    private static String hdfsUsername = "root";

    private static FileSystem hdfs;

    static {
        Configuration conf = new Configuration();
        // 上传到云服务器需要配置下面这个句话
        conf.set("dfs.client.use.datanode.hostname","true");
        try {
            hdfs = FileSystem.get(URI.create(hdfsUrl), conf, hdfsUsername);
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
    }

    /**
     * 创建文件或目录
     */
    public boolean mkdir(String path) {
        boolean res = false;
        try {
            hdfs.mkdirs(new Path(path));
            res = true;
        } catch (IOException e) {
            throw new RuntimeException(e);
        }
        return res;
    }

    /**
     * 删除文件或目录
     */
    public boolean delete(String path) {
        boolean res = false;
        try {
            res = hdfs.delete(new Path(path), true);
        } catch (IOException e) {
            throw new RuntimeException(e);
        }
        return res;
    }

    /**
     * 修改或移动文件或目录
     */
    public boolean rename(String oldFile, String newFlie) {
        boolean res = false;
        try {
            res = hdfs.rename(new Path(oldFile), new Path(newFlie));
        } catch (IOException e) {
            throw new RuntimeException(e);
        }
        return res;
    }

    /**
     * 查询当前路径下所有的文件或目录（只查当前目录下）
     */
    public FileStatus[] findCurrent(String path) {
        FileStatus[] res = null;
        try {
            res = hdfs.listStatus(new Path(path));
        } catch (IOException e) {
            throw new RuntimeException(e);
        }
        return res;
    }

    /**
     * 查询当前路径下所有的文件或目录（递归查下面所有）
     */
    public RemoteIterator<LocatedFileStatus> findAll(String path){
        RemoteIterator<LocatedFileStatus> iterator = null;
        try {
            iterator = hdfs.listFiles(new Path(path),true);
        } catch (IOException e) {
            throw new RuntimeException(e);
        }
        return iterator;
    }

    /**
     * 上传
     */
    public boolean upload(String localPath, String path) {
        boolean res = false;
        try {
            hdfs.copyFromLocalFile(new Path(localPath), new Path(path));
            res = true;
        } catch (IOException e) {
            res = false;
            throw new RuntimeException(e);
        }
        return res;
    }

    /**
     * 下载
     */
    public boolean download(String hdfsPath, String localPath) {
        boolean res = false;
        try {
            hdfs.copyToLocalFile(new Path(hdfsPath), new Path(localPath));
            res = true;
        } catch (IOException e) {
            throw new RuntimeException(e);
        }
        return res;
    }
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137

三、测试hdfs工具类

import com.example.springbootonline.utils.HdfsApiUtils;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.RemoteIterator;
import org.junit.jupiter.api.Test;
import org.springframework.stereotype.Component;

import java.io.IOException;

/**
 * Created with IntelliJ IDEA.
 *
 * @Author: Print
 * @Date: 2023/07/17/10:59
 * @Description:
 */
@Component
public class HdfsApiUtilsTest {
    HdfsApiUtils hdfsApiUtils = new HdfsApiUtils();

    @Test
    public void mkdir(){
        String newFile = "/file";
        System.out.println(hdfsApiUtils.mkdir(newFile));
    }

    @Test
    public void delete(){
        String path = "/aaa";
        System.out.println(hdfsApiUtils.delete(path));
    }

    @Test
    public void rename(){
        String oldFile = "/aaa",newFile = "/newAAA";
        System.out.println(hdfsApiUtils.rename(oldFile,newFile));
    }

    @Test
    public void upload(){
        String localPath = "F:\\Users\\HP\\Videos\\Captures\\demo.mp4",path = "/abc/aaa";
        System.out.println(hdfsApiUtils.upload(localPath,path));
    }

    @Test
    public void findCurrent(){
        String path = "/file";
        FileStatus[] fss = hdfsApiUtils.findCurrent(path);
        for (FileStatus fs:fss) {
            System.out.println(fs.toString()+"\n");
        }
        System.out.println();
    }

    @Test
    public void findAll() throws IOException {
        String path = "/file";
        RemoteIterator<LocatedFileStatus> iterator = hdfsApiUtils.findAll(path);
        while (iterator.hasNext()){
            System.out.println(iterator.next().toString());
        }
    }
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65

反思

好像应该再写一个服务器如何配置hadoop，后面再看有没有时间吧