【Hadoop-HDFS-Java】用Java代码对HDFS进行增删改查等操作_基于hadoop文件 增删改查

package cn.itcast;

import com.google.inject.internal.cglib.core.$LocalVariablesSorter;
import com.sun.jndi.toolkit.url.Uri;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.hdfs.server.namenode.EditLogInputException;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet;
import org.junit.Test;
import sun.security.krb5.Config;

import javax.swing.tree.ExpandVetoException;
import java.net.MalformedURLException;
import java.net.URI;
import java.net.URISyntaxException;

public class HdfsDemo {
    @Test
    //获取文件
    public void demo01() throws Exception {
        FileSystem fileSystem = FileSystem.newInstance(new URI("hdfs://node1:8020"), new Configuration());
        RemoteIterator<LocatedFileStatus> listFiles = fileSystem.listFiles(new Path("/"), true);
        while(listFiles.hasNext()){
            LocatedFileStatus next = listFiles.next();
            Path path = next.getPath();
            String name = path.getName();
            System.out.println("文件名:"+name+"; 文件路径:"+path);
        }
    }

    @Test
    //创建目录
    public void demo03() throws Exception {
        FileSystem fileSystem = FileSystem.get(new URI("hdfs://node1:8020"), new Configuration());
        fileSystem.mkdirs(new Path("/aaa/bbb"));
        fileSystem.close();
    }

    @Test
    //复制文件
    public void demo04() throws Exception {
        FileSystem fileSystem = FileSystem.newInstance(new URI("hdfs://node1:8020"), new Configuration());
        fileSystem.copyToLocalFile(new Path("/aaa/a.txt") , new Path("D:\\WorkSpace\\uTorrent_download\\hadoop\\day03_HDFS分布式文件系统01"));
        fileSystem.close();
    }

    @Test
    //复制hdfs文件到本地
    public void demo07() throws Exception {
        FileSystem fileSystem = FileSystem.newInstance(new URI("hdfs://node1:8020"), new Configuration());
        fileSystem.copyToLocalFile(new Path("/aaa/a.txt") , new Path("D:\\WorkSpace\\uTorrent_download\\hadoop\\day03_HDFS分布式文件系统02\\上午代码"));
        fileSystem.close();
    }

    @Test
    //复制本地文件到hdfs
    public void demo08() throws Exception {
        FileSystem fileSystem = FileSystem.newInstance(new URI("hdfs://node1:8020"), new Configuration());
        fileSystem.copyFromLocalFile(new Path("D:\\WorkSpace\\uTorrent_download\\hadoop\\day03_HDFS分布式文件系统02\\上午代码\\a.txt") , new Path("/bbb/b.txt"));
        fileSystem.close();
    }

    @Test
    //读取hdfs文件
    public void demo09() throws Exception {
        FileSystem fileSystem = FileSystem.newInstance(new URI("hdfs://node1:8020"), new Configuration());
        FSDataOutputStream outputStream = fileSystem.create(new Path("/aaa/sum.xml"));
        LocalFileSystem localFileSystem = FileSystem.getLocal(new Configuration());
        RemoteIterator<LocatedFileStatus> listFiles = localFileSystem.listFiles(new Path("D:\\WorkSpace\\uTorrent_download\\hadoop\\day03_HDFS分布式文件系统02\\资料\\上传小文件合并"), false);
        while(listFiles.hasNext()){
            LocatedFileStatus next = listFiles.next();
            Path path = next.getPath();
            FSDataInputStream inputStream = localFileSystem.open(path);
            int len ;
            byte[] bytes = new byte[1024];
            while((len = inputStream.read(bytes)) != -1){
                outputStream.write(bytes , 0 , len);
                outputStream.flush();
            }
            inputStream.close();
        }
        outputStream.close();
        localFileSystem.close();
        fileSystem.close();
    }

    @Test
    //删除hdfs文件
    public void demo14() throws Exception {
        FileSystem fileSystem = FileSystem.newInstance(new URI("hdfs://node1:8020"), new Configuration());
        fileSystem.delete(new Path("/lianxi"));
        fileSystem.close();
    }

    @Test
    public void demo15() throws Exception {
        FileSystem fileSystem = FileSystem.get(new URI("hdfs://node1:8020"), new Configuration());
        FSDataOutputStream outputStream = fileSystem.create(new Path("/sum/sum.xml"));
        LocalFileSystem localFileSystem = FileSystem.getLocal(new Configuration());
        RemoteIterator<LocatedFileStatus> listFiles = localFileSystem.listFiles(new Path("D:\\WorkSpace\\uTorrent_download\\hadoop\\day03_HDFS分布式文件系统02\\资料\\上传小文件合并"), false);
        while(listFiles.hasNext()){
            Path path = listFiles.next().getPath();
            FSDataInputStream inputStream = localFileSystem.open(path);
            int len ;
            byte[] bytes = new byte[1024];
            while((len = inputStream.read(bytes)) != -1){
                outputStream.write(bytes , 0 , len);
            }
            inputStream.close();
        }
        localFileSystem.close();
        outputStream.close();
        fileSystem.close();
    }
}
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115