windows下操作HDFS_libhdfs windows

windows上部署hadoop包

1：部署win版本hadoop包
2：整和lib包（包括common,hdfs,mapreduce,yarn,tools（lib,.））下所有的jar包
3：将windows版本hadoop/bin/hadoop.dll 放到c:/windows/system32下

windows环境变量配置

hadoop的bin和sbin目录放PATH中
HADOOP_HOME+HADOOP_USER_NAME=root

HADOOP_HOME是自己hadoop包的位置

idea-java操作HDFS

将之前整合的jar包，添加到项目中图示lib文件夹下。将linux集群配置的core-site.xml和hdfs-site.xml文件拷贝到resources文件夹下。

配置pom.xml文件

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>

    <groupId>com.hsd.counter</groupId>
    <artifactId>hdfs-api-exise</artifactId>
    <version>1.0-SNAPSHOT</version>
    <!-- 添加下述文件 -->
    <properties>
        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
        <hadoop.version>2.6.5</hadoop.version>
    </properties>
    <dependencies>
        <dependency>
            <groupId>junit</groupId>
            <artifactId>junit</artifactId>
            <version>4.12</version>
        </dependency>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-client</artifactId>
            <version>${hadoop.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-common</artifactId>
            <version>${hadoop.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-hdfs</artifactId>
            <version>${hadoop.version}</version>
        </dependency>
        <dependency>
            <groupId>junit</groupId>
            <artifactId>junit</artifactId>
            <version>4.12</version>
        </dependency>
    </dependencies>
</project>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42

package com.xpu.hdfs.test;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.io.IOUtils;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import sun.nio.ch.IOUtil;

import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;

public class HdfsTest {
    private Configuration conf;
    private FileSystem fileSystem;
    @Before
    public void before() throws Exception {
//        System.out.println("before");
        conf = new Configuration(true);
        //设置block大小为1M，上传文件使用
        conf.set("dfs.blocksize","1048576");
        fileSystem = FileSystem.get(conf);
    }
    @After
    public void after() throws Exception {
//        System.out.println("after");
        fileSystem.close();
    }
    @Test
    public void test(){
//        System.out.println("hello world");
    }
    //查看目录下有什么
    @Test
    public void testLs() throws Exception {
        FileStatus[] fileStatuses = fileSystem.listStatus(new Path("/"));
        for(FileStatus fs : fileStatuses){
//            System.out.println(fs);
            System.out.println(fs.getPath());
        }
    }
    //创建目录
    @Test
    public void testAddDir() throws Exception {
        boolean mkdirs = fileSystem.mkdirs(new Path("/usr/local/hadoop"));
        System.out.println(mkdirs ? "创建成功" : "创建失败");
    }
    //上传文件
    @Test
    public void testAddFile() throws IOException {
        FSDataOutputStream fsDataOutputStream = fileSystem.create(new Path("/usr/local/hadoop/log4j.properties"));
        FileInputStream fis = new FileInputStream("F:\\开发实战学习\\hdfspro\\src\\main\\resources\\log4j.properties");

        byte[] buf = new byte[1024];
        int len = -1 ;
        while((len =fis.read(buf))!=-1){
            fsDataOutputStream.write(buf,0,len);
        }
        fis.close();
        fsDataOutputStream.flush();
        fsDataOutputStream.close();
    }
	//工具类上传文件
    @Test
    public void testAddFile2() throws IOException {
        FSDataOutputStream fsDataOutputStream = fileSystem.create(new Path("/usr/local/hadoop/dataPdf"));
        FileInputStream fis = new FileInputStream("F:\\参考资料\\数据生成\\生成模型理论与应用研究_淦艳.caj");
        IOUtils.copyBytes(fis,fsDataOutputStream,conf);
        fis.close();
        fsDataOutputStream.flush();
        fsDataOutputStream.close();
    }
    @Test
    public void testUpdate(){

    }
    @Test
    public void deleteFile(){

    }
    //删除目录
    @Test
    public void deleteDir() throws Exception {
        boolean delete = fileSystem.delete(new Path("/usr/local/hadoop"),false);//boolean表示目录是否有数据，迭代删除
        System.out.println(delete ? "ok" : "no ok");
    }
	//下载文件 fos是下载后文件保存的地址
    @Test
    public void download() throws IOException {
        FSDataInputStream open = fileSystem.open(new Path("/usr/local/hadoop/dataPdf"));
        FileOutputStream fos = new FileOutputStream("F:\\参考资料\\数据生成\\生成模型理论与应用研究_淦艳111.caj");

        byte[] buf = new byte[1024];
        int len =-1 ;
        while((len = open.read(buf))!=-1){
            fos.write(buf,0,len);
        }
        open.close();
        fos.flush();
        fos.close();
    }
	//工具类下载
    @Test
    public void download2() throws IOException {
       /* FSDataInputStream open = fileSystem.open(new Path("/usr/local/hadoop/dataPdf"));
        FileOutputStream fos = new FileOutputStream("F:\\参考资料\\数据生成\\生成模型理论与应用研究_淦艳2222.caj");

        IOUtils.copyBytes(open,fos,conf);
        open.close();
        fos.flush();
        fos.close();*/
        FSDataInputStream open = fileSystem.open(new Path("/home/root/hello1.txt"));
        FileOutputStream fos = new FileOutputStream("F:\\参考资料\\数据生成\\hello1.txt");
        IOUtils.copyBytes(open,fos,conf);
        open.close();
        fos.flush();
        fos.close();
    }
    //获取block块位置信息
     @Test
     public void getBlockLocations() throws IOException {
         FileStatus fileStatus = fileSystem.getFileStatus(new Path("/home/root/hello1.txt"));
         BlockLocation[] fileBlockLocations = fileSystem.getFileBlockLocations(fileStatus, 0, fileStatus.getLen());
         for (BlockLocation fileBlockLocation : fileBlockLocations) {
             System.out.println(fileBlockLocation);
         }
     }
     //读取文件
    @Test
    public void readFile() throws IOException {
        FSDataInputStream fsDataInputStream = fileSystem.open(new Path("/home/root/hello1.txt"));
        //seek是从什么某位置开始，读取字节文件
        fsDataInputStream.seek(0);
        System.out.println(fsDataInputStream.readLine());
    }
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139