JAVA操作hadoop上传下载_java hadoop-client 下载

一、客户端环境准备

1.相关包

<dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-common</artifactId>
            <version>2.6.5</version>
        </dependency>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-hdfs</artifactId>
            <version>2.6.5</version>
        </dependency>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-client</artifactId>
            <version>2.6.5</version>
        </dependency>
        <dependency>
            <groupId>jdk.tools</groupId>
            <artifactId>jdk.tools</artifactId>
            <version>1.8</version>
            <scope>system</scope>
            <systemPath>C:/Program Files/Java/jdk1.8.0_152/lib/tools.jar</systemPath>
        </dependency>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22

二、基础操作代码

1.创建连接

	public  FileSystem createNewFileSystem(){
		try {
			Configuration  conf = new Configuration();
			URI uri = new URI("hdfs://ip:port");
			String scheme = uri.getScheme();
			String disableCacheName = String.format("fs.%s.impl.disable.cache", scheme);
	        conf.setBoolean(disableCacheName, true);
	        System.setProperty("hadoop.home.dir", "D:\\hadoop-3.1.1");
        	FileSystem fs = FileSystem.newInstance(uri,conf , "用户名");
			return fs;
		} catch (Exception e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
			return null;
		}
	}
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16

2.文件上传

 FSDataOutputStream out = null;
			InputStream is = null;
			try {
				//hdfs路径格式 /权限文件夹/系统名称/当天日期/文件名称
				String hdfsPath = "/"+fileFolder+"/"+resourceSystem+"/"+DateUtils.getDate()+"/"+fileName;
				FileSystem fs  = createNewFileSystem();
				out = fs.create(new Path(hdfsPath));
      			is = handler.getInputStream();
      			byte[] b = new byte[is.available()];
      	        int read = 0;
      	        while((read = is.read(b)) > 0){
      	            out.write(b, 0, read);
      	        }
      	        //保存数据入库
      	        return JSONArray.toJSON(AjaxResult.success(uuid)).toString();
			} catch (Exception e) {
				e.printStackTrace();
				return JSONArray.toJSON(AjaxResult.error("文件上传失败")).toString();
			} finally {
				try {
					if (is != null) {
						is.close();
					}
					if (out != null) {
						out.close();
					}
					if(fs != null){
						fs.close();
					}			
				} catch (Exception e) {
					e.printStackTrace();
				}
			}
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33

3.文件下载

1.本地下载

try{
 	FileSystem fs  = createNewFileSystem();
 	fs.copyToLocalFile(false, new Path("/hello.txt"), new Path("d:\\hello.txt"), true);
}catch (Exception e) {
	// TODO Auto-generated catch block
	e.printStackTrace();
}finally {
	if(fs != null){
		fs.close();
	}
}
1
2
3
4
5
6
7
8
9
10
11

2.返回byte数组

FSDataInputStream hdfsInStream = null;
		 byte ioBuffer[] = null; 
		 FileSystem fs  = createNewFileSystem();
			 //查询文件是否存在，
			 FileResource fileResource = iFileResourceService.selectFileResourceByFileid(fileid);
			 if(null!= fileResource) {
				 Path path = new Path(fileResource.getResourceFilePath()); 
				 try {
					hdfsInStream = fs.open(path);
					ioBuffer = new byte[hdfsInStream.available()];
				    IOUtils.read(hdfsInStream, ioBuffer);
				} catch (Exception e) {
					e.printStackTrace();
				}finally {
					try {
						if (hdfsInStream!= null) {
							hdfsInStream.close();
						}
						if(fs != null{
							fs.close();
						}	
					} catch (IOException e) {
						// TODO Auto-generated catch block
						e.printStackTrace();
					}
				}
				 return ioBuffer;
			 }else {
				 return null;
			 }
		  }
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31

4.文件删除

待添加