1.讀取單個文件

?

?

[java]?view plain?copy

Date?date?=?DateUtil.getSpecifiedDayBefore();??

String?yesterday?=?DateUtil.dateToStr(date,?"yyyy-MM-dd");??

String?path?=?"hdfs://ip:9000/output_log/output_log_click"?+?yesterday;??

??

?Configuration?conf?=?new?Configuration();??

FileSystem?fs?=?FileSystem.get(URI.create(path),?conf);??

FSDataInputStream?hdfsInStream?=?fs.open(new?Path(path));??

InputStreamReader?isr?=?new?InputStreamReader(hdfsInStream,?"utf-8");??

BufferedReader?br?=?new?BufferedReader(isr);??

String?line;??

//?int?k?=?0;??

while?((line?=?br.readLine())?!=?null)?{??

????System.out.println(line);??

}??

???

?

?

2.讀取文件夾

?

[java]?view plain?copy

Date?date?=?DateUtil.getSpecifiedDayBefore();??

String?yesterday?=?DateUtil.dateToStr(date,?"yyyy-MM-dd");??

String?path?=?"hdfs://ip:9000/output_log/output_log_click"?+?yesterday;??

??

Configuration?conf?=?new?Configuration();??

FileSystem?fs?=?FileSystem.get(URI.create(path),?conf);??

FileStatus[]?status?=?fs.listStatus(new?Path(path));??

for?(FileStatus?file?:?status)?{??

????if?(!file.getPath().getName().startsWith("newsMap"))?{??

??????????????continue;??

??????????}??

????FSDataInputStream?hdfsInStream?=?fs.open(file.getPath());??

????InputStreamReader?isr?=?new?InputStreamReader(hdfsInStream,?"utf-8");??

????BufferedReader?br?=?new?BufferedReader(isr);??

????String?line;??

????//?int?k?=?0;??

????while?((line?=?br.readLine())?!=?null)?{??

????????System.out.println(line);??

????}??

??????

}??

?

package archy.com;

import java.io.InputStream;
import java.net.URI;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;

public class MyHdfs {
?? ?// hadoop fs的配置文件
?? ?static Configuration conf = new Configuration(true);
?? ?static {
?? ??? ?// 指定hadoop fs的地址
?? ??? ?conf.set("fs.default.name", "hdfs://bigdata01.com:8020");
?? ?}

?? ?/**
?? ? * 將本地文件(filePath)上傳到HDFS服務(wù)器的指定路徑(dst)
?? ? *?
?? ? * @author 馮琪
?? ? */
?? ?public static void uploadFileToHDFS(String filePath, String dst)
?? ??? ??? ?throws Exception {
?? ??? ?// 創(chuàng)建一個文件系統(tǒng)
?? ??? ?FileSystem fs = FileSystem.get(conf);
?? ??? ?Path srcPath = new Path(filePath);
?? ??? ?Path dstPath = new Path(dst);
?? ??? ?Long start = System.currentTimeMillis();
?? ??? ?fs.copyFromLocalFile(false, srcPath, dstPath);
?? ??? ?System.out.println("Time:" + (System.currentTimeMillis() - start));
?? ??? ?System.out.println("________準(zhǔn)備上傳文件" + conf.get("fs.default.name")
?? ??? ??? ??? ?+ "____________");
?? ??? ?fs.close();
?? ??? ?getDirectoryFromHdfs(dst);
?? ?}

?? ?/**
?? ? * 下載文件
?? ? *?
?? ? * @author 馮琪
?? ? */
?? ?public static void downLoadFileFromHDFS(String src) throws Exception {
?? ??? ?FileSystem fs = FileSystem.get(conf);
?? ??? ?Path srcPath = new Path(src);
?? ??? ?InputStream in = fs.open(srcPath);
?? ??? ?try {
?? ??? ??? ?// 將文件COPY到標(biāo)準(zhǔn)輸出(即控制臺輸出)
?? ??? ??? ?IOUtils.copyBytes(in, System.out, 4096, false);
?? ??? ?} finally {
?? ??? ??? ?IOUtils.closeStream(in);
?? ??? ??? ?fs.close();
?? ??? ?}
?? ?}

?? ?/**
?? ? * 遍歷指定目錄(direPath)下的所有文件
?? ? *?
?? ? * @author 馮琪
?? ? */
?? ?public static void getDirectoryFromHdfs(String direPath) throws Exception {

?? ??? ?FileSystem fs = FileSystem.get(URI.create(direPath), conf);
?? ??? ?FileStatus[] filelist = fs.listStatus(new Path(direPath));
?? ??? ?for (int i = 0; i < filelist.length; i++) {
?? ??? ??? ?System.out.println("_________________第" + i + "個文件"
?? ??? ??? ??? ??? ?+ "____________________");
?? ??? ??? ?FileStatus fileStatus = filelist[i];
?? ??? ??? ?System.out.println("Name:" + fileStatus.getPath().getName());
?? ??? ??? ?System.out.println("size:" + fileStatus.getLen());
?? ??? ??? ?System.out.println("_________________第" + i + "個文件"
?? ??? ??? ??? ??? ?+ "____________________");
?? ??? ?}
?? ??? ?fs.close();
?? ?}

?? ?/**
?? ? * 測試方法
?? ? *?
?? ? * @author 馮琪
?? ? */
?? ?public static void main(String[] args) {
?? ??? ?try {
?? ??? ??? ?// 獲取目錄下的所有文件,hdfs服務(wù)器文件夾路徑
?? ??? ??? ?getDirectoryFromHdfs("/fengqi/1104/");
?? ??? ??? ?// 上傳文件,第一個參數(shù)為本地要上傳的文件路徑，第二個參數(shù)為HDFS的路徑。
?? ??? ??? ?// uploadFileToHDFS("D:/456.txt", "/fengqi/1104/");
?? ??? ??? ?// 下載文件，第一個參數(shù)為HDFS上要下載的文件路徑
?? ??? ??? ?// downLoadFileFromHDFS("/fengqi/1104/456.txt");
?? ??? ?} catch (Exception e) {
?? ??? ??? ?// TODO 自動生成的 catch 塊
?? ??? ??? ?e.printStackTrace();
?? ??? ?}
?? ?}
}
---------------------?
?

總結(jié)

以上是生活随笔為你收集整理的Java操作HDFS文件的全部內(nèi)容，希望文章能夠幫你解決所遇到的問題。

如果覺得生活随笔網(wǎng)站內(nèi)容還不錯，歡迎將生活随笔推薦給好友。