- Hadoop + Spark生态系统操作与实战指南
- 余辉
- 1038字
- 2021-03-12 19:20:37
3.3 HDFS实战
3.3.1 HDFS客户端的操作
（1）HDFS的FileSystem Shell（如图3-7所示）
图3-7 HDFS的FileSystem Shell
（2）启动命令
[root@hadoop11~]# sh /usr/app/hadoop-2.6.0/sbin/start-dfs.sh
17/03/06 06:10:30 WARN util.NativeCodeLoader: Unable to load native-
hadoop library for your platform... using builtin-java classes where
applicable
Starting namenodes on [hadoop11 hadoop12]
hadoop11: starting namenode, logging to /usr/app/hadoop-
2.6.0/logs/hadoop-root-namenode-hadoop11.out
hadoop12: starting namenode, logging to /usr/app/hadoop-
2.6.0/logs/hadoop-root-namenode-hadoop12.out
hadoop11: starting datanode, logging to /usr/app/hadoop-
2.6.0/logs/hadoop-root-datanode-hadoop11.out
hadoop12: starting datanode, logging to /usr/app/hadoop-
2.6.0/logs/hadoop-root-datanode-hadoop12.out
hadoop13: starting datanode, logging to /usr/app/hadoop-
2.6.0/logs/hadoop-root-datanode-hadoop13.out
Starting journal nodes [hadoop11 hadoop12 hadoop13]
hadoop12: starting journalnode, logging to /usr/app/hadoop-
2.6.0/logs/hadoop-root-journalnode-hadoop12.out
hadoop13: starting journalnode, logging to /usr/app/hadoop-
2.6.0/logs/hadoop-root-journalnode-hadoop13.out
hadoop11: starting journalnode, logging to /usr/app/hadoop-
2.6.0/logs/hadoop-root-journalnode-hadoop11.out
17/03/06 06:10:55 WARN util.NativeCodeLoader: Unable to load native-
hadoop library for your platform... using builtin-java classes where
applicable
Starting ZK Failover Controllers on NN hosts [hadoop11 hadoop12]
hadoop11: starting zkfc, logging to /usr/app/hadoop-
2.6.0/logs/hadoop-root-zkfc-hadoop11.out
hadoop12: starting zkfc, logging to /usr/app/hadoop-
2.6.0/logs/hadoop-root-zkfc-hadoop12.out
（3）启动进程
[root@hadoop11~]# jps
3733 NameNode
3831 DataNode
4013 JournalNode
2682 QuorumPeerMain
4155 DFSZKFailoverController
4262 Jps
（4）HDFS的界面
HDFS的界面访问地址为http://hadoop11:50070/dfshealth.html#tab-overview。
HDFS的界面如图3-8所示。
图3-8 HDFS的界面
（5）查看目录（HDFS目录结构如图3-9所示）
图3-9 HDFS目录结构
[root@hadoop11~]# hadoop fs -ls /
drwxr-xr-x - root supergroup 02017-03-05 06:14
/hbase
drwx------ - root supergroup 02017-03-03 05:39 /tmp
drwxr-xr-x - Administrator supergroup 02017-03-05 12:06
/user
drwxr-xr-x - root supergroup 02017-03-03 05:39 /usr
（6）增加目录
[root@hadoop11~]# hadoop fs -ls /usr
drwxr-xr-x - root supergroup 02017-03-05 02:11 /usr/yuhui
hadoop fs -mkdir /usr/orcale
[root@hadoop11~]# hadoop fs -ls /usr
drwxr-xr-x - root supergroup 02017-03-06 06:22 /usr/orcale
drwxr-xr-x - root supergroup 02017-03-05 02:11 /usr/yuhui
（7）删除目录
hadoop fs -rm /usr/orcale
（8）修改目录
hadoop fs -mkdir /usr/orcale
@@@
hadoop fs -mv /usr/orcale /usr/xiaohui
（9）上传文件
hadoop fs -put hadoop-stop.sh /usr/xiaohui/
（10）查看文件
hadoop fs -cat /user/xiaohui/hadoop-stop.sh
（11）删除文件
hadoop fs -rm /usr/xiaohui/hadoop-stop.sh
3.3.2 Java操作HDFS
1．注意事项01
Windows操作HDFS的时候，需要让主机用户Administrator有操作权限。HDFS文件权限如图3-10所示。
图3-10 HDFS文件权限
执行命令如下：
hadoop fs -chmod 777 /
2．注意事项02
将4个配置文件core-site.xml、hdfs-site.xml、mapred-site.xml、yarn-site.xml放入resources中。Java操作HDFS类如图3-11所示。
图3-11 Java操作HDFS类
package cn.orcale.com.bigdata.hdfs;
@@@
import java.io.IOException;
import java.io.InputStream;
@@@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;
@@@
public class HdfsTest {
@@@
FileSystem fs = null;
@@@
// 创建新文件
public static void createFile(String dst, byte[] contents)
throws IOException {
Configuration conf = new Configuration();
FileSystem fs = FileSystem.get(conf);
Path dstPath = new Path(dst); // 目标路径
// 打开一个输出流
FSDataOutputStream outputStream = fs.create(dstPath);
outputStream.write(contents);
outputStream.close();
fs.close();
System.out.println("文件创建成功！");
}
@@@
// 上传本地文件
public static void uploadFile(String src, String dst) throws
IOException {
Configuration conf = new Configuration();
FileSystem fs = FileSystem.get(conf);
Path srcPath = new Path(src); // 原路径
Path dstPath = new Path(dst); // 目标路径
// 调用文件系统的文件复制函数，前面参数是指是否删除原文件，true为删除，默
认为false
fs.copyFromLocalFile(false, srcPath, dstPath);
@@@
// 打印文件路径
System.out.println("Upload to " +
conf.get("fs.default.name"));
System.out.println("------------list files------------" +
"\n");
FileStatus[] fileStatus = fs.listStatus(dstPath);
for (FileStatus file : fileStatus) {
System.out.println(file.getPath());
}
fs.close();
}
@@@
// 文件重命名
public static void rename(String oldName, String newName)
throws IOException {
Configuration conf = new Configuration();
FileSystem fs = FileSystem.get(conf);
Path oldPath = new Path(oldName);
Path newPath = new Path(newName);
boolean isok = fs.rename(oldPath, newPath);
if (isok) {
System.out.println("rename ok! ");
} else {
System.out.println("rename failure");
}
fs.close();
}
@@@
// 删除文件
public static void delete(String filePath) throws IOException {
Configuration conf = new Configuration();
FileSystem fs = FileSystem.get(conf);
Path path = new Path(filePath);
boolean isok = fs.deleteOnExit(path);
if (isok) {
System.out.println("delete ok! ");
} else {
System.out.println("delete failure");
}
fs.close();
}
@@@
// 创建目录
public static void mkdir(String path) throws IOException {
Configuration conf = new Configuration();
FileSystem fs = FileSystem.get(conf);
Path srcPath = new Path(path);
boolean isok = fs.mkdirs(srcPath);
if (isok) {
System.out.println("create dir ok! ");
} else {
System.out.println("create dir failure");
}
fs.close();
}
@@@
// 读取文件的内容
public static void readFile(String filePath) throws IOException {
Configuration conf = new Configuration();
FileSystem fs = FileSystem.get(conf);
Path srcPath = new Path(filePath);
InputStream in = null;
try {
in = fs.open(srcPath);
IOUtils.copyBytes(in, System.out, 4096, false); // 复制到标
准输出流
} finally {
IOUtils.closeStream(in);
}
}
@@@
public static void main(String[] args) throws IOException {
@@@
/***
* 本地操作HDFS的时候，需要让主机用户Administrator有操作权限hadoop
fs -chmod 777 /
*/
//
@@@
// 测试新建目录
// mkdir("/test1");
@@@
// 测试上传文件
// uploadFile("D:\\aaa.txt", "/test1");
@@@
// 测试创建文件
// byte[] contents = "hello world世界你好\n".getBytes();
// createFile("/test1/d.txt", contents);
@@@
// 测试重命名
// rename("/test1/d.txt", "/test1/dd.txt");
@@@
// 测试读取文件
// readFile("/test1/aaa.txt");
@@@
// 测试删除文件
// delete("/test1/dd.txt"); //使用相对路径
// delete("/test1"); //删除目录
@@@
}
@@@
}