当前位置：首页 > news >正文

Hadoop3.3.5搭建指南(双NN版本)

news 2025/9/19 9:37:33

由最初的单NN双DN的模式，扩展到高HA架构，增加一个节点NN，现在变成双NN和双DN的架构，需要了解的请移步上一篇文章

服务器规划如下

54、44服务器 NN RM

61.62服务器 DN NM

Zookpeer/JournalNode 54.61.62

ZKFC 54.44

1、配置hosts

54 master

44 master2

61 slave1

62 slave2

2、修改Hadoop的配置

修改：vi core-site.xml

<configuration>
<property>
<!-- 指定HDFS的通信地址，在NameNode HA中连接到nameservice的，
值为mycluster是需要在下一个hdfs-site.xml中配置的 -->
<name>fs.defaultFS</name>
<value>hdfs://ns1</value>
</property>
<property>
<!-- 配置Hadoop 的临时目录 -->
<name>hadoop.tmp.dir</name>
<value>file:/datas/hadoop/hadoop-3.3.5/tmp</value>
</property>
<property>
<!-- 指定Zookeeper地址（2181端口参考zoo.cfg配置文件） -->
<name>ha.zookeeper.quorum</name>
<value>master:2181,slave1:2181,slave2:2181</value>
</property>
<property>
<name>hadoop.http.staticuser.user</name>
<value>root</value>
</property>
<property>
<name>hadoop.proxyuser.root.hosts</name>
<value></value></property><property><name>hadoop.proxyuser.root.groups</name><value></value>
</property>
</configuration>

修改vi hdfs-site.xml配置

<property>
<name>dfs.replication</name>
<value>3</value>
</property>
<property>
<name>dfs.namenode.name.dir</name>
<value>file:/datas/hadoop/hadoop-3.3.5/tmp/master</value>
</property>
<property>
<name>dfs.datanode.data.dir</name>
<value>file:/datas/hadoop/hadoop-3.3.5/tmp/slave</value>
</property>
<!--指定hdfs的nameservice为ns，需要和core-site.xml中的保持一致 -->
<property>
<name>dfs.nameservices</name>
<value>ns1</value>
</property>
<!-- ns下面有两个NameNode，分别是master，master2 -->
<property>
<name>dfs.ha.namenodes.ns1</name>
<value>master,master2</value>
</property>
<property>
<name>dfs.namenode.rpc-address.ns1.master</name>
<value>master:9001</value>
</property>
<property>
<name>dfs.namenode.http-address.ns1.master</name>
<value>master:9870</value>
</property>
<property>
<name>dfs.namenode.rpc-address.ns1.master2</name>
<value>master2:9001</value>
</property>
<property>
<name>dfs.namenode.http-address.ns1.master2</name>
<value>master2:9870</value>
</property>
<!-- 指定NameNode的元数据在JournalNode上的存放位置（即NameNode集群间的用于共享的edits日志的journal节点列表） -->
<property>
<name>dfs.namenode.shared.edits.dir</name>
<value>qjournal://master:8485;slave1:8485;slave2:8485/ns1</value>
</property>
<!-- 指定JournalNode在本地磁盘存放数据的位置  -->
<property>
<name>dfs.journalnode.edits.dir</name>
<value>/hadoop/export/data/journaldata/</value>
</property>
<!-- 开启NameNode失败自动切换 -->
<property>
<name>dfs.ha.automatic-failover.enabled</name>
<value>true</value>
</property>
<!-- 配置NameNode失败后自动切换实现方式（客户端连接可用状态的NameNode所用的代理类） -->
<property>
<name>dfs.client.failover.proxy.provider.ns1</name>
<value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
</property>
<property>
<name>dfs.permissions.enable</name>
<value>false</value>
</property>
<!-- 配置隔离机制 -->
<property>
<name>dfs.ha.fencing.methods</name>
<value>
sshfence
shell(/bin/true)
</value>
</property>
<!-- 设置使用隔离机制时需要的SSH免密登录 -->
<property>
<name>dfs.ha.fencing.ssh.private-key-files</name>
<value>/root/.ssh/id_rsa</value>
</property>
<property>
<name>dfs.ha.fencing.ssh.connect-timeout</name>
<value>30000</value>
</property>
<property>
<name>dfs.namenode.name.dir</name>
<value>file:/datas/hadoop/hadoop-3.3.5/tmp/master</value>
</property>
<property>
<name>dfs.datanode.data.dir</name>
<value>file:/datas/hadoop/hadoop-3.3.5/tmp/slave</value>
</property>

修改vi yarn-site.xml

<configuration>
<property>
<!-- 启用ResourceManager的HA功能 -->
<name>yarn.resourcemanager.ha.enabled</name>
<value>true</value>
</property>
<property>
<!-- 开启ResourceManager失败自动切换 -->
<name>yarn.resourcemanager.ha.automatic-failover.enabled</name>
<value>true</value>
</property>
<property>
<!-- 给ResourceManager HA集群命名id -->
<name>yarn.resourcemanager.cluster-id</name>
<value>yarn-cluster</value>
</property>
<property>
<!-- 指定ResourceManager HA有哪些节点 -->
<name>yarn.resourcemanager.ha.rm-ids</name>
<value>rm1,rm2</value>
</property>
<property>
<!-- 指定第一个节点在那一台机器 -->
<name>yarn.resourcemanager.hostname.rm1</name>
<value>master</value>
</property>
<property>
<!-- 指定第二个节点在那一台机器 -->
<name>yarn.resourcemanager.hostname.rm2</name>
<value>master2</value>
</property>
<property>
<!-- 指定ResourceManager HA所用的Zookeeper节点 -->
<name>yarn.resourcemanager.zk-address</name>
<value>master:2181,slave1:2181,slave2:2181</value>
</property>
<property>
<!-- 启用RM重启的功能，默认为false -->
<name>yarn.resourcemanager.recovery.enabled</name>
<value>true</value>
</property>
<property>
<!-- 用于状态存储的类 -->
<name>yarn.resourcemanager.store.class</name>
<value>org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore</value>
</property>
<property>
<!-- NodeManager启用server的方式 -->
<name>yarn-nodemanager.aux-services</name>
<value>mapreduce_shuffle</value>
</property>
<property>
<!-- NodeManager启用server使用算法的类 -->
<name>yarn-nodemanager.aux-services.mapreduce.shuffle.class</name>
<value>org.apache.hadoop.mapred.ShuffleHandler</value>
</property>
<property>
<!-- 启用日志聚合功能 -->
<name>yarn.log-aggregation-enable</name>
<value>true</value>
</property>
<property>
<!-- 聚集的日志在HDFS上保存最长的时间 -->
<name>yarn.log-aggregation.retain-seconds</name>
<value>106800</value>
</property>
<property>
<!-- 聚集的日志在HDFS上保存最长的时间 -->
<name>yarn.nodemanager.remote-app-log-dir</name>
<value>/datas/hadoop/hadoopHA/logs</value>
</property>
<!--hadoop3里面必须要添加的classpath-->
<property>
<name>yarn.application.classpath</name>
<value>/datas/hadoop/hadoop-3.3.5/etc/hadoop:/datas/hadoop/hadoop-3.3.5/share/hadoop/common/lib/:/datas/hadoop/hadoop-3.3.5/share/hadoop/common/:/datas/hadoop/hadoop-3.3.5/share/hadoop/hdfs:/datas/hadoop/hadoop-3.3.5/share/hadoop/hdfs/lib/:/datas/hadoop/hadoop-3.3.5/share/hadoop/hdfs/:/datas/hadoop/hadoop-3.3.5/share/hadoop/mapreduce/lib/:/datas/hadoop/hadoop-3.3.5/share/hadoop/mapreduce/:/datas/hadoop/hadoop-3.3.5/share/hadoop/yarn:/datas/hadoop/hadoop-3.3.5/share/hadoop/yarn/lib/:/datas/hadoop/hadoop-3.3.5/share/hadoop/yarn/</value>
</property>
</configuration>