hadoop 单节点极速安装

Last updated on July 19, 2025 am

🧙 Questions

单节点安装hadoop3.3.5

☄️ Ideas

前提hadoop.proxyuser

下载hadoop

cd /tmp
nohup wget https://archive.apache.org/dist/hadoop/common/hadoop-3.3.5/hadoop-3.3.5.tar.gz >> download_hadoop.log 2>&1 &
tail -f download_hadoop.log

创建安装目录

sudo mkdir -p /data/hadoop/
sudo chown -R ispong:ispong /data/hadoop/

解压且创建软连接

tar -vzxf /tmp/hadoop-3.3.5.tar.gz -C /data/hadoop/
sudo ln -s /data/hadoop/hadoop-3.3.5 /opt/hadoop

配置环境变量

新版本不用配置 YARN_CONF_DIR

sudo tee -a /etc/profile <<-'EOF'
export HADOOP_HOME=/opt/hadoop
export HADOOP_CLASSPATH=$HADOOP_HOME/etc/hadoop:$HADOOP_HOME/share/hadoop/common/lib/*:$HADOOP_HOME/share/hadoop/common/*:$HADOOP_HOME/share/hadoop/hdfs:$HADOOP_HOME/share/hadoop/hdfs/lib/*:$HADOOP_HOME/share/hadoop/hdfs/*:$HADOOP_HOME/share/hadoop/mapreduce/lib/*:$HADOOP_HOME/share/hadoop/mapreduce/*:$HADOOP_HOME/share/hadoop/yarn:$HADOOP_HOME/share/hadoop/yarn/lib/*:$HADOOP_HOME/share/hadoop/yarn/* 
export HADOOP_CONF_DIR=$HADOOP_HOME/etc/hadoop
export PATH=$PATH:$HADOOP_HOME/bin 
export PATH=$PATH:$HADOOP_HOME/sbin 
EOF
source /etc/profile

修改环境文件hadoop-env.sh

直接新增

sudo tee -a /opt/hadoop/etc/hadoop/hadoop-env.sh <<-'EOF'
export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk
export HDFS_NAMENODE_USER=ispong
export HDFS_DATANODE_USER=ispong
export HDFS_SECONDARYNAMENODE_USER=ispong
export YARN_RESOURCEMANAGER_USER=ispong
export YARN_NODEMANAGER_USER=ispong
export HDFS_ZKFC_USER=ispong
export HDFS_JOURNALNODE_USER=ispong
EOF

检查安装

hadoop version

设置localhost免密

ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa
cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
chmod 0600 ~/.ssh/authorized_keys

core-site.xml

<name>fs.defaultFS</name> 最好使用内网

vim /opt/hadoop/etc/hadoop/core-site.xml

<configuration>
    <property>
        <name>fs.defaultFS</name>
        <value>hdfs://isxcode:9000</value>
    </property>

    <!-- ispong :) 设置用户免权限 -->
    <property>
        <!-- <name>hadoop.proxyuser.${user}.hosts</name> -->
        <name>hadoop.proxyuser.ispong.hosts</name>
        <value>*</value>
    </property>

    <!-- ispong :) 设置用户免权限 -->
    <property>
        <!-- <name>hadoop.proxyuser.${user}.groups</name> -->
        <name>hadoop.proxyuser.ispong.groups</name>
        <value>*</value>
    </property>

</configuration>

hdfs-site.xml

vim /opt/hadoop/etc/hadoop/hdfs-site.xml

<configuration>

     <!-- ispong :) namenode数据存储位置 -->
    <property>
        <name>dfs.namenode.name.dir</name>
        <value>/data/hadoop/name</value>
    </property>

    <!-- ispong :) datanode数据存储位置 -->
    <property>
        <name>dfs.datanode.data.dir</name>
        <value>/data/hadoop/data</value>
    </property>

    <!-- ispong :) 备份数量 -->
    <property>
        <name>dfs.replication</name>
        <value>1</value>
    </property>

    <!-- ispong :) namenode web访问端口号，默认9870 -->
    <property>
        <name>dfs.namenode.http-address</name>
        <value>isxcode:40000</value> 
    </property>

    <!-- 传输文件端口号 -->
    <property>
        <name>dfs.datanode.address</name>
        <value>isxcode:9866</value> 
    </property>

</configuration>

格式化namenode

hdfs namenode -format

启动hdfs

关闭命令 stop-dfs.sh

start-dfs.sh

jps
# 2553 SecondaryNameNode
# 2285 DataNode
# 2142 NameNode

默认端口号9870

http://isxcode:40000/

20220922122530

mapred-site.xml

vim /opt/hadoop/etc/hadoop/mapred-site.xml

<configuration>
    <property>
        <name>mapreduce.framework.name</name>
        <value>yarn</value>
    </property>
    <property>
        <name>mapreduce.application.classpath</name>
        <value>$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/*:$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/lib/*</value>
    </property>

    <!-- ispong :) jobhistory 网页访问地址，默认19888  -->
    <property>
        <name>mapreduce.jobhistory.webapp.address</name>
        <value>isxcode:40001</value>
    </property>

    <property>
        <name>mapreduce.jobhistory.address</name>
        <value>isxcode:10020</value>
    </property>
</configuration>

yarn-site.xml

vim /opt/hadoop/etc/hadoop/yarn-site.xml

<configuration>

     <!-- ispong :) yarn 网页访问地址，默认8088  -->
     <property>
        <name>yarn.resourcemanager.webapp.address</name>
        <value>isxcode:40002</value>
    </property>
    
    <property>
        <name>yarn.nodemanager.aux-services</name>
        <value>mapreduce_shuffle</value>
    </property>
    <property>
        <name>yarn.nodemanager.env-whitelist</name>
        <value>JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,YARN_CONF_DIR,CLASSPATH_PREPEND_DISTCACHE,HADOOP_YARN_HOME,HADOOP_HOME,PATH,LANG,TZ,HADOOP_MAPRED_HOME</value>
    </property>
    <property>
	    <name>yarn.log-aggregation-enable</name>
	    <value>true</value>
    </property>

    <!-- ispong :) 配置外网ip -->
    <property>
	    <name>yarn.log.server.url</name>
	    <value>http://47.92.152.18:40001/jobhistory/logs</value>
    </property>

    <!-- ispong :) 根据需求设置，总内存资源, 越大越好，20GB -->
    <property>
        <name>yarn.nodemanager.resource.memory-mb</name>
        <value>20480</value>
    </property>

    <!-- ispong :) 根据需求设置，总cpu资源，可以和真实cpu保持一致，或者2倍 -->
    <property>
        <name>yarn.nodemanager.resource.cpu-vcores</name>
        <value>8</value>
    </property>

    <!-- ispong :) 根据需求设置，容器最小cpu，推荐1核-->
    <property>
        <name>yarn.scheduler.minimum-allocation-vcores</name>
        <value>1</value>
    </property>

    <!-- ispong :) 根据需求设置，容器最大cpu，推荐2核 -->
    <property>
        <name>yarn.scheduler.maximum-allocation-vcores</name>
        <value>2</value>
    </property>

    <!-- ispong :) 根据需求设置，容器最大内存，推荐4g -->
    <property>
        <name>yarn.scheduler.maximum-allocation-mb</name>
        <value>4096</value>
    </property>

    <!-- ispong :) 根据需求设置，容器最小内存，推荐1g -->
    <property>
        <name>yarn.scheduler.minimum-allocation-mb</name>
        <value>1024</value>
    </property>

     <!-- ispong :) node节点 网页访问地址，默认8042  -->
    <property>
        <name>yarn.nodemanager.webapp.address</name>
        <value>isxcode:40003</value>
    </property>

    <property>
        <name>yarn.resourcemanager.address</name>
        <value>isxcode:8032</value>
    </property>

</configuration>

启动yarn

关闭yarn命令 stop-yarn.sh

start-yarn.sh

jps
# 3079 NodeManager
# 2926 ResourceManager

默认端口号8088

http://isxcode:40002/

20220922122720

启动jobhistory

关闭服务: mapred –daemon stop historyserver

mapred --daemon start historyserver

http://isxcode:40001

20221214143519

🔗 Links

hadoop

hadoop 单节点极速安装

https://ispong.isxcode.com/hadoop/hadoop/hadoop 单节点极速安装/

Author

ispong

Posted on

January 1, 2020

Licensed under

spring logging Previous

hbase 单节点安装 Next