mac hadoop安装

Last updated on September 15, 2024 pm

🧙 Questions

mac的m2版本hadoop安装 3.3.5版本

☄️ Ideas

前提

开启远程登录

Setting –> General –> Sharing –> Remote login

20240424102117

本地免密
# 如果没有初始化则需要执行
# ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa
cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
chmod 0600 ~/.ssh/authorized_keys
安装java

下载安装hadoop

tar -vzxf /Users/ispong/OneDrive/Downloads/linux/hadoop/hadoop-3.3.5.tar.gz -C /Users/ispong
mv /Users/ispong/hadoop-3.3.5 /Users/ispong/hadoop
修改环境变量
tee -a ~/.zshrc <<-'EOF'
export HADOOP_HOME=/Users/ispong/hadoop
export HADOOP_CLASSPATH=$HADOOP_HOME/etc/hadoop:$HADOOP_HOME/share/hadoop/common/lib/*:$HADOOP_HOME/share/hadoop/common/*:$HADOOP_HOME/share/hadoop/hdfs:$HADOOP_HOME/share/hadoop/hdfs/lib/*:$HADOOP_HOME/share/hadoop/hdfs/*:$HADOOP_HOME/share/hadoop/mapreduce/lib/*:$HADOOP_HOME/share/hadoop/mapreduce/*:$HADOOP_HOME/share/hadoop/yarn:$HADOOP_HOME/share/hadoop/yarn/lib/*:$HADOOP_HOME/share/hadoop/yarn/* 
export HADOOP_CONF_DIR=$HADOOP_HOME/etc/hadoop
export PATH=$PATH:$HADOOP_HOME/bin
export PATH=$PATH:$HADOOP_HOME/sbin 
EOF
source ~/.zshrc
tee -a $HADOOP_HOME/etc/hadoop/hadoop-env.sh <<-'EOF'
export JAVA_HOME=/Users/ispong/.sdkman/candidates/java/current
export HDFS_NAMENODE_USER=ispong
export HDFS_DATANODE_USER=ispong
export HDFS_SECONDARYNAMENODE_USER=ispong
export YARN_RESOURCEMANAGER_USER=ispong
export YARN_NODEMANAGER_USER=ispong
export HDFS_ZKFC_USER=ispong
export HDFS_JOURNALNODE_USER=ispong
EOF
hadoop version
获取内网ip

inet 192.168.34.113 

ifconfig | grep broadcast
修改配置文件
code $HADOOP_HOME/etc/hadoop/core-site.xml
<configuration>
    <property>
        <name>fs.defaultFS</name>
        <value>hdfs://192.168.34.113:9000</value>
    </property>

    <property>
        <name>hadoop.proxyuser.ispong.hosts</name>
        <value>*</value>
    </property>

    <property>
        <name>hadoop.proxyuser.ispong.groups</name>
        <value>*</value>
    </property>
</configuration>
  • 创建hadoop的存储文件夹
mkdir -p /Users/ispong/data/hadoop/name
mkdir -p /Users/ispong/data/hadoop/data
code $HADOOP_HOME/etc/hadoop/hdfs-site.xml
<configuration>

    <property>
        <name>dfs.namenode.name.dir</name>
        <value>/Users/ispong/data/hadoop/name</value>
    </property>

    <property>
        <name>dfs.datanode.data.dir</name>
        <value>/Users/ispong/data/hadoop/data</value>
    </property>

    <property>
        <name>dfs.replication</name>
        <value>1</value>
    </property>

</configuration>
格式化hdfs
hdfs namenode -format
  • 降低hadoop日志等级
code $HADOOP_HOME/etc/hadoop/log4j.properties

新增

log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR
启动hdfs

stop-dfs.sh

start-dfs.sh
配置yarn
code $HADOOP_HOME/etc/hadoop/mapred-site.xml
<configuration>

    <property>
        <name>mapreduce.framework.name</name>
        <value>yarn</value>
    </property>

    <property>
        <name>mapreduce.application.classpath</name>
        <value>$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/*:$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/lib/*</value>
    </property>

      <!-- ispong :) ui访问地址，默认端口号19888 -->
    <property>
        <name>mapreduce.jobhistory.webapp.address</name>
        <value>192.168.34.113:19888</value>
    </property>
    
    <!-- ispong :) 默认端口号10020-->
    <property>
        <name>mapreduce.jobhistory.address</name>
        <value>192.168.34.113:10020</value>
    </property>
</configuration>
code $HADOOP_HOME/etc/hadoop/yarn-site.xml

容器最低推荐2GB

<configuration>

    <property>
        <name>yarn.resourcemanager.address</name>
        <value>192.168.34.113:8032</value>
    </property>
    <property>
        <name>yarn.nodemanager.aux-services</name>
        <value>mapreduce_shuffle</value>
    </property>
    <property>
        <name>yarn.nodemanager.env-whitelist</name>
        <value>JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,CLASSPATH_PREPEND_DISTCACHE,HADOOP_YARN_HOME,HADOOP_HOME,PATH,LANG,TZ,HADOOP_MAPRED_HOME</value>
    </property>

    <!-- 开启日志聚合 -->
    <property>
	    <name>yarn.log-aggregation-enable</name>
	    <value>true</value>
    </property>

    <!-- 启动日志严控后，一定要配置url，否则ui无法查看日志 -->
    <property>
	    <name>yarn.log.server.url</name>
	    <value>http://192.168.34.113:19888/jobhistory/logs</value>
    </property>

     <!-- ispong :) 配置当前结点yarn可用资源 -1为自动调整 默认为8192MB(8GB) 单位MB -->
    <property>
        <name>yarn.nodemanager.resource.memory-mb</name>
        <value>8192</value>
    </property>

    <!-- ispong :) 每个容器最大分配内存 默认为8192MB(8GB) 单位MB -->
    <!-- 看自己的需求，单个容器一般需要多大内存，配置的太大，会导致资源不足 -->
    <property>
        <name>yarn.scheduler.maximum-allocation-mb</name>
        <value>4096</value>
    </property>

     <!-- ispong :) 每个容器最小分配内存 默认为1024MB(1GB) 单位MB -->
    <property>
        <name>yarn.scheduler.minimum-allocation-mb</name>
        <value>2048</value>
    </property>

    <!-- ispong :) yarn上虚拟总vcore个数，默认值:-1-->
    <property>
        <name>yarn.nodemanager.resource.cpu-vcores</name>
        <value>4</value>
    </property>

    <!-- ispong :) 每个容器最小可分配内核，默认值:1-->
    <property>
        <name>yarn.scheduler.minimum-allocation-vcores</name>
        <value>1</value>
    </property>

    <!-- ispong :) 每个容器最小可分配内核，默认值:4-->
    <property>
        <name>yarn.scheduler.maximum-allocation-vcores</name>
        <value>1</value>
    </property>
    
</configuration>
启动yarn

stop-yarn.sh

start-yarn.sh
开启jobhistory

mapred –daemon stop historyserver

mapred --daemon start historyserver

卸载hadoop

# 停止hadoop集群
bash stop-all.sh
# 停止historyserver
mapred --daemon stop historyserver
# 删除hadoop安装文件
rm -rf /Users/ispong/hadoop
# 删除hadoop资源文件
rm -rf /Users/ispong/data/hadoop
# 复原~/.zshrc环境变量
code ~/.zshrc
source ~/.zshrc
mac
mac hadoop安装
https://ispong.isxcode.com/os/mac/mac hadoop安装/
Author
ispong
Posted on
February 15, 2023
Licensed under