mac hadoop安装

🧙 Questions

mac的m2版本hadoop安装 3.3.5版本

☄️ Ideas

开启远程登录

Setting –> General –> Sharing –> Remote login

本地免密

# 如果没有初始化则需要执行
# ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa
cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
chmod 0600 ~/.ssh/authorized_keys

下载安装hadoop

mkdir -p /Users/ispong/hadoop/
tar -vzxf /Users/ispong/OneDrive/Downloads/linux/hadoop/hadoop-3.3.5.tar.gz -C /Users/ispong/hadoop

修改环境变量

tee -a ~/.zshrc <<-'EOF'
export HADOOP_HOME=/Users/ispong/hadoop/hadoop-3.3.5
export HADOOP_CLASSPATH=$HADOOP_HOME/etc/hadoop:$HADOOP_HOME/share/hadoop/common/lib/*:$HADOOP_HOME/share/hadoop/common/*:$HADOOP_HOME/share/hadoop/hdfs:$HADOOP_HOME/share/hadoop/hdfs/lib/*:$HADOOP_HOME/share/hadoop/hdfs/*:$HADOOP_HOME/share/hadoop/mapreduce/lib/*:$HADOOP_HOME/share/hadoop/mapreduce/*:$HADOOP_HOME/share/hadoop/yarn:$HADOOP_HOME/share/hadoop/yarn/lib/*:$HADOOP_HOME/share/hadoop/yarn/* 
export HADOOP_CONF_DIR=$HADOOP_HOME/etc/hadoop
export PATH=$PATH:$HADOOP_HOME/bin
export PATH=$PATH:$HADOOP_HOME/sbin 
export JAVA_HOME=/Users/ispong/.sdkman/candidates/java/current
EOF
source ~/.zshrc

tee -a $HADOOP_HOME/etc/hadoop/hadoop-env.sh <<-'EOF'
export JAVA_HOME=/Users/ispong/.sdkman/candidates/java/current
export HDFS_NAMENODE_USER=ispong
export HDFS_DATANODE_USER=ispong
export HDFS_SECONDARYNAMENODE_USER=ispong
export YARN_RESOURCEMANAGER_USER=ispong
export YARN_NODEMANAGER_USER=ispong
export HDFS_ZKFC_USER=ispong
export HDFS_JOURNALNODE_USER=ispong
EOF

hadoop version

修改配置文件

code $HADOOP_HOME/etc/hadoop/core-site.xml

<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>

<configuration>
    <property>
        <name>fs.defaultFS</name>
        <value>hdfs://ispong-mac.local:9000</value>
    </property>

    <property>
        <name>hadoop.proxyuser.ispong.hosts</name>
        <value>*</value>
    </property>

    <property>
        <name>hadoop.proxyuser.ispong.groups</name>
        <value>*</value>
    </property>
</configuration>

创建hadoop的存储文件夹

mkdir -p /Users/ispong/hadoop/name
mkdir -p /Users/ispong/hadoop/data

code $HADOOP_HOME/etc/hadoop/hdfs-site.xml

<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>

<configuration>

    <property>
        <name>dfs.namenode.name.dir</name>
        <value>/Users/ispong/hadoop/name</value>
    </property>

    <property>
        <name>dfs.datanode.data.dir</name>
        <value>/Users/ispong/hadoop/data</value>
    </property>

    <property>
        <name>dfs.replication</name>
        <value>1</value>
    </property>

</configuration>

格式化hdfs

hdfs namenode -format

• 降低hadoop日志等级

code $HADOOP_HOME/etc/hadoop/log4j.properties

新增

log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR

配置yarn

code $HADOOP_HOME/etc/hadoop/mapred-site.xml

<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>

<configuration>
    <property>
        <name>mapreduce.framework.name</name>
        <value>yarn</value>
    </property>

    <property>
        <name>mapreduce.application.classpath</name>
        <value>$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/*:$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/lib/*</value>
    </property>

    <property>
        <name>mapreduce.jobhistory.webapp.address</name>
        <value>ispong-mac.local:19888</value>
    </property>
    
    <property>
        <name>mapreduce.jobhistory.address</name>
        <value>ispong-mac.local:10020</value>
    </property>
</configuration>

code $HADOOP_HOME/etc/hadoop/yarn-site.xml

容器最低推荐2GB

<?xml version="1.0"?>
<configuration>

    <property>
        <name>yarn.resourcemanager.address</name>
        <value>ispong-mac.local:8032</value>
    </property>
    <property>
        <name>yarn.nodemanager.aux-services</name>
        <value>mapreduce_shuffle</value>
    </property>
    <property>
        <name>yarn.nodemanager.env-whitelist</name>
        <value>JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,CLASSPATH_PREPEND_DISTCACHE,HADOOP_YARN_HOME,HADOOP_HOME,PATH,LANG,TZ,HADOOP_MAPRED_HOME</value>
    </property>
    <property>
	    <name>yarn.log-aggregation-enable</name>
	    <value>true</value>
    </property>
    <property>
	    <name>yarn.log.server.url</name>
	    <value>http://ispong-mac.local:19888/jobhistory/logs</value>
    </property>

    <property>
        <name>yarn.nodemanager.resource.memory-mb</name>
        <value>8192</value>
    </property>
    <property>
        <name>yarn.scheduler.maximum-allocation-mb</name>
        <value>4096</value>
    </property>
    <property>
        <name>yarn.scheduler.minimum-allocation-mb</name>
        <value>1024</value>
    </property>
    <property>
        <name>yarn.nodemanager.resource.cpu-vcores</name>
        <value>4</value>
    </property>
    <property>
        <name>yarn.scheduler.minimum-allocation-vcores</name>
        <value>1</value>
    </property>
    <property>
        <name>yarn.scheduler.maximum-allocation-vcores</name>
        <value>2</value>
    </property>
    
</configuration>

启动

echo $(ifconfig | grep "inet " | grep -v 127.0.0.1 | awk '{print $2}' | head -n 1)
# 替换ispong-mac.local为ip: 192.168.0.130
sed -i '' 's/ispong-mac.local/192.168.0.130/g' $HADOOP_HOME/etc/hadoop/core-site.xml
sed -i '' 's/ispong-mac.local/192.168.0.130/g' $HADOOP_HOME/etc/hadoop/yarn-site.xml
sed -i '' 's/ispong-mac.local/192.168.0.130/g' $HADOOP_HOME/etc/hadoop/mapred-site.xml

# 访问 http://192.168.0.130:9870
# 访问 http://192.168.0.130:8088
# 访问 http://192.168.0.130:19888
bash start-all.sh && mapred --daemon start historyserver

# hdfs dfsadmin -safemode leave        

# 关闭
bash stop-all.sh && mapred --daemon stop historyserver

卸载hadoop

bash stop-all.sh && mapred --daemon stop historyserver
rm -rf /Users/ispong/hadoop

🔗 Links

• hadoop docs