hive 集群安装

🧙 Questions

hive集群安装3.1.3

☄️ Ideas

前提

• 安装mysql

下载hive

cd /tmp
nohup wget https://archive.apache.org/dist/hive/hive-3.1.3/apache-hive-3.1.3-bin.tar.gz >> download_hive.log 2>&1 &
tail -f download_hive.log

scp /tmp/apache-hive-3.1.3-bin.tar.gz isxcode@main2:/tmp/
scp /tmp/apache-hive-3.1.3-bin.tar.gz isxcode@node1:/tmp/
scp /tmp/apache-hive-3.1.3-bin.tar.gz isxcode@node2:/tmp/
scp /tmp/apache-hive-3.1.3-bin.tar.gz isxcode@node3:/tmp/

创建安装文件夹

sudo mkdir -p /data/hive/
sudo chown -R isxcode:isxcode /data/hive/

解压并安装

tar -vzxf /tmp/apache-hive-3.1.3-bin.tar.gz -C /data/hive/
sudo ln -s /data/hive/apache-hive-3.1.3-bin /opt/hive

配置系统环境

sudo tee -a /etc/profile <<-'EOF'
export HIVE_HOME=/opt/hive 
export PATH=$PATH:$HIVE_HOME/bin 
EOF
source /etc/profile

检测安装

hive --version

安装mysql驱动

cd /tmp
wget https://dev.mysql.com/get/Downloads/Connector-J/mysql-connector-java-8.0.23.tar.gz
tar vzxf mysql-connector-java-8.0.23.tar.gz
cp /tmp/mysql-connector-java-8.0.23/mysql-connector-java-8.0.23.jar /opt/hive/lib/

配置文件

mkdir -p /data/hive/tmp
mkdir -p /data/hive/logs

配置一次，后面的分发文件

cp /opt/hive/conf/hive-log4j2.properties.template /opt/hive/conf/hive-log4j2.properties
vim /opt/hive/conf/hive-log4j2.properties

property.hive.log.dir=/opt/hive/logs

cp /opt/hive/conf/hive-env.sh.template /opt/hive/conf/hive-env.sh
tee -a /opt/hive/conf/hive-env.sh <<-'EOF'
export HADOOP_HOME=/opt/hadoop
export HIVE_CONF_DIR=/opt/hive/conf
EOF

在mysql中创建数据库

docker exec -it isxcode-mysql /bin/bash
mysql -h localhost -u root -pispong123 -P 30306

CREATE USER IF NOT EXISTS 'ispong'@'%' identified by 'ispong123';
CREATE DATABASE IF NOT EXISTS hive_db DEFAULT CHARSET utf8mb4;
GRANT ALL PRIVILEGES on hive_db.* to 'ispong'@'%';
FLUSH PRIVILEGES;
exit;

配置hive-site

Note: [3215,9] 3215行删掉描述 一定要删除！！！

cp /opt/hive/conf/hive-default.xml.template /opt/hive/conf/hive-site.xml
vim /opt/hive/conf/hive-site.xml

<configuration>

    <!-- ispong :) mysql url -->
    <property>
        <name>javax.jdo.option.ConnectionURL</name>
        <value>jdbc:mysql://node3:30201/hive_db</value>
    </property>
        
    <!-- ispong :) mysql driver -->
    <property>
        <name>javax.jdo.option.ConnectionDriverName</name>
        <value>com.mysql.cj.jdbc.Driver</value>
    </property>
    
    <!-- ispong :) mysql username -->
    <property>
        <name>javax.jdo.option.ConnectionUserName</name>
        <value>root</value>
    </property>
    
    <!-- ispong :) mysql password -->
    <property>
        <name>javax.jdo.option.ConnectionPassword</name>
        <value>isxcode123</value>
    </property>

    <!-- ispong :) beeline username -->
    <property>
        <name>hive.server2.thrift.client.user</name>
        <value>isxcode</value>
    </property>
    
    <!-- ispong :) beeline password -->
    <property>
        <name>hive.server2.thrift.client.password</name>
        <value>isxcode123</value>
    </property>

    <!-- 直接新增 -->
    <property>
      <name>system:java.io.tmpdir</name>
      <value>/data/hive/tmp</value>
    </property>

    <!-- 直接新增,安装用户  -->
    <property>
      <name>system:user.name</name>
      <value>isxcode</value>
    </property>

    <!-- ispong :) metastore 默认端口号（9083） -->
    <property>
        <name>hive.metastore.port</name>
        <value>9083</value>
    </property>

    <!-- ispong :) hive:jdbc 默认端口号（10000） -->
    <property>
        <name>hive.server2.thrift.port</name>
        <value>10000</value>
    </property>

    <property>
      <name>hive.server2.zookeeper.publish.configs</name>
      <value>true</value>
      <description>Whether we should publish HiveServer2's configs to ZooKeeper.</description>
    </property>

    <property>
      <name>hive.zookeeper.client.port</name>
      <value>2181</value>
      <description>
        The port of ZooKeeper servers to talk to.
        If the list of Zookeeper servers specified in hive.zookeeper.quorum
        does not contain port numbers, this value is used.
      </description>
    </property>

    <property>
      <name>hive.zookeeper.quorum</name>
      <value>main1:2181,main2:2181,node1:2181,node2:2181,node3:2181</value>
      <description>
        List of ZooKeeper servers to talk to. This is needed for:
        1. Read/write locks - when hive.lock.manager is set to
        org.apache.hadoop.hive.ql.lockmgr.zookeeper.ZooKeeperHiveLockManager,
        2. When HiveServer2 supports service discovery via Zookeeper.
        3. For delegation token storage if zookeeper store is used, if
        hive.cluster.delegation.token.store.zookeeper.connectString is not set
        4. LLAP daemon registry service
        5. Leader selection for privilege synchronizer
      </description>
    </property>

    <property>
      <name>hive.server2.zookeeper.namespace</name>
      <value>isxcode-hiveserver2</value>
      <description>The parent node in ZooKeeper used by HiveServer2 when supporting dynamic service discovery.</description>
    </property>

    <property>
      <name>hive.server2.support.dynamic.service.discovery</name>
      <value>true</value>
      <description>Whether HiveServer2 supports dynamic service discovery for its clients. To support this, each instance of HiveServer2 currently uses ZooKeeper to register itself, when it is brought up. JDBC/ODBC clients should use the ZooKeeper ensemble: hive.zookeeper.quorum in their connection string.</description>
    </property>

</configuration>

替换guava版本

每个服务器都要做

rm /opt/hive/lib/guava-*.jar
cp /opt/hadoop/share/hadoop/hdfs/lib/guava-*.jar /opt/hive/lib/

分发配置

scp -r /opt/hive/conf isxcode@main2:/opt/hive/
scp -r /opt/hive/conf isxcode@node1:/opt/hive/
scp -r /opt/hive/conf isxcode@node2:/opt/hive/
scp -r /opt/hive/conf isxcode@node3:/opt/hive/

初始化mysql

执行一次就好

schematool -dbType mysql -initSchema

启动hive

nohup /opt/hive/bin/hive –service metastore >> /data/hive/logs/metastore.log 2>&1 &
tail -f /data/hive/logs/metastore.log
不推荐，Restart=always

sudo vim /usr/lib/systemd/system/hive_metastore.service

[Unit]
Description=Hive Metastore Service
After=hdfs_namenode.service

[Service]
Environment=JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk
Type=forking
WorkingDirectory=/opt/hive/logs
PermissionsStartOnly=true
ExecStart=/opt/hive/bin/hive --service metastore
ExecStop=pwd
ExecReload=pwd
KillMode=none
# Restart=always
User=isxcode
Group=isxcode

[Install]
WantedBy=multi-user.target

sudo systemctl daemon-reload
sudo systemctl enable hive_metastore

sudo systemctl start hive_metastore
sudo systemctl status hive_metastore
sudo netstat -ntpl | grep 9083

nohup hive –service hiveserver2 >> /data/hive/logs/hiveserver2.log 2>&1 &
tail -f /data/hive/logs/hiveserver2.log
不推荐，Restart=always

sudo vim /usr/lib/systemd/system/hive_hiveserver2.service

[Unit]
Description=Hive Server2 Service
After=hive_metastore.service

[Service]
Environment=JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk
Type=forking
WorkingDirectory=/opt/hive/logs
PermissionsStartOnly=true
ExecStart=/opt/hive/bin/hive --service hiveserver2
ExecStop=pwd
ExecReload=pwd
KillMode=none
# Restart=always
User=isxcode
Group=isxcode

[Install]
WantedBy=multi-user.target

多等几分钟再查看端口状态

sudo systemctl daemon-reload
sudo systemctl enable hive_hiveserver2

sudo systemctl start hive_hiveserver2
sudo systemctl status hive_hiveserver2
sudo netstat -ntpl | grep 10000

测试连接

beeline
!connect jdbc:hive2://main1:2181,main2:2181,node1:2181,node2:2181,node3:2181/;serviceDiscoveryMode=zooKeeper;zooKeeperNamespace=isxcode-hiveserver2

• http://main1:10002/
• http://main2:10002/
• http://node1:10002/
• http://node2:10002/
• http://node3:10002/

🔗 Links

• hive website
• ha install