docker 打包cdh镜像

Last updated on January 17, 2025 am

🧙 Questions

使用docker安装cdh

☄️ Ideas

抢占一台可以联网的服务器

推荐centos系统

docker run -itd \
  --privileged=true \
  --name=cdh \
  -e TZ=Asia/Shanghai \
  centos:centos7.9.2009 /sbin/init
下载cdh安装包
cd /tmp
nohup wget xxxxxxx >> download_cdh.log 2>&1 &
tail -f download_cdh.log
将安装包拷贝到容器
docker cp /tmp/cdh.zip cdh:/tmp/
进入容器
docker exec -it cdh /bin/bash
安装阿里软件源
mkdir -p /etc/yum.repos.d_bak
mv /etc/yum.repos.d/*.repo /etc/yum.repos.d_bak
curl -o /etc/yum.repos.d/CentOS-Base.repo https://mirrors.aliyun.com/repo/Centos-7.repo
yum clean all && yum makecache
前提安装软件和jdbc驱动
yum install python3 bind-utils postgresql-server java-1.8.0-openjdk-devel java-1.8.0-openjdk unzip openssh-clients vim net-tools ntp telnet -y
cd /tmp && unzip cdh.zip
cp /tmp/cdh/jdbc/mysql-connector-java-8.0.23.jar /usr/share/java/mysql-connector-java.jar
安装cdh-scm软件
cd /tmp/cdh/cdh6.2.0/cloudera-repos-6.2.0
yum install -y cloudera-manager-daemons-6.2.0-968826.el7.x86_64.rpm 
yum install -y cloudera-manager-server-6.2.0-968826.el7.x86_64.rpm
yum install -y cloudera-manager-server-db-2-6.2.0-968826.el7.x86_64.rpm
优化配置
echo never > /sys/kernel/mm/transparent_hugepage/defrag
echo never > /sys/kernel/mm/transparent_hugepage/enabled
安装ntp服务
vim /etc/ntp.conf
# server 0.centos.pool.ntp.org iburst
# server 1.centos.pool.ntp.org iburst
# server 2.centos.pool.ntp.org iburst
# server 3.centos.pool.ntp.org iburst
server 127.127.1.0 iburst
fudge 127.127.1.0 stratum 3
systemctl enable ntpd
systemctl start ntpd
systemctl status ntpd
ntpq -np
timedatectl status
打包新的cdh-cms镜像
# 删除多余的安装资源
rm -rf /tmp/cdh.zip /tmp/cdh /tmp/__MACOSX
exit
docker commit cdh cdh-cms:1.0
# docker stop cdh
# docker rm cdh
docker images -a
生成一个httpd的镜像
docker run -itd \
  --privileged=true \
  --name=cdh-httpd \
  -e TZ=Asia/Shanghai \
  centos:centos7.9.2009 /sbin/init
docker cp /tmp/cdh.zip cdh-httpd:/tmp/
docker exec -it cdh-httpd /bin/bash
mkdir -p /etc/yum.repos.d_bak
mv /etc/yum.repos.d/*.repo /etc/yum.repos.d_bak
curl -o /etc/yum.repos.d/CentOS-Base.repo https://mirrors.aliyun.com/repo/Centos-7.repo
yum clean all && yum makecache

安装httpd软件

yum install httpd unzip vim createrepo -y
cd /tmp && unzip cdh.zip
systemctl start httpd
vim /etc/httpd/conf/httpd.conf

修改端口号

#Listen 12.34.56.78:80
Listen 30108
systemctl restart httpd
systemctl enable httpd
systemctl status httpd
mkdir -p /var/www/html/cdh6_parcel
mkdir -p /var/www/html/cdh6_parcel/cm6
cp /tmp/cdh/cdh6.2.0/parcel-6.2.0/* /var/www/html/cdh6_parcel
cp /tmp/cdh/cdh6.2.0/cloudera-repos-6.2.0/* /var/www/html/cdh6_parcel/cm6/

生成cdh软件源

cd /var/www/html/cdh6_parcel/cm6/
createrepo .
rm -rf /tmp/cdh.zip /tmp/cdh /tmp/__MACOSX
exit
docker commit cdh-httpd cdh-httpd:1.0
docker images
启动一个httpd
docker run -itd \
  --privileged=true \
  --name=cdh-httpd-test \
  -e TZ=Asia/Shanghai \
  --net=host \
  cdh-httpd:1.0 /sbin/init
启动一个docker节点，打包cdh-node镜像
docker run -itd \
  --privileged=true \
  --name=cdh-agent \
  --add-host=cdh-master:172.19.189.246 \
  centos:centos7.9.2009 /sbin/init
docker cp /tmp/cdh.zip cdh-agent:/tmp/
docker exec -it cdh-agent /bin/bash
安装必须软件
mkdir -p /etc/yum.repos.d_bak
mv /etc/yum.repos.d/*.repo /etc/yum.repos.d_bak
curl -o /etc/yum.repos.d/CentOS-Base.repo https://mirrors.aliyun.com/repo/Centos-7.repo
yum clean all && yum makecache
yum install openssh-server vim net-tools iproute passwd openssl java-1.8.0-openjdk-devel java-1.8.0-openjdk openssh-clients ntp python3 bind-utils postgresql-server unzip telnet -y
优化配置
echo never > /sys/kernel/mm/transparent_hugepage/defrag
echo never > /sys/kernel/mm/transparent_hugepage/enabled
拷贝jdbc
cd /tmp && unzip cdh.zip
cp /tmp/cdh/jdbc/mysql-connector-java-8.0.23.jar /usr/share/java/mysql-connector-java.jar
安装ntp服务

指向cms

vim /etc/ntp.conf
# server 0.centos.pool.ntp.org iburst
# server 1.centos.pool.ntp.org iburst
# server 2.centos.pool.ntp.org iburst
# server 3.centos.pool.ntp.org iburst
server cdh-master iburst
systemctl enable ntpd
systemctl start ntpd
systemctl status ntpd
ntpq -np
timedatectl status
节点容器中启动ssh服务，修改端口号
vim /etc/ssh/sshd_config
Port 30115
PermitRootLogin yes
sed -i "s/#UseDNS.*/UseDNS no/g" /etc/ssh/sshd_config
ssh-keygen -t rsa -f /etc/ssh/ssh_host_rsa_key -N ''
ssh-keygen -t ecdsa -f /etc/ssh/ssh_host_ecdsa_key -N ''
ssh-keygen -t ed25519 -f /etc/ssh/ssh_host_ed25519_key  -N ''

启动ssh服务

/usr/sbin/sshd
# 修改密码
passwd root
# 设置密码 xCQUuqSidwV2SA_
在节点容器内添加cdh软件源
vim /etc/yum.repos.d/cloudera-manager.repo
[cloudera-manager]
name=Cloudera Manager 6.2.0
baseurl=http://cdh-master:30108/cdh6_parcel/cm6
gpgcheck=0
enabled=1

提前安装oracle-j2sdk1.8.x86_64和cloudera-manager-agent

yum-config-manager --enable cloudera-manager
yum clean all && yum makecache
yum -y install oracle-j2sdk1.8.x86_64 cloudera-manager-agent
容器节点打包新的镜像cdh-node
rm -rf /tmp/cdh.zip /tmp/cdh /tmp/__MACOSX
exit
docker commit cdh-agent cdh-node:1.0
# docker stop cdh-agent
# docker rm cdh-agent
docker images -a

使用镜像安装cdh

在服务器上使用docker安装一个mysql（外部/master主机）
mkdir -p /data/mysql/data
mkdir -p /data/mysql/conf.d

docker run \
  --name cdh-mysql \
  --privileged=true \
  --restart=always \
  -d \
  -p 30306:3306 \
  -e MYSQL_ROOT_PASSWORD=ispong123 \
  -v /data/mysql/data:/var/lib/mysql \
  -v /data/mysql/conf.d:/etc/mysql/conf.d \
  mysql:8.0

创建数据库

docker exec -it cdh-mysql bash
mysql -h localhost -u root -pispong123 -P 3306
CREATE DATABASE cdh_scm DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
CREATE DATABASE cdh_hive DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
CREATE DATABASE cdh_monitor DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
重启一个cdh-cms的真实容器
hostnamectl set-hostname cdh-master
vim /etc/hosts
#172.19.189.246 iZuf68w6yjxihq83rqzxeiZ iZuf68w6yjxihq83rqzxeiZ
172.19.189.246   cdh-master
docker run -itd \
  --privileged=true \
  --name=cdh-cms \
  --net=host \
  -e TZ=Asia/Shanghai \
  cdh-cms:1.0 /sbin/init
在cms容器中，初始化数据库，并启动cms服务
docker exec -it cdh-cms /bin/bash
/opt/cloudera/cm/schema/scm_prepare_database.sh -h cdh-master -P 30306 mysql cdh_scm root ispong123

启动cms服务

systemctl enable cloudera-scm-server
systemctl start cloudera-scm-server
systemctl status cloudera-scm-server
tail -f /var/log/cloudera-scm-server/cloudera-scm-server.log
访问cdh服务
启动一个node容器
docker run -itd \
  --privileged=true \
  --name=cdh-node \
  --net=host \
  -e TZ=Asia/Shanghai \
  cdh-node:1.0 /sbin/init
docker exec -it cdh-node /bin/bash
使用scm界面安装

使用内网ip
节点信息
hostname: cdh-master
port: 30115
user: root
password: xCQUuqSidwV2SA_

tail -n 500 /var/log/cloudera-scm-agent/cloudera-scm-agent.log
tail -f /var/log/cloudera-scm-agent/cloudera-scm-agent.log
DataNode Data Directory: /data/dfs/dn
NameNode Data Directories: /data/dfs/nn
HDFS Checkpoint Directories: /data/dfs/snn
NodeManager Local Directories: /data/yarn/nm
/data/dfs/dn
/data/dfs/nn
/data/dfs/snn
/data/yarn/nm
/var/lib/zookeeper
/var/lib/cloudera-service-monitor
/var/lib/cloudera-host-monitor
/var/local/kafka/data
/data/cloudera/parcels
/opt/cloudera/parcel-repo
压缩镜像导出
docker save -o /tmp/cdh-node-1.0-amd64.tar cdh-node:1.0
docker save -o /tmp/cdh-cms-1.0-amd64.tar cdh-cms:1.0
docker save -o /tmp/cdh-httpd-1.0-amd64.tar cdh-httpd:1.0

scp root@101.132.253.79:/tmp/cdh-node-1.0-amd64.tar ~/Downloads
scp root@101.132.253.79:/tmp/cdh-cms-1.0-amd64.tar ~/Downloads
scp root@101.132.253.79:/tmp/cdh-httpd-1.0-amd64.tar ~/Downloads
docker
docker 打包cdh镜像
https://ispong.isxcode.com/kubernetes/docker/docker 打包cdh镜像/
Author
ispong
Posted on
October 18, 2024
Licensed under