rancher 离线安装debezium
Last updated on November 22, 2024 pm
🧙 Questions
离线安装debezium
☄️ Ideas
前提
先安装kafka
下载镜像
docker pull --platform=linux/amd64 debezium/debezium-ui:1.9
docker pull --platform=linux/amd64 debezium/connect:2.7
docker save -o /Users/ispong/Downloads/debezium-ui-1.9-amd64.tar debezium/debezium-ui:1.9
docker save -o /Users/ispong/Downloads/debezium-connect-2.7-amd64.tar debezium/connect:2.7上传镜像解压
scp /Users/ispong/OneDrive/Downloads/docker/debezium-ui-1.9-amd64.tar ispong@192.168.115.104:/tmp/
scp /Users/ispong/OneDrive/Downloads/docker/debezium-connect-2.7-amd64.tar ispong@192.168.115.104:/tmp/
docker load -i /tmp/debezium-ui-1.9-amd64.tar
docker load -i /tmp/debezium-connect-2.7-amd64.tardocker启动debezium
BOOTSTRAP_SERVERS: kafka连接信息
docker run -it \
--name debezium \
-p 30192:8083 \
-e GROUP_ID=1 \
-e BOOTSTRAP_SERVERS=192.168.115.104:30196 \
-e CONFIG_STORAGE_TOPIC=my_connect_configs \
-e OFFSET_STORAGE_TOPIC=my_connect_offsets \
-e STATUS_STORAGE_TOPIC=my_connect_statuses \
debezium/connect:2.7rancher安装
Name: debezium
Image: docker.io/library/debezium:2.7
NodePort: 8083 -> 30192
env
BOOTSTRAP_SERVERS: 192.168.115.104:30196
CONFIG_STORAGE_TOPIC: my_connect_configs
OFFSET_STORAGE_TOPIC: my_connect_offsets
STATUS_STORAGE_TOPIC: my_connect_statuses启动后，会默认创建my_connect_configs、my_connect_offsets、my_connect_statuses这三个topic
docker启动debezium-ui
KAFKA_CONNECT_URIS: debezium的地址
docker run -it \
--name debezium-ui \
-p 30193:8080 \
-e KAFKA_CONNECT_URIS=http://192.168.115.104:30192 \
debezium/debezium-ui:1.9rancher安装
Name: debezium-ui
Image: docker.io/library/debezium-ui:1.9
NodePort: 8080 -> 30193
env
KAFKA_CONNECT_URIS: http://192.168.115.104:30192界面添加一个管道
Connector name: inventory-connector
Topic prefix: dbserver1
Cluster ID: 184054
database.hostname: mysql
database.port: 30102
database.user: root
database.password: ispong123
Kafka broker addresses: 192.168.115.104:30196
Database schema history topic name: schema-changes.inventory
Database filter: ispong_db手动调用接口创建
curl -i -X POST -H "Accept:application/json" -H "Content-Type:application/json" 192.168.168.116:30192/connectors/ -d '{
"name": "inventory-connector",
"config": {
"connector.class": "io.debezium.connector.mysql.MySqlConnector",
"tasks.max": "1",
"database.hostname": "192.168.168.116",
"database.port": "30191",
"database.user": "root",
"database.password": "ispong123",
"database.server.id": "184054",
"topic.prefix": "dbserver1",
"database.include.list": "ispong_db",
"schema.history.internal.kafka.bootstrap.servers": "192.168.168.116:30196",
"schema.history.internal.kafka.topic": "schemahistory.inventory"
}
}'会报错，需要手动创建schemahistory.inventory
2024-08-16T18:14:30.877270996+08:00 Caused by: org.apache.kafka.common.errors.UnsupportedVersionException: Creating topics with default partitions/replication factor are only supported in CreateTopicRequest version 4+. The following topics need values for partitions and replicas: [schemahistory.inventory]kafka-topics --create --zookeeper 192.168.115.104:30121/kafka --topic schemahistory.inventory --replication-factor 1 --partitions 1重新点击 restart
这时候会自动创建两个新的topic
dbserver1
dbserver1.ispong_db.users
测试监听
dbserver1.ispong_db.users是对表单独监听的管道
kafka-console-consumer --bootstrap-server 192.168.168.116:30196 --topic dbserver1.ispong_db.users{
"payload": {
"after": {
"age": 2,
"username": "1"
},
"before": null,
"op": "c",
"source": {
"connector": "mysql",
"db": "ispong_db",
"file": "binlog.000002",
"gtid": null,
"name": "dbserver1",
"pos": 6030,
"query": null,
"row": 0,
"sequence": null,
"server_id": 1,
"snapshot": "false",
"table": "users",
"thread": 34,
"ts_ms": 1723803728000,
"ts_ns": 1723803728000000000,
"ts_us": 1723803728000000,
"version": "2.7.1.Final"
},
"transaction": null,
"ts_ms": 1723803728538,
"ts_ns": 1723803728538751700,
"ts_us": 1723803728538751
},
"schema": {
"fields": [
{
"field": "before",
"fields": [
{
"field": "username",
"optional": true,
"type": "string"
},
{
"field": "age",
"optional": true,
"type": "int32"
}
],
"name": "dbserver1.ispong_db.users.Value",
"optional": true,
"type": "struct"
},
{
"field": "after",
"fields": [
{
"field": "username",
"optional": true,
"type": "string"
},
{
"field": "age",
"optional": true,
"type": "int32"
}
],
"name": "dbserver1.ispong_db.users.Value",
"optional": true,
"type": "struct"
},
{
"field": "source",
"fields": [
{
"field": "version",
"optional": false,
"type": "string"
},
{
"field": "connector",
"optional": false,
"type": "string"
},
{
"field": "name",
"optional": false,
"type": "string"
},
{
"field": "ts_ms",
"optional": false,
"type": "int64"
},
{
"default": "false",
"field": "snapshot",
"name": "io.debezium.data.Enum",
"optional": true,
"parameters": {
"allowed": "true,last,false,incremental"
},
"type": "string",
"version": 1
},
{
"field": "db",
"optional": false,
"type": "string"
},
{
"field": "sequence",
"optional": true,
"type": "string"
},
{
"field": "ts_us",
"optional": true,
"type": "int64"
},
{
"field": "ts_ns",
"optional": true,
"type": "int64"
},
{
"field": "table",
"optional": true,
"type": "string"
},
{
"field": "server_id",
"optional": false,
"type": "int64"
},
{
"field": "gtid",
"optional": true,
"type": "string"
},
{
"field": "file",
"optional": false,
"type": "string"
},
{
"field": "pos",
"optional": false,
"type": "int64"
},
{
"field": "row",
"optional": false,
"type": "int32"
},
{
"field": "thread",
"optional": true,
"type": "int64"
},
{
"field": "query",
"optional": true,
"type": "string"
}
],
"name": "io.debezium.connector.mysql.Source",
"optional": false,
"type": "struct"
},
{
"field": "transaction",
"fields": [
{
"field": "id",
"optional": false,
"type": "string"
},
{
"field": "total_order",
"optional": false,
"type": "int64"
},
{
"field": "data_collection_order",
"optional": false,
"type": "int64"
}
],
"name": "event.block",
"optional": true,
"type": "struct",
"version": 1
},
{
"field": "op",
"optional": false,
"type": "string"
},
{
"field": "ts_ms",
"optional": true,
"type": "int64"
},
{
"field": "ts_us",
"optional": true,
"type": "int64"
},
{
"field": "ts_ns",
"optional": true,
"type": "int64"
}
],
"name": "dbserver1.ispong_db.users.Envelope",
"optional": false,
"type": "struct",
"version": 2
}
}完整搭建
debezium-zookeeper
mkdir -p /data/debezium/zookeeper/logs
mkdir -p /data/debezium/zookeeper/data
mkdir -p /data/debezium/zookeeper/txns
chmod -R 777 /data/debezium/zookeeper
# data
/zookeeper/logs
/zookeeper/data
/zookeeper/txns
# image
docker.io/debezium-zookeeper:3.0.0.Final
# port
2181 -> 30151debezium-kafka
mkdir -p /data/debezium/kafka/logs
mkdir -p /data/debezium/kafka/data
chmod -R 777 /data/debezium/kafka
# data
/kafka/data
/kafka/logs
# image
debezium-kafka:3.0.0.Final
# port
9092 -> 30152
30155 -> 30155
# params
ZOOKEEPER_CONNECT : 172.23.39.236:30151
KAFKA_INTER_BROKER_LISTENER_NAME : INTERNAL
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP : EXTERNAL:PLAINTEXT,INTERNAL:PLAINTEXT
KAFKA_ADVERTISED_LISTENERS : EXTERNAL://${外网ip}:30155,INTERNAL://${service-name(debezium-kafka)}:9092
KAFKA_LISTENERS : EXTERNAL://0.0.0.0:30155,INTERNAL://0.0.0.0:9092
KAFKA_MAX_REQUEST_SIZE : 200000000debezium-connect
# image
debezium-connect:3.0.0.Final
# port
8083-> 30153
# params
GROUP_ID : 1
BOOTSTRAP_SERVERS : 172.23.39.236:30152
ZOOKEEPER_CONNECT : 172.23.39.236:30151
CONFIG_STORAGE_TOPIC : ispong_connect_configs
OFFSET_STORAGE_TOPIC : ispong_connect_offsets
STATUS_STORAGE_TOPIC : ispong_connect_statusesdebezium-ui
# image
debezium-ui:2.1.2.Final
# port
8080 -> 30154
# params
KAFKA_CONNECT_URIS : http://172.23.39.236:30153🔗 Links
rancher 离线安装debezium
https://ispong.isxcode.com/kubernetes/rancher/rancher 离线安装debezium/