spark mac本地调试至轻云

Last updated on September 15, 2024 pm

🧙 Questions

在mac中调试至轻云

☄️ Ideas

前提

安装java
安装gradle
安装pnpm
安装isx
开启mac远程登录

下载编译代码

至轻云项目: spark-yun

# 下载代码
isx clone
# 安装依赖
isx install
# 构建代码
isx package

启动项目

或者使用idea运行项目

# 清理项目缓存文件
isx clean
# 启动项目
isx backend

http://localhost:8080
用户: admin
密码: admin123

创建项目基础信息

添加用户
添加租户
下载上传许可证

standalone调试

添加集群
集群: standalone
类型: standalone
- 节点: localhost
- Host: localhost
- 默认安装spark
- 用户名: ispong
- 密码:
节点检测
节点安装
查看本地启动的spark: http://localhost:8081

偷梁换柱

bash /Users/ispong/zhiqingyun-agent/bin/stop.sh
# 然后使用idea启动至轻云代理

创建作业

短时间任务sparksql

CREATE TEMPORARY VIEW from_table
USING org.apache.spark.sql.jdbc
OPTIONS (
    driver 'com.mysql.cj.jdbc.Driver',
    url 'jdbc:mysql://47.92.152.18:40004/isxcode_db',
    user 'root',
    password 'Zhiqingyun@isxcode',
    dbtable 'users'
);

select count(1) from from_table;

长时间任务sparksql

CREATE TEMPORARY VIEW to_table
USING org.apache.spark.sql.jdbc
OPTIONS (
    driver 'com.mysql.cj.jdbc.Driver',
    url 'jdbc:mysql://47.92.152.18:40004/isxcode_db',
    user 'root',
    password 'Zhiqingyun@isxcode',
    dbtable 'supplier'
);

select count(1) from to_table;

k8s调试

启动k8s服务

# 检测服务是否启动
kubectl cluster-info

# 拉取镜像
docker pull apache/spark:v3.1.3
# 创建zhiqingyun-space空间
kubectl create namespace zhiqingyun-space
# 创建账号zhiqingyun
kubectl create serviceaccount zhiqingyun -n zhiqingyun-space
# 构建权限spark-role
kubectl create clusterrolebinding spark-role --clusterrole=edit --serviceaccount=zhiqingyun-space:zhiqingyun --namespace=zhiqingyun-space

添加集群
- 集群: k8s
- 类型: kubernetes
- 节点: localhost
- Host: localhost
- 用户名: ispong
- 密码:
节点检测
节点安装

偷梁换柱

bash /Users/ispong/zhiqingyun-agent/bin/stop.sh
# idea启动至流云代理

yarn调试

启动hadoop服务

start-all.sh

查看yarn服务: http://ispong-mac.local:8088
添加集群
- 集群: yarn
- 类型: yarn
- 节点: localhost
- Host: localhost
- 用户名: ispong
- 密码:
节点检测
节点安装

偷梁换柱

bash /Users/ispong/zhiliuyun-agent/bin/stop.sh
# idea启动至流云代理

调试代码

sparkLauncher.addAppArgs(Base64.getEncoder()
               .encodeToString(yagExecuteWorkReq.getPluginReq() == null ? yagExecuteWorkReq.getArgsStr().getBytes()
                   : JSON.toJSONString(yagExecuteWorkReq.getPluginReq()).getBytes()));

修改plugin


args = new String[]{"eyJhZ2VudFR5cGUiOiJzdGFuZGFsb25lIiwiY29udGFpbmVyUG9ydCI6MCwiY3N2RmlsZU5hbWUiOiJzeV8xODIyOTQ0MTI0OTE2MTk1MzI4LmNzdiIsImNzdkZpbGVQYXRoIjoiL1VzZXJzL2lzcG9uZy96aGlxaW5neXVuLWFnZW50L2ZpbGUvc3lfMTgyMjk0NDEyNDkxNjE5NTMyOC5jc3YiLCJleGNlbFN5bmNDb25maWciOnsiY29sdW1uTWFwIjpbeyJzb3VyY2UiOiJhY2NvdW50X2lkIiwidGFyZ2V0IjoidXNlcm5hbWUifV0sImZpbGVQYXR0ZXJuIjoiIiwiZmlsZVJlcGxhY2UiOmZhbHNlLCJoYXNIZWFkZXIiOnRydWUsIm92ZXJNb2RlIjoiT1ZFUldSSVRFIiwicXVlcnlDb25kaXRpb24iOiIiLCJzb3VyY2VGaWxlSWQiOiJzeV8xODIyOTQ0MTI0OTE2MTk1MzI4Iiwic291cmNlVGFibGVDb2x1bW4iOlt7ImNvZGUiOiJsb2d0aW1lIiwic3FsIjoiIiwidHlwZSI6IlN0cmluZyJ9LHsiY29kZSI6ImFjY291bnRfaWQiLCJzcWwiOiIiLCJ0eXBlIjoiU3RyaW5nIn0seyJjb2RlIjoiaXAiLCJzcWwiOiIiLCJ0eXBlIjoiU3RyaW5nIn1dLCJ0YXJnZXREQklkIjoic3lfMTgxODEzMDYxNzk2NDA0MDE5MiIsInRhcmdldERCVHlwZSI6IkhJVkUiLCJ0YXJnZXREYXRhYmFzZSI6eyJkYlRhYmxlIjoidXNlcnMiLCJkcml2ZXIiOiJvcmcuYXBhY2hlLmhpdmUuamRiYy5IaXZlRHJpdmVyIiwidXJsIjoiamRiYzpoaXZlMjovLzguMTQyLjE0NC4xMDA6MzAxMTUvaXNwb25nX2RiIiwidXNlciI6ImRlaG9vcCJ9LCJ0YXJnZXRUYWJsZSI6InVzZXJzIiwidGFyZ2V0VGFibGVDb2x1bW4iOlt7ImNvZGUiOiJ1c2VybmFtZSIsInR5cGUiOiJTVFJJTkcifSx7ImNvZGUiOiJhZ2UiLCJ0eXBlIjoiSU5UIn1dLCJ3b3JrSWQiOiJzeV8xODIyODU5MzI2NDQwODUzNTA0In0sImZ1bmNJbmZvTGlzdCI6W10sInNwYXJrQ29uZmlnIjp7InNwYXJrLmV4ZWN1dG9yLm1lbW9yeSI6IjJnIiwic3Bhcmsuc3FsLnBhcnF1ZXQud3JpdGVMZWdhY3lGb3JtYXQiOiJ0cnVlIiwic3BhcmsuZHJpdmVyLm1lbW9yeSI6IjFnIiwic3Bhcmsuc3FsLnBhcnF1ZXQuZGF0ZXRpbWVSZWJhc2VNb2RlSW5Xcml0ZSI6IkxFR0FDWSIsInNwYXJrLmRyaXZlci5jb3JlcyI6IjEiLCJzcGFyay5zcWwuc3RvcmVBc3NpZ25tZW50UG9saWN5IjoiTEVHQUNZIiwic3BhcmsuZXhlY3V0b3IuY29yZXMiOiIxIiwic3Bhcmsuc3FsLnBhcnF1ZXQuaW50OTZSZWJhc2VNb2RlSW5SZWFkIjoiTEVHQUNZIiwic3Bhcmsuc3FsLnBhcnF1ZXQuZGF0ZXRpbWVSZWJhc2VNb2RlSW5SZWFkIjoiTEVHQUNZIiwic3Bhcmsuc3FsLmxlZ2FjeS50aW1lUGFyc2VyUG9saWN5IjoiTEVHQUNZIiwic3Bhcmsuc3FsLnBhcnF1ZXQuaW50OTZSZWJhc2VNb2RlSW5Xcml0ZSI6IkxFR0FDWSIsInNwYXJrLnNxbC5hdXRvQnJvYWRjYXN0Sm9pblRocmVzaG9sZCI6Ii0xIiwic3BhcmsuZXhlY3V0b3IuaW5zdGFuY2VzIjoiMSIsInNwYXJrLmV4ZWN1dG9yLmV4dHJhSmF2YU9wdGlvbnMiOiItRGZpbGUuZW5jb2Rpbmc9dXRmLTgiLCJzcGFyay5jb3Jlcy5tYXgiOiIxIiwic3BhcmsuZHJpdmVyLmV4dHJhSmF2YU9wdGlvbnMiOiItRGZpbGUuZW5jb2Rpbmc9dXRmLTgiLCJzcGFyay5zcWwucGFycXVldC5lbmFibGVWZWN0b3JpemVkUmVhZGVyIjoiZmFsc2UifSwic3luY1J1bGUiOnsibnVtQ29uY3VycmVuY3kiOjEsIm51bVBhcnRpdGlvbnMiOjEsInNldE1vZGUiOiJTSU1QTEUiLCJzcWxDb25maWdKc29uIjoibnVsbCJ9fQ=="};
ï
    public static SparkConf initSparkConf(Map<String, String> sparkConfig) {
        SparkConf conf = new SparkConf();
//        if (sparkConfig != null) {
//            for (Map.Entry<String, String> entry : sparkConfig.entrySet()) {
//                conf.set(entry.getKey(), entry.getValue());
//            }
//        }
        conf.set("spark.master", "local[*]");
        return conf;
    }