flink 提交到k8s中

🧙 Questions

☄️ Ideas

容器中的yml

apiVersion: v1
kind: Pod
metadata:
  creationTimestamp: "2023-08-04T08:24:47Z"
  generateName: my-first-application-cluster-69c95dc944-
  labels:
    app: my-first-application-cluster
    component: jobmanager
    pod-template-hash: 69c95dc944
    type: flink-native-kubernetes
  name: my-first-application-cluster-69c95dc944-zqtrd
  namespace: default
  ownerReferences:
    - apiVersion: apps/v1
      blockOwnerDeletion: true
      controller: true
      kind: ReplicaSet
      name: my-first-application-cluster-69c95dc944
      uid: 372ed005-e0ac-4150-8da6-f6fe7054c9a8
  resourceVersion: "1227957"
  uid: 51fb989f-3dc6-42df-983e-fe8ec35e7f2f
spec:
  containers:
    - args:
        - native-k8s
        - $JAVA_HOME/bin/java -classpath $FLINK_CLASSPATH -Xmx1073741824 -Xms1073741824
          -XX:MaxMetaspaceSize=268435456 -Dlog.file=/opt/flink/log/jobmanager.log -Dlogback.configurationFile=file:/opt/flink/conf/logback-console.xml
          -Dlog4j.configuration=file:/opt/flink/conf/log4j-console.properties -Dlog4j.configurationFile=file:/opt/flink/conf/log4j-console.properties
          org.apache.flink.kubernetes.entrypoint.KubernetesApplicationClusterEntrypoint
          -D jobmanager.memory.off-heap.size=134217728b -D jobmanager.memory.jvm-overhead.min=201326592b
          -D jobmanager.memory.jvm-metaspace.size=268435456b -D jobmanager.memory.heap.size=1073741824b
          -D jobmanager.memory.jvm-overhead.max=201326592b
      command:
        - /docker-entrypoint.sh
      env:
        - name: _POD_IP_ADDRESS
          valueFrom:
            fieldRef:
              apiVersion: v1
              fieldPath: status.podIP
      image: apache/flink:1.12.7-scala_2.12
      imagePullPolicy: IfNotPresent
      name: flink-job-manager
      ports:
        - containerPort: 8081
          name: rest
          protocol: TCP
        - containerPort: 6123
          name: jobmanager-rpc
          protocol: TCP
        - containerPort: 6124
          name: blobserver
          protocol: TCP
      resources:
        limits:
          cpu: "1"
          memory: 1600Mi
        requests:
          cpu: "1"
          memory: 1600Mi
      terminationMessagePath: /dev/termination-log
      terminationMessagePolicy: File
      volumeMounts:
        - mountPath: /opt/flink/conf
          name: flink-config-volume
        - mountPath: /var/run/secrets/kubernetes.io/serviceaccount
          name: kube-api-access-g9wnl
          readOnly: true
  dnsPolicy: ClusterFirst
  enableServiceLinks: true
  nodeName: docker-desktop
  preemptionPolicy: PreemptLowerPriority
  priority: 0
  restartPolicy: Always
  schedulerName: default-scheduler
  securityContext: {}
  serviceAccount: default
  serviceAccountName: default
  terminationGracePeriodSeconds: 30
  tolerations:
    - effect: NoExecute
      key: node.kubernetes.io/not-ready
      operator: Exists
      tolerationSeconds: 300
    - effect: NoExecute
      key: node.kubernetes.io/unreachable
      operator: Exists
      tolerationSeconds: 300
  volumes:
    - configMap:
        defaultMode: 420
        items:
          - key: logback-console.xml
            path: logback-console.xml
          - key: log4j-console.properties
            path: log4j-console.properties
          - key: flink-conf.yaml
            path: flink-conf.yaml
        name: flink-config-my-first-application-cluster
      name: flink-config-volume
    - name: kube-api-access-g9wnl
      projected:
        defaultMode: 420
        sources:
          - serviceAccountToken:
              expirationSeconds: 3607
              path: token
          - configMap:
              items:
                - key: ca.crt
                  path: ca.crt
              name: kube-root-ca.crt
          - downwardAPI:
              items:
                - fieldRef:
                    apiVersion: v1
                    fieldPath: metadata.namespace
                  path: namespace
status:
  conditions:
    - lastProbeTime: null
      lastTransitionTime: "2023-08-04T08:24:47Z"
      status: "True"
      type: Initialized
    - lastProbeTime: null
      lastTransitionTime: "2023-08-04T08:25:05Z"
      status: "True"
      type: Ready
    - lastProbeTime: null
      lastTransitionTime: "2023-08-04T08:25:05Z"
      status: "True"
      type: ContainersReady
    - lastProbeTime: null
      lastTransitionTime: "2023-08-04T08:24:47Z"
      status: "True"
      type: PodScheduled
  containerStatuses:
    - containerID: docker://8243c39483184d02316f557518956ad391f57153191078ac3e745a02b67a009b
      image: apache/flink:1.12.7
      imageID: docker-pullable://apache/flink@sha256:90bcdc6e6abba4fe118c100229b5ee69386d6ad13a42e871828398683055a65c
      lastState:
        terminated:
          containerID: docker://0b439a080ad6b4473d7bcdb96843a75d69654a049c0901cf4035b5903b30b489
          exitCode: 1
          finishedAt: "2023-08-04T08:25:03Z"
          reason: Error
          startedAt: "2023-08-04T08:24:52Z"
      name: flink-job-manager
      ready: true
      restartCount: 1
      started: true
      state:
        running:
          startedAt: "2023-08-04T08:25:04Z"
  hostIP: 192.168.65.4
  phase: Running
  podIP: 10.1.0.121
  podIPs:
    - ip: 10.1.0.121
  qosClass: Guaranteed
  startTime: "2023-08-04T08:24:47Z"

vim /Users/ispong/pod-init.yaml

apiVersion: v1
kind: Pod
metadata:
  name: host-pod
spec:
  containers:
    - name: flink-main-container
      # ports:
      #   - containerPort: 8081
      #     hostPort: 8081
      #     protocol: TCP
      #   - containerPort: 6123
      #     hostPort: 6123
      #     protocol: TCP
      #   - containerPort: 6124
      #     hostPort: 6124
      #     protocol: TCP
      volumeMounts:
        - name: file1
          mountPath: /opt/flink/examples/streaming/flink-execute-sql-plugin.jar
        - name: mysql-jdbc
          mountPath: /opt/flink/lib/mysql-connector-j-8.1.0.jar
        - name: mysql-cdc
          mountPath: /opt/flink/lib/flink-connector-mysql-cdc-2.4.2.jar
        - name: log-cdc
          mountPath: /opt/flink/lib/flink-out-connector.jar
        - name: flink-jdbc
          mountPath: /opt/flink/lib/flink-connector-jdbc-3.1.1-1.17.jar
        # - name: flink-yml
        #   mountPath: /opt/flink/log/flink-conf.yaml
        # - name: log4j
        #   mountPath: /opt/flink/log/log4j.properties
        # - name: logback
        #   mountPath: /opt/flink/log/logback.xml
  hostAliases:
    - ip: "192.168.34.113"
      hostnames:
        - "isxcode"
  volumes:
    - name: file1
      hostPath:
        path: /Users/ispong/code/flink-yun/flink-yun-vip/flink-yun-plugins/flink-sql-plugin/build/libs/flink-execute-sql-plugin.jar
    - name: mysql-cdc
      hostPath:
        path: /Users/ispong/code/flink-yun/flink-yun-dist/flink-min/lib/flink-connector-mysql-cdc-2.4.2.jar
    - name: log-cdc
      hostPath:
        path: /Users/ispong/code/flink-yun/flink-yun-dist/flink-min/lib/flink-out-connector.jar
    - name: mysql-jdbc
      hostPath:
        path: /Users/ispong/code/flink-yun/flink-yun-dist/flink-min/lib/mysql-connector-j-8.1.0.jar
    - name: flink-jdbc
      hostPath:
        path: /Users/ispong/code/flink-yun/flink-yun-dist/flink-min/lib/flink-connector-jdbc-3.1.1-1.17.jar
    # - name: flink-yml
    #   hostPath:
    #     path: /Users/ispong/code/flink-yun/flink-yun-dist/flink-min/conf/flink-conf.yaml
    # - name: log4j
    #   hostPath:
    #     path: /Users/ispong/code/flink-yun/flink-yun-dist/flink-min/conf/log4j.properties
    # - name: logback
    #   hostPath:
    #     path: /Users/ispong/code/flink-yun/flink-yun-dist/flink-min/conf/logback.xml

提交命令

# 创建空间名称
kubectl create namespace zhiliuyun-space
# 创建账号
kubectl create serviceaccount zhiliuyun -n zhiliuyun-space
# 绑定权限
kubectl create clusterrolebinding flink-role --clusterrole=edit --serviceaccount=zhiliuyun-space:zhiliuyun --namespace=zhiliuyun-space
# 检查是否成功
kubectl auth can-i create pods --as=system:serviceaccount:zhiliuyun-space:zhiliuyun

# 查看pods
kubectl get pods -n zhiliuyun-space -o wide

# 查看pods日志
kubectl logs -f zhiliuyun-cluster-1712720389574-7887d46ff9-w4brf -n zhiliuyun-space

# 查看deployment
kubectl get deployment -n zhiliuyun-space -o wide

# 查看deployment日志
kubectl describe deployment zhiliuyun-cluster-1712720389574  -n zhiliuyun-space

# 关闭deployment
kubectl scale deployment zhiliuyun-cluster --replicas=0 -n zhiliuyun-space

# 删除deployment
kubectl delete deployment zhiliuyun-cluster-1712763933629 -n zhiliuyun-space
kubectl delete deployment flink-cluster-9617446f3c3d102f5ced4ef5bbfbf23

vim /Users/ispong/Downloads/pod-template.yaml

containers.name 一定要匹配上

apiVersion: v1
kind: Pod
metadata:
  name: host-pod
spec:
  initContainers:
    - name: artifacts-fetcher
      image: busybox:latest
      command: [ 'cp','-r','/conf','/opt/flink/conf' ]
      volumeMounts:
        - name: flink-config
          mountPath: /conf
  containers:
    - name: flink-main-container
      volumeMounts:
        - name: app
          mountPath: /opt/flink/examples/app.jar
        - name: flink-connector-jdbc
          mountPath: /opt/flink/lib/flink-connector-jdbc-3.1.2-1.18.jar
        - name: mysql-driver
          mountPath: /opt/flink/lib/mysql-connector-j-8.1.0.jar
        - name: flink-log
          mountPath: /opt/flink/log
        - name: flink-config
          mountPath: /opt/flink/examples/conf2
  volumes:
    - name: app
      hostPath:
        path: /Users/ispong/Downloads/flink-sql-execute-plugin.jar
    - name: flink-connector-jdbc
      hostPath:
        path: /Users/ispong/Downloads/flink-connector-jdbc-3.1.2-1.18.jar
    - name: mysql-driver
      hostPath:
        path: /Users/ispong/Downloads/mysql-connector-j-8.1.0.jar
    - name: flink-log
      hostPath:
        path: /Users/ispong/Downloads/log
        type: Directory
    - name: flink-config
      hostPath:
        path: /Users/ispong/isxcode/flink-yun/flink-yun-dist/flink-min/conf

vim /Users/ispong/Downloads/pod-template.yaml

apiVersion: v1
kind: Pod
metadata:
  name: host-pod
spec:
  containers:
    - name: flink-main-container
      volumeMounts:
        - name: app
          mountPath: /opt/flink/examples/streaming/app.jar
        - name: flink-connector-jdbc
          mountPath: /opt/flink/lib/flink-connector-jdbc-3.1.2-1.18.jar
        - name: mysql-driver
          mountPath: /opt/flink/lib/mysql-connector-j-8.1.0.jar
        - name: flink-log
          mountPath: /zhiliuyun/log
  hostAliases:
    - ip: "210.87.110.108"
      hostnames:
        - "isxcode"
  volumes:
    - name: app
      hostPath:
        path: /Users/ispong/Downloads/flink-sql-execute-plugin.jar
    - name: flink-connector-jdbc
      hostPath:
        path: /Users/ispong/Downloads/flink-connector-jdbc-3.1.2-1.18.jar
    - name: mysql-driver
      hostPath:
        path: /Users/ispong/Downloads/mysql-connector-j-8.1.0.jar
    - name: flink-log
      hostPath:
        path: /Users/ispong/Downloads/log
        type: Directory

apiVersion: v1
kind: Pod
metadata:
  name: pod-template
spec:
  containers:
    - name: flink-main-container
      volumeMounts:
        - name: app
          mountPath: /opt/flink/examples/streaming/app.jar
        - name: flink-connector-jdbc
          mountPath: /opt/flink/lib/flink-connector-jdbc-3.1.2-1.18.jar
        - name: mysql-driver
          mountPath: /opt/flink/lib/mysql-connector-j-8.1.0.jar
        - name: flink-log
          mountPath: /log
        - name: checkpoint-pvc
          mountPath: /opt/flink/checkpoints
        - name: savepoint-pvc
          mountPath: /opt/flink/savepoints
  hostAliases:
    - ip: "210.87.110.108"
      hostnames:
        - "isxcode"
  volumes:
    - name: app
      hostPath:
        path: /Users/ispong/Downloads/flink-sql-execute-plugin.jar
    - name: flink-connector-jdbc
      hostPath:
        path: /Users/ispong/Downloads/flink-connector-jdbc-3.1.2-1.18.jar
    - name: mysql-driver
      hostPath:
        path: /Users/ispong/Downloads/mysql-connector-j-8.1.0.jar
    - name: flink-log
      hostPath:
        path: /Users/ispong/Downloads/log
    - name: checkpoint-pvc
      persistentVolumeClaim:
        claimName: flink-checkpoint
    - name: savepoint-pvc
      persistentVolumeClaim:
        claimName: flink-savepoint

apiVersion: v1
kind: Pod
metadata:
  name: pod-template
spec:
  containers:
    - name: flink-main-container
      volumeMounts:
        - name: app
          mountPath: /opt/flink/examples/app.jar
        - name: flink-connector-jdbc
          mountPath: /opt/flink/lib/flink-connector-jdbc-3.1.2-1.18.jar
        - name: mysql-driver
          mountPath: /opt/flink/lib/mysql-connector-j-8.1.0.jar
        - name: flink-log
          mountPath: /log
  hostAliases:
    - ip: "47.92.152.18"
      hostnames:
        - "isxcode"
  volumes:
    - name: app
      hostPath:
        path: /Users/ispong/isxcode/flink-yun/flink-yun-plugins/flink-sql-execute-plugin/build/libs/flink-sql-execute-plugin.jar
    - name: flink-connector-jdbc
      hostPath:
        path: /Users/ispong/isxcode/flink-yun/resources/cdc/flink-connector-jdbc-3.1.2-1.18.jar
    - name: mysql-driver
      hostPath:
        path: /Users/ispong/isxcode/flink-yun/resources/jdbc/system/mysql-connector-j-8.1.0.jar
    - name: flink-log
      hostPath:
        path: /Users/ispong/isxcode/flink-yun/flink-yun-dist/flink-min/log

🔗 Links

• flink docs
• flink docs history
• flink k8s application deploy
• doc1