1"use strict";(globalThis.webpackChunkdocs_site=globalThis.webpackChunkdocs_site||[]).push([[19670],{73254(e,n,o){o.r(n),o.d(n,{assets:()=>s,contentTitle:()=>r,default:()=>d,frontMatter:()=>t,metadata:()=>l,toc:()=>c});const l=JSON.parse('{"id":"Ecosystem/FlinkOnVolcano","title":"Flink on Volcano","description":"Flink introduction","source":"@site/versioned_docs/version-v1.13.0/Ecosystem/FlinkOnVolcano.md","sourceDirName":"Ecosystem","slug":"/Ecosystem/FlinkOnVolcano","permalink":"/docs/v1.13.0/Ecosystem/FlinkOnVolcano","draft":false,"unlisted":false,"editUrl":"https://github.com/volcano-sh/website/tree/master/versioned_docs/version-v1.13.0/Ecosystem/FlinkOnVolcano.md","tags":[],"version":"v1.13.0","lastUpdatedAt":1781163901000,"sidebarPosition":1,"frontMatter":{"title":"Flink on Volcano","sidebar_position":1},"sidebar":"docsSidebar","previous":{"title":"GPU Virtualization","permalink":"/docs/v1.13.0/KeyFeatures/GPUVirtualization"},"next":{"title":"Kubeflow on Volcano","permalink":"/docs/v1.13.0/Ecosystem/KubeflowOnVolcano"}}');var a=o(74848),i=o(28453);const t={title:"Flink on Volcano",sidebar_position:1},r=void 0,s={},c=[{value:"Flink introduction",id:"flink-introduction",level:3},{value:"The premise condition",id:"the-premise-condition",level:3},{value:"Deployment process",id:"deployment-process",level:3},{value:"1. Download",id:"1-download",level:5},{value:"2. Start a Cluster",id:"2-start-a-cluster",level:5},{value:"3. Submit a job",id:"3-submit-a-job",level:5},{value:"Flink on Volcano",id:"flink-on-volcano",level:3},{value:"1. The deployment of the component",id:"1-the-deployment-of-the-component",level:5},{value:"2. Outward publishing service",id:"2-outward-publishing-service",level:5}];function p(e){const n={code:"code",h3:"h3",h5:"h5",li:"li",p:"p",pre:"pre",ul:"ul",...(0,i.R)(),...e.components};return(0,a.jsxs)(a.Fragment,{children:[(0,a.jsx)(n.h3,{id:"flink-introduction",children:"Flink introduction"}),"\n",(0,a.jsx)(n.p,{children:"Apache Flink is an open-source streaming framework developed by the Apache Software Foundation. At its core, Apache Flink is a distributed streaming data streaming engine written in Java and Scala. Flink executes any stream data program in data parallelism and pipelining. Flink's pipelined runtime system can execute both batch and stream programs. In addition, the Flink runtime itself supports the execution of iterative algorithms."}),"\n",(0,a.jsx)(n.h3,{id:"the-premise-condition",children:"The premise condition"}),"\n",(0,a.jsx)(n.p,{children:"Make sure the deployed Kubernetes, Kubectl, Volcano are installed correctly."}),"\n",(0,a.jsx)(n.h3,{id:"deployment-process",children:"Deployment process"}),"\n",(0,a.jsx)(n.h5,{id:"1-download",children:"1. Download"}),"\n",(0,a.jsx)(n.p,{children:"To run Flink, which requires a Java 8 or 11 environment, use the following instructions to determine the Java version."}),"\n",(0,a.jsx)(n.pre,{children:(0,a.jsx)(n.code,{className:"language-bash",children:"java -version\n"})}),"\n",(0,a.jsx)(n.p,{children:"Download the package and go to the directory."}),"\n",(0,a.jsx)(n.pre,{children:(0,a.jsx)(n.code,{children:"$ wget https://www.apache.org/dyn/closer.lua/flink/flink-1.12.2/flink-1.12.2-src.tgz\n$ cd flink-1.12.2\n"})}),"\n",(0,a.jsx)(n.h5,{id:"2-start-a-cluster",children:"2. Start a Cluster"}),"\n",(0,a.jsx)(n.p,{children:"Running the script completes the deployment of Flink on the cluster."}),"\n",(0,a.jsx)(n.pre,{children:(0,a.jsx)(n.code,{className:"language-bash",children:"$ ./bin/start-cluster.sh\n"})}),"\n",(0,a.jsx)(n.h5,{id:"3-submit-a-job",children:"3. Submit a job"}),"\n",(0,a.jsx)(n.p,{children:"Submit the job using the following instructions."}),"\n",(0,a.jsx)(n.pre,{children:(0,a.jsx)(n.code,{className:"language-bash",children:"$ ./bin/flink run examples/streaming/WordCount.jar\n$ tail log/flink-*-taskexecutor-*.out\n"})}),"\n",(0,a.jsx)(n.h3,{id:"flink-on-volcano",children:"Flink on Volcano"}),"\n",(0,a.jsx)(n.h5,{id:"1-the-deployment-of-the-component",children:"1. The deployment of the component"}),"\n",(0,a.jsxs)(n.p,{children:["Deploying a Flink Cluster requires creating two deploys, a Service, and a ConfigMap. The scheduling strategy is Volcano.The contents of ",(0,a.jsx)(n.code,{children:"flink-configuration-configmap.yaml"})," are shown below."]}),"\n",(0,a.jsx)(n.pre,{children:(0,a.jsx)(n.code,{children:"apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: flink-config\n labels:\n app: flink\ndata:\n flink-conf.yaml: |+\n jobmanager.rpc.address: flink-jobmanager\n taskmanager.numberOfTaskSlots: 2\n blob.server.port: 6124\n jobmanager.rpc.port: 6123\n taskmanager.rpc.port: 6122\n queryable-state.proxy.ports: 6125\n jobmanager.memory.process.size: 1600m\n taskmanager.memory.process.size: 1728m\n parallelism.default: 2\n log4j-console.properties: |+\n # This affects logging for both user code and Flink\n rootLogger.level = INFO\n rootLogger.appenderRef.console.ref = ConsoleAp
1pender\n rootLogger.appenderRef.rolling.ref = RollingFileAppender\n\n # Uncomment this if you want to _only_ change Flink's logging\n #logger.flink.name = org.apache.flink\n #logger.flink.level = INFO\n\n # The following lines keep the log level of common libraries/connectors on\n # log level INFO. The root logger does not override this. You have to manually\n # change the log levels here.\n logger.akka.name = akka\n logger.akka.level = INFO\n logger.kafka.name= org.apache.kafka\n logger.kafka.level = INFO\n logger.hadoop.name = org.apache.hadoop\n logger.hadoop.level = INFO\n logger.zookeeper.name = org.apache.zookeeper\n logger.zookeeper.level = INFO\n\n # Log all infos to the console\n appender.console.name = ConsoleAppender\n appender.console.type = CONSOLE\n appender.console.layout.type = PatternLayout\n appender.console.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n\n\n # Log all infos in the given rolling file\n appender.rolling.name = RollingFileAppender\n appender.rolling.type = RollingFile\n appender.rolling.append = false\n appender.rolling.fileName = ${sys:log.file}\n appender.rolling.filePattern = ${sys:log.file}.%i\n appender.rolling.layout.type = PatternLayout\n appender.rolling.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n\n appender.rolling.policies.type = Policies\n appender.rolling.policies.size.type = SizeBasedTriggeringPolicy\n appender.rolling.policies.size.size=100MB\n appender.rolling.strategy.type = DefaultRolloverStrategy\n appender.rolling.strategy.max = 10\n\n # Suppress the irrelevant (wrong) warnings from the Netty channel handler\n logger.netty.name = org.apache.flink.shaded.akka.org.jboss.netty.channel.DefaultChannelPipeline\n logger.netty.level = OFF\n"})}),"\n",(0,a.jsxs)(n.p,{children:["Service is used to provide services for the REST and UI ports of the JobManager.The contents of ",(0,a.jsx)(n.code,{children:"jobManager-Service.yaml"})," are as follows."]}),"\n",(0,a.jsx)(n.pre,{children:(0,a.jsx)(n.code,{children:"apiVersion: v1\nkind: Service\nmetadata:\n name: flink-jobmanager\nspec:\n type: ClusterIP\n ports:\n - name: rpc\n port: 6123\n - name: blob-server\n port: 6124\n - name: webui\n port: 8081\n selector:\n app: flink\n component: jobmanager\n"})}),"\n",(0,a.jsxs)(n.p,{children:["The contents of ",(0,a.jsx)(n.code,{children:"jobmanager-session-deployment.yaml"})," are as follows."]}),"\n",(0,a.jsx)(n.pre,{children:(0,a.jsx)(n.code,{children:'apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: flink-jobmanager\nspec:\n replicas: 1\n selector:\n matchLabels:\n app: flink\n component: jobmanager\n template:\n metadata:\n labels:\n app: flink\n component: jobmanager\n spec:\n containers:\n - name: jobmanager\n image: flink:1.11.0-scala_2.11\n args: ["jobmanager"]\n ports:\n - containerPort: 6123\n name: rpc\n - containerPort: 6124\n name: blob-server\n - containerPort: 8081\n name: webui\n livenessProbe:\n tcpSocket:\n port: 6123\n initialDelaySeconds: 30\n periodSeconds: 60\n volumeMounts:\n - name: flink-config-volume\n mountPath: /opt/flink/conf\n securityContext:\n runAsUser: 9999 # refers to user _flink_ from official flink image, change if necessary\n volumes:\n - name: flink-config-volume\n configMap:\n name: flink-config\n items:\n - key: flink-conf.yaml\n path: flink-conf.yaml\n - key: log4j-console.properties\n path: log4j-console.properties\n'})}),"\n",(0,a.jsxs)(n.p,{children:["The contents of ",(0,a.jsx)(n.code,{children:"taskmanager-session-deployment.yaml"})," are as follows."]}),"\n",(0,a.jsx)(n.pre,{children:(0,a.jsx)(n.code,{children:'apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: flink-taskmanager\nspec:\n replicas: 2\n selector:\n matchLabels:\n app: flink\n component: taskmanager\n template:\n metadata:\n labels:\n app: flink\n component: taskmanager\n spec:\n containers:\n - name: taskmanager\n image: flink:1.11.0-scala_2.11\n args: ["taskmanager"]\n ports:\n - containerPort: 6122\n name: rpc\n - containerPort: 6125\n name: query-state\n livenessProbe:\n tcpSocket:\n port: 6122\n initialDelaySeconds: 30\n periodSeconds: 60\n volumeMounts:\n - name: flink-config-volume\n mountPath: /opt/flink/conf/\n securityContext:\n runAsUser: 9999 # refers to user _flink_ from official flink image, change if necessary\n volumes:\n - name: flink-config-volume\n configMap:\n name: flink-config\n items:\n - key: flink-conf.yaml\n path: flink-conf.yaml\n - key: log4j-console.properties\n path: log4j-console.properties\n'})}),"\n",(0,a.jsx)(n.p,{children:"Create the above four YAML configuration files on the cluster node and deploy them using the following instructions."}),"\n",(0,a.jsx)(n.pre,{children:(0,a.jsx)(n.code,{children:"kubectl create -f flink-configuration-configmap.yaml\nkubectl create -f jobmanager-service.yaml\nkubectl create -f jobmanager-session-deployment.yaml\nkubectl create -f taskmanager-session-deployment.yaml\n"})}),"\n",(0,a.jsx)(n.p,{children:"Query to see if the payload was successfully created\uff1a"}),"\n",(0,a.jsx)(n.pre,{children:(0,a.jsx)(n.code,{children:"kubectl get cm| grep flink\nkubectl get svc | grep flink\nkubectl get pod -owide | grep Flink\n"})}),"\n",(0,a.jsx)(n.h5,{id:"2-outward-publishing-service",children:"2. Outward publishing service"}),"\n",(0,a.jsx)(n.p,{children:"Once the Flink payload is created, you need to publish the service externally\u3002"}),"\n",(0,a.jsxs)(n.ul,{children:["\n",(0,a.jsx)(n.li,{children:'If you use Huawei Cloud CCE for testing, go to the "Workloads - Stateless Loads" page of CCE. Select Flink-JobManager and click Access Mode.'}),"\n",(0,a.jsx)(n.li,{children:'Click "Add Service", select node access, and enter container port bit 8081.'}),"\n",(0,a.jsx)(n.li,{children:"Click Network Management in CCE, you can see the service we just added, and visit the link for external publication."}),"\n",(0,a.jsxs)(n.li,{children:["Go to the Dashboard page of Flink and click Submit New Job to submit the task. Here you have the option to submit an officially-provided WordCount sample.The directory is ",(0,a.jsx)(n.code,{children:"flink-1.12.2/examples/streaming/WordCount.jar"})]}),"\n"]})]})}function d(e={}){const{wrapper:n}={...(0,i.R)(),...e.components};return n?(0,a.jsx)(n,{...e,children:(0,a.jsx)(p,{...e})}):p(e)}},28453(e,n,o){o.d(n,{R:()=>t,x:()=>r});var l=o(96540);const a={},i=l.createContext(a);function t(e){const n=l.useContext(i);return l.useMemo(function(){return"function"==typeof e?e(n):{...n,...e}},[n,e])}function r(e){let n;return n=e.disableParentContext?"function"==typeof e.components?e.components(a):e.components||a:t(e.components),l.createElement(i.Provider,{value:n},e.children)}}}]);
Line numbers count LF bytes from the start of the resource, as the search results do. Vendor segments are library code the classifier recognised; they are stored but not indexed. Bytes are shown as Latin1 characters, one per byte.