镜像下载# apache官方的镜像 docker pull apache/seatunnel:2.3.13 # 如果网络不好可以通过阿里的地址下载修改docker-compose中image docker pull registry.cn-hangzhou.aliyuncs.com/images-docker-cn/seatunnel:2.3.13部署docker-compose.yml文件一个master节点两个worker节点也可以用一个worker节点services: master: image: apache/seatunnel:2.3.13 container_name: seatunnel_master restart: unless-stopped environment: - ST_DOCKER_MEMBER_LISTseatunnel_master:5801 - SEATUNNEL_ENGINE_HTTP_ENABLE_HTTPtrue - SEATUNNEL_ENGINE_HTTP_PORT8080 entrypoint: /bin/sh -c /opt/seatunnel/bin/seatunnel-cluster.sh -r master ports: - 5801:5801 - 8080:8080 networks: - seatunnel_network worker1: image: apache/seatunnel:2.3.13 container_name: seatunnel_worker_1 restart: unless-stopped environment: - ST_DOCKER_MEMBER_LISTseatunnel_master:5801 - HZ_NETWORK_RESTAPI_ENABLEDtrue entrypoint: /bin/sh -c /opt/seatunnel/bin/seatunnel-cluster.sh -r worker depends_on: - master networks: - seatunnel_network worker2: image: apache/seatunnel:2.3.13 container_name: seatunnel_worker_2 restart: unless-stopped environment: - ST_DOCKER_MEMBER_LISTseatunnel_master:5801 - HZ_NETWORK_RESTAPI_ENABLEDtrue entrypoint: /bin/sh -c /opt/seatunnel/bin/seatunnel-cluster.sh -r worker depends_on: - master networks: - seatunnel_network networks: seatunnel_network: name: seatunnel-network driver: bridge启动docker compose up -d验证页面访问ip8080 端口访问页面CLI命令提交任务docker run --name seatunnel_client \ --network seatunnel-network \ -e ST_DOCKER_MEMBER_LISTseatunnel_master:5801 \ --rm \ apache/seatunnel:2.3.13 \ ./bin/seatunnel.sh -c config/v2.batch.config.template -m cluster日志输出查看任务命令模式查看docker run --name seatunnel_client \ --network seatunnel-network \ -e ST_DOCKER_MEMBER_LISTseatunnel_master:5801 \ --rm \ apache/seatunnel:2.3.13 \ ./bin/seatunnel.sh -l也可以在看版本中查看刚提交的任务用API接口提交任务示例一提交任务POST 请求 ip8080端口 / submit-job示例JSON{ env: { parallelism: 2, job.mode: BATCH, checkpoint.interval: 10000, job.name: api-batch-test }, source: [ { plugin_name: FakeSource, parallelism: 2, row.num: 16, plugin_output: fake, schema: { fields: { name: string, age: int } } } ], sink: [ { plugin_name: Console } ] }示例二:迁移mysql to mysql接口同上json示例{ env: { parallelism: 1, job.mode: BATCH, job.name: batch-mysql-test }, source: [ { plugin_name: Jdbc, url: jdbc:mysql://192.168.56.100:3000/test?serverTimezoneGMT%2b8useUnicodetruecharacterEncodingUTF-8rewriteBatchedStatementstrue, driver: com.mysql.cj.jdbc.Driver, connection_check_timeout_sec: 100, username: root, password: 123456, query: select * from student } ], transform: [], sink: [ { plugin_name: jdbc, url: jdbc:mysql://192.168.56.100:3000/test2?useUnicodetruecharacterEncodingUTF-8rewriteBatchedStatementstrue, driver: com.mysql.cj.jdbc.Driver, username: root, password: 123456, generate_sink_sql: true, database: test2, table: student } ] }补充job任务列表持久化当前docker 容器重启后job列表就不见了需要持久化配置。配置文件为master 节点的 /opt/seatunnel/config/hazelcast-master.yaml 文件可参考以下配置# 1. 创建目录并导出容器内配置 mkdir -p /data/seatunnel /data/seatunnel/imap docker run --rm apache/seatunnel:2.3.13 cat /opt/seatunnel/config/hazelcast-master.yaml /data/seatunnel/hazelcast-master.yaml # 2. 追加持久化存储配置 cat /data/seatunnel/hazelcast-master.yaml EOF map: engine*: map-store: enabled: true initial-mode: LAZY factory-class-name: org.apache.seatunnel.engine.server.persistence.FileMapStoreFactory properties: type: hdfs namespace: /opt/seatunnel/imap-data fs.defaultFS: file:/// clusterName: seatunnel EOFdocker-compose.ymlversion: 3.8 services: master: image: apache/seatunnel:2.3.13 container_name: seatunnel_master restart: unless-stopped environment: - ST_DOCKER_MEMBER_LISTseatunnel_master:5801 - SEATUNNEL_ENGINE_HTTP_ENABLE_HTTPtrue - SEATUNNEL_ENGINE_HTTP_PORT8080 entrypoint: /bin/sh -c /opt/seatunnel/bin/seatunnel-cluster.sh -r master ports: - 5801:5801 - 8080:8080 volumes: # 覆盖容器内置hazelcast-master.yaml - /data/seatunnel/hazelcast-master.yaml:/opt/seatunnel/config/hazelcast-master.yaml # 持久化任务数据映射到宿主机 - /data/seatunnel/imap:/opt/seatunnel/imap-data networks: - seatunnel_network worker1: image: apache/seatunnel:2.3.13 container_name: seatunnel_worker_1 restart: unless-stopped environment: - ST_DOCKER_MEMBER_LISTseatunnel_master:5801 - HZ_NETWORK_RESTAPI_ENABLEDtrue entrypoint: /bin/sh -c /opt/seatunnel/bin/seatunnel-cluster.sh -r worker depends_on: - master volumes: # Worker不需要自定义hazelcast-master但共享持久化目录集群共享任务状态 - /data/seatunnel/imap:/opt/seatunnel/imap-data networks: - seatunnel_network worker2: image: apache/seatunnel:2.3.13 container_name: seatunnel_worker_2 restart: unless-stopped environment: - ST_DOCKER_MEMBER_LISTseatunnel_master:5801 - HZ_NETWORK_RESTAPI_ENABLEDtrue entrypoint: /bin/sh -c /opt/seatunnel/bin/seatunnel-cluster.sh -r worker depends_on: - master volumes: - /data/seatunnel/imap:/opt/seatunnel/imap-data networks: - seatunnel_network networks: seatunnel_network: name: seatunnel-network driver: bridge
)
