下载 flink 镜像
[root@localhost ~]# docker pull flink
Using default tag: latest
latest: Pulling from library/flink
762bedf4b1b7: Pull complete
95f9bd9906fa: Pull complete
a880dee0d8e9: Pull complete
8c5deab9cbd6: Pull complete
56c142282fae: Pull complete
429468ac77e4: Pull complete
c392a87beaaa: Pull complete
f9dbdda9fdd9: Pull complete
79ba4b85d71e: Pull complete
fbb582e0bb4f: Pull complete
3a1636493da1: Pull complete
Digest: sha256:000006919395479c51dff9b1db67e6957bc2a9cdcda07c4e409edb5ba79a2a9a
Status: Downloaded newer image for flink:latest
docker.io/library/flink:latest
[root@localhost ~]# docker images
REPOSITORY TAG IMAGE ID CREATED SIZE
flink latest 19a46ae75631 5 weeks ago 798MB
[root@localhost ~]# docker history flink
IMAGE CREATED CREATED BY SIZE COMMENT
19a46ae75631 5 weeks ago CMD ["help"] 0B buildkit.dockerfile.v0
<missing> 5 weeks ago EXPOSE map[6123/tcp:{} 8081/tcp:{}] 0B buildkit.dockerfile.v0
<missing> 5 weeks ago ENTRYPOINT ["/docker-entrypoint.sh"] 0B buildkit.dockerfile.v0
<missing> 5 weeks ago COPY docker-entrypoint.sh / # buildkit 5.98kB buildkit.dockerfile.v0
<missing> 5 weeks ago RUN /bin/sh -c set -ex; wget -nv -O flink.… 527MB buildkit.dockerfile.v0
<missing> 5 weeks ago WORKDIR /opt/flink 0B buildkit.dockerfile.v0
<missing> 5 weeks ago RUN /bin/sh -c groupadd --system --gid=9999 … 3.24MB buildkit.dockerfile.v0
<missing> 5 weeks ago ENV PATH=/opt/flink/bin:/opt/java/openjdk/bi… 0B buildkit.dockerfile.v0
<missing> 5 weeks ago ENV FLINK_HOME=/opt/flink 0B buildkit.dockerfile.v0
<missing> 5 weeks ago ENV FLINK_TGZ_URL=https://dlcdn.apache.org/f… 0B buildkit.dockerfile.v0
<missing> 5 weeks ago RUN /bin/sh -c set -ex; wget -nv -O /usr/l… 2.3MB buildkit.dockerfile.v0
<missing> 5 weeks ago ENV GOSU_VERSION=1.11 0B buildkit.dockerfile.v0
<missing> 5 weeks ago RUN /bin/sh -c set -ex; apt-get update; … 10.7MB buildkit.dockerfile.v0
<missing> 5 weeks ago ENTRYPOINT ["/__cacert_entrypoint.sh"] 0B buildkit.dockerfile.v0
<missing> 5 weeks ago COPY --chmod=755 entrypoint.sh /__cacert_ent… 4.74kB buildkit.dockerfile.v0
<missing> 5 weeks ago RUN /bin/sh -c set -eux; echo "Verifying… 0B buildkit.dockerfile.v0
<missing> 5 weeks ago RUN /bin/sh -c set -eux; ARCH="$(dpkg --… 141MB buildkit.dockerfile.v0
<missing> 5 weeks ago ENV JAVA_VERSION=jdk-11.0.24+8 0B buildkit.dockerfile.v0
<missing> 5 weeks ago RUN /bin/sh -c set -eux; apt-get update;… 36.1MB buildkit.dockerfile.v0
<missing> 5 weeks ago ENV LANG=en_US.UTF-8 LANGUAGE=en_US:en LC_AL… 0B buildkit.dockerfile.v0
<missing> 5 weeks ago ENV PATH=/opt/java/openjdk/bin:/usr/local/sb… 0B buildkit.dockerfile.v0
<missing> 5 weeks ago ENV JAVA_HOME=/opt/java/openjdk 0B buildkit.dockerfile.v0
<missing> 5 weeks ago /bin/sh -c #(nop) CMD ["/bin/bash"] 0B
<missing> 5 weeks ago /bin/sh -c #(nop) ADD file:2f8a54a5efd080fb8… 77.9MB
<missing> 5 weeks ago /bin/sh -c #(nop) LABEL org.opencontainers.… 0B
<missing> 5 weeks ago /bin/sh -c #(nop) LABEL org.opencontainers.… 0B
<missing> 5 weeks ago /bin/sh -c #(nop) ARG LAUNCHPAD_BUILD_ARCH 0B
<missing> 5 weeks ago /bin/sh -c #(nop) ARG RELEASE 0B
测试运行
[root@localhost ~]# docker run -it --rm flink java
[0.004s][warning][os,thread] Failed to start thread "GC Thread#0" - pthread_create failed (EPERM) for attributes: stacksize: 1024k, guardsize: 4k, detached.
#
# There is insufficient memory for the Java Runtime Environment to continue.
# Cannot create worker GC thread. Out of system resources.
# An error report file with more information is saved as:
# /opt/flink/hs_err_pid44.log
搜索原因，需要开启 privileged 权限
privileged: true 这行配置启用了 taskmanager 容器的特权模式，使得容器内的进程可以获得几乎与主机上相同的权限。
注意事项：
- 安全性：启用特权模式会降低容器的安全性，因为它允许容器内的进程执行可能影响主机系统的操作。因此，只有在确实需要时才启用特权模式。
- 使用场景：特权模式通常用于需要访问主机设备（如 GPU、网络接口等）或执行特权操作（如挂载文件系统、修改内核参数等）的容器。
[root@localhost ~]# docker run -it --privileged --rm flink java -version
openjdk version "11.0.24" 2024-07-16
OpenJDK Runtime Environment Temurin-11.0.24+8 (build 11.0.24+8)
OpenJDK 64-Bit Server VM Temurin-11.0.24+8 (build 11.0.24+8, mixed mode, sharing)
docker-compose.yml
version: "3"
services:
jobmanager:
image: flink:latest
privileged: true
ports:
- "8181:8081"
command: jobmanager
environment:
- |
FLINK_PROPERTIES=
jobmanager.rpc.address: jobmanager
taskmanager:
image: flink:latest
privileged: true
depends_on:
- jobmanager
command: taskmanager
environment:
- |
FLINK_PROPERTIES=
jobmanager.rpc.address: jobmanager
taskmanager.numberOfTaskSlots: 2
启动服务
[root@localhost xwtes]# docker-compose up -d --scale taskmanager=3
xwtes_jobmanager_1 is up-to-date
Starting xwtes_taskmanager_1 ... done
Creating xwtes_taskmanager_2 ... done
Creating xwtes_taskmanager_3 ... done
安装 PyFlink
官方镜像不支持python，进入容器手动安装，然后commit 镜像flink:py
sed -i "s@http://.*archive.ubuntu.com@https://mirrors.tuna.tsinghua.edu.cn@g" /etc/apt/sources.list
sed -i "s@http://.*security.ubuntu.com@https://mirrors.tuna.tsinghua.edu.cn@g" /etc/apt/sources.list
apt-get update
apt install python3 python3-pip -y
pip install apache-flink notebook -i https://pypi.tuna.tsinghua.edu.cn/simple/
ln -s /usr/bin/python3 /usr/bin/python
docker-compose
version: "3"
services:
jobmanager:
image: flink:py
privileged: true
ports:
- "8181:8081"
command: jobmanager
environment:
- |
FLINK_PROPERTIES=
jobmanager.rpc.address: jobmanager
taskmanager:
image: flink:py
privileged: true
depends_on:
- jobmanager
command: taskmanager
environment:
- |
FLINK_PROPERTIES=
jobmanager.rpc.address: jobmanager
taskmanager.numberOfTaskSlots: 2
jupyter:
image: flink:py
privileged: true
ports:
- "9999:8888"
command: jupyter notebook --ip 0.0.0.0 --port 8888 --allow-root --NotebookApp.password=sha1:6587feaef3b1:6b243404e4cfaafe611fdf494ee71fdaa8c4a563
参考资料
- https://nightlies.apache.org/flink/flink-docs-master/docs/deployment/resource-providers/standalone/docker/
