diff --git a/bitnami/spark/3.5/debian-11/Dockerfile b/bitnami/spark/3.5/debian-11/Dockerfile index 4fb0679fd4b7..94c4cc713595 100644 --- a/bitnami/spark/3.5/debian-11/Dockerfile +++ b/bitnami/spark/3.5/debian-11/Dockerfile @@ -8,10 +8,10 @@ ARG TARGETARCH LABEL com.vmware.cp.artifact.flavor="sha256:1e1b4657a77f0d47e9220f0c37b9bf7802581b93214fff7d1bd2364c8bf22e8e" \ org.opencontainers.image.base.name="docker.io/bitnami/minideb:bullseye" \ - org.opencontainers.image.created="2023-10-10T22:27:42Z" \ + org.opencontainers.image.created="2023-10-26T01:07:07Z" \ org.opencontainers.image.description="Application packaged by VMware, Inc" \ org.opencontainers.image.licenses="Apache-2.0" \ - org.opencontainers.image.ref.name="3.5.0-debian-11-r10" \ + org.opencontainers.image.ref.name="3.5.0-debian-11-r11" \ org.opencontainers.image.title="spark" \ org.opencontainers.image.vendor="VMware, Inc." \ org.opencontainers.image.version="3.5.0" @@ -28,9 +28,9 @@ SHELL ["/bin/bash", "-o", "pipefail", "-c"] RUN install_packages ca-certificates curl libbz2-1.0 libcom-err2 libcrypt1 libffi7 libgcc-s1 libgssapi-krb5-2 libk5crypto3 libkeyutils1 libkrb5-3 libkrb5support0 liblzma5 libncursesw6 libnsl2 libreadline8 libsqlite3-0 libssl1.1 libstdc++6 libtinfo6 libtirpc3 procps zlib1g RUN mkdir -p /tmp/bitnami/pkg/cache/ && cd /tmp/bitnami/pkg/cache/ && \ COMPONENTS=( \ - "python-3.9.18-2-linux-${OS_ARCH}-debian-11" \ - "java-17.0.8-7-5-linux-${OS_ARCH}-debian-11" \ - "spark-3.5.0-0-linux-${OS_ARCH}-debian-11" \ + "python-3.11.6-6-linux-${OS_ARCH}-debian-11" \ + "java-17.0.9-11-1-linux-${OS_ARCH}-debian-11" \ + "spark-3.5.0-1-linux-${OS_ARCH}-debian-11" \ ) && \ for COMPONENT in "${COMPONENTS[@]}"; do \ if [ ! -f "${COMPONENT}.tar.gz" ]; then \ diff --git a/bitnami/spark/3.5/debian-11/prebuildfs/opt/bitnami/.bitnami_components.json b/bitnami/spark/3.5/debian-11/prebuildfs/opt/bitnami/.bitnami_components.json index 36280f388581..3b816b719e23 100644 --- a/bitnami/spark/3.5/debian-11/prebuildfs/opt/bitnami/.bitnami_components.json +++ b/bitnami/spark/3.5/debian-11/prebuildfs/opt/bitnami/.bitnami_components.json @@ -3,18 +3,18 @@ "arch": "amd64", "distro": "debian-11", "type": "NAMI", - "version": "17.0.8-7-5" + "version": "17.0.9-11-1" }, "python": { "arch": "amd64", "distro": "debian-11", "type": "NAMI", - "version": "3.9.18-2" + "version": "3.11.6-6" }, "spark": { "arch": "amd64", "distro": "debian-11", "type": "NAMI", - "version": "3.5.0-0" + "version": "3.5.0-1" } } \ No newline at end of file diff --git a/bitnami/spark/3.5/debian-11/rootfs/opt/bitnami/scripts/spark/entrypoint.sh b/bitnami/spark/3.5/debian-11/rootfs/opt/bitnami/scripts/spark/entrypoint.sh index 59eecd50dd53..ec07b0552d2c 100755 --- a/bitnami/spark/3.5/debian-11/rootfs/opt/bitnami/scripts/spark/entrypoint.sh +++ b/bitnami/spark/3.5/debian-11/rootfs/opt/bitnami/scripts/spark/entrypoint.sh @@ -30,14 +30,15 @@ if [[ "$1" = "/opt/bitnami/scripts/spark/run.sh" ]]; then info "** Spark setup finished! **" fi -# Spark has an special 'driver' command which is an alias for spark-submit -# https://github.com/apache/spark/blob/master/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/entrypoint.sh +# ref: https://spark.apache.org/docs/latest/running-on-kubernetes.html +# inspired by https://github.com/apache/spark/blob/master/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/entrypoint.sh case "$1" in driver) shift 1 CMD=( "/opt/bitnami/spark/bin/spark-submit" - --master "${SPARK_MASTER_URL}" + --conf "spark.driver.bindAddress=$SPARK_DRIVER_BIND_ADDRESS" + --conf "spark.executorEnv.SPARK_DRIVER_POD_IP=$SPARK_DRIVER_BIND_ADDRESS" --conf "spark.jars.ivy=/tmp/.ivy" --deploy-mode client "$@"