diff --git a/.github/workflows/deps.yml b/.github/workflows/deps.yml index f40e194a594..efa28a2eb84 100644 --- a/.github/workflows/deps.yml +++ b/.github/workflows/deps.yml @@ -129,6 +129,7 @@ jobs: matrix: module: - 'spark-4.0' + - 'spark-4.1' steps: - uses: actions/checkout@v4 - name: Setup JDK 17 diff --git a/README.md b/README.md index 6eb54c972da..bdf66e02725 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,7 @@ Celeborn worker's slot count decreases when a partition is allocated and increme Build Celeborn via `make-distribution.sh`: ```shell -./build/make-distribution.sh -Pspark-2.4/-Pspark-3.0/-Pspark-3.1/-Pspark-3.2/-Pspark-3.3/-Pspark-3.4/-Pspark-3.5/-Pspark-4.0/-Pflink-1.16/-Pflink-1.17/-Pflink-1.18/-Pflink-1.19/-Pflink-1.20/-Pflink-2.0/-Pflink-2.1/-Pflink-2.2/-Pmr +./build/make-distribution.sh -Pspark-2.4/-Pspark-3.0/-Pspark-3.1/-Pspark-3.2/-Pspark-3.3/-Pspark-3.4/-Pspark-3.5/-Pspark-4.0/-Pspark-4.1/-Pflink-1.16/-Pflink-1.17/-Pflink-1.18/-Pflink-1.19/-Pflink-1.20/-Pflink-2.0/-Pflink-2.1/-Pflink-2.2/-Pmr ``` Package `apache-celeborn-${project.version}-bin.tgz` will be generated. @@ -63,6 +63,7 @@ Package `apache-celeborn-${project.version}-bin.tgz` will be generated. | Spark 3.4 | ❌ | ✔ | ✔ | ✔ | ✔ | ✔ | ✔ | | Spark 3.5 | ❌ | ✔ | ✔ | ✔ | ✔ | ✔ | ✔ | | Spark 4.0 | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✔ | +| Spark 4.1 | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✔ | | Flink 1.16 | ❌ | ✔ | ✔ | ❌ | ❌ | ❌ | ❌ | | Flink 1.17 | ❌ | ✔ | ✔ | ❌ | ❌ | ❌ | ❌ | | Flink 1.18 | ❌ | ✔ | ✔ | ❌ | ❌ | ❌ | ❌ | diff --git a/build/make-distribution.sh b/build/make-distribution.sh index f0e659c5d6f..e160b758dda 100755 --- a/build/make-distribution.sh +++ b/build/make-distribution.sh @@ -356,6 +356,7 @@ if [ "$SBT_ENABLED" == "true" ]; then sbt_build_client -Pspark-3.5 export JAVA_HOME=$JAVA17_HOME sbt_build_client -Pspark-4.0 + sbt_build_client -Pspark-4.1 export JAVA_HOME=$JAVA8_HOME sbt_build_client -Pflink-1.16 sbt_build_client -Pflink-1.17 @@ -399,6 +400,7 @@ else build_spark_client -Pspark-3.5 export JAVA_HOME=$JAVA17_HOME build_spark_client -Pspark-4.0 + build_spark_client -Pspark-4.1 export JAVA_HOME=$JAVA8_HOME build_flink_client -Pflink-1.16 build_flink_client -Pflink-1.17 diff --git a/build/release/release.sh b/build/release/release.sh index 019f6383fd4..b3954c33357 100755 --- a/build/release/release.sh +++ b/build/release/release.sh @@ -115,7 +115,7 @@ upload_nexus_staging() { export JAVA_HOME=$JAVA17_HOME echo "Deploying celeborn-client-spark-4-shaded_2.13" - ${PROJECT_DIR}/build/sbt -Pspark-4.0 "clean;celeborn-client-spark-4-shaded/publishSigned" + ${PROJECT_DIR}/build/sbt -Pspark-4.1 "clean;celeborn-client-spark-4-shaded/publishSigned" export JAVA_HOME=$JAVA8_HOME echo "Deploying celeborn-client-flink-1.16-shaded_2.12" diff --git a/dev/deps/dependencies-client-spark-4.1 b/dev/deps/dependencies-client-spark-4.1 new file mode 100644 index 00000000000..be857ab55ee --- /dev/null +++ b/dev/deps/dependencies-client-spark-4.1 @@ -0,0 +1,105 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +RoaringBitmap/1.0.6//RoaringBitmap-1.0.6.jar +commons-crypto/1.0.0//commons-crypto-1.0.0.jar +commons-io/2.17.0//commons-io-2.17.0.jar +commons-lang3/3.17.0//commons-lang3-3.17.0.jar +commons-logging/1.1.3//commons-logging-1.1.3.jar +failureaccess/1.0.2//failureaccess-1.0.2.jar +guava/33.1.0-jre//guava-33.1.0-jre.jar +hadoop-client-api/3.3.6//hadoop-client-api-3.3.6.jar +hadoop-client-runtime/3.3.6//hadoop-client-runtime-3.3.6.jar +jackson-annotations/2.15.3//jackson-annotations-2.15.3.jar +jackson-core/2.15.3//jackson-core-2.15.3.jar +jackson-databind/2.15.3//jackson-databind-2.15.3.jar +jackson-module-scala_2.13/2.15.3//jackson-module-scala_2.13-2.15.3.jar +jcl-over-slf4j/1.7.36//jcl-over-slf4j-1.7.36.jar +jsr305/1.3.9//jsr305-1.3.9.jar +jul-to-slf4j/1.7.36//jul-to-slf4j-1.7.36.jar +leveldbjni-all/1.8//leveldbjni-all-1.8.jar +lz4-java/1.8.0//lz4-java-1.8.0.jar +metrics-core/4.2.25//metrics-core-4.2.25.jar +metrics-graphite/4.2.25//metrics-graphite-4.2.25.jar +metrics-jvm/4.2.25//metrics-jvm-4.2.25.jar +netty-all/4.1.118.Final//netty-all-4.1.118.Final.jar +netty-buffer/4.1.118.Final//netty-buffer-4.1.118.Final.jar +netty-buffer/4.2.7.Final//netty-buffer-4.2.7.Final.jar +netty-codec-base/4.2.7.Final//netty-codec-base-4.2.7.Final.jar +netty-codec-compression/4.2.7.Final//netty-codec-compression-4.2.7.Final.jar +netty-codec-dns/4.1.118.Final//netty-codec-dns-4.1.118.Final.jar +netty-codec-dns/4.2.7.Final//netty-codec-dns-4.2.7.Final.jar +netty-codec-haproxy/4.1.118.Final//netty-codec-haproxy-4.1.118.Final.jar +netty-codec-http/4.1.118.Final//netty-codec-http-4.1.118.Final.jar +netty-codec-http/4.2.7.Final//netty-codec-http-4.2.7.Final.jar +netty-codec-http2/4.1.118.Final//netty-codec-http2-4.1.118.Final.jar +netty-codec-http2/4.2.7.Final//netty-codec-http2-4.2.7.Final.jar +netty-codec-marshalling/4.2.7.Final//netty-codec-marshalling-4.2.7.Final.jar +netty-codec-memcache/4.1.118.Final//netty-codec-memcache-4.1.118.Final.jar +netty-codec-mqtt/4.1.118.Final//netty-codec-mqtt-4.1.118.Final.jar +netty-codec-protobuf/4.2.7.Final//netty-codec-protobuf-4.2.7.Final.jar +netty-codec-redis/4.1.118.Final//netty-codec-redis-4.1.118.Final.jar +netty-codec-smtp/4.1.118.Final//netty-codec-smtp-4.1.118.Final.jar +netty-codec-socks/4.1.118.Final//netty-codec-socks-4.1.118.Final.jar +netty-codec-socks/4.2.7.Final//netty-codec-socks-4.2.7.Final.jar +netty-codec-stomp/4.1.118.Final//netty-codec-stomp-4.1.118.Final.jar +netty-codec-xml/4.1.118.Final//netty-codec-xml-4.1.118.Final.jar +netty-codec/4.1.118.Final//netty-codec-4.1.118.Final.jar +netty-codec/4.2.7.Final//netty-codec-4.2.7.Final.jar +netty-common/4.1.118.Final//netty-common-4.1.118.Final.jar +netty-common/4.2.7.Final//netty-common-4.2.7.Final.jar +netty-handler-proxy/4.1.118.Final//netty-handler-proxy-4.1.118.Final.jar +netty-handler-proxy/4.2.7.Final//netty-handler-proxy-4.2.7.Final.jar +netty-handler/4.1.118.Final//netty-handler-4.1.118.Final.jar +netty-handler/4.2.7.Final//netty-handler-4.2.7.Final.jar +netty-resolver-dns-classes-macos/4.1.118.Final//netty-resolver-dns-classes-macos-4.1.118.Final.jar +netty-resolver-dns-native-macos/4.1.118.Final/osx-aarch_64/netty-resolver-dns-native-macos-4.1.118.Final-osx-aarch_64.jar +netty-resolver-dns-native-macos/4.1.118.Final/osx-x86_64/netty-resolver-dns-native-macos-4.1.118.Final-osx-x86_64.jar +netty-resolver-dns/4.1.118.Final//netty-resolver-dns-4.1.118.Final.jar +netty-resolver-dns/4.2.7.Final//netty-resolver-dns-4.2.7.Final.jar +netty-resolver/4.1.118.Final//netty-resolver-4.1.118.Final.jar +netty-resolver/4.2.7.Final//netty-resolver-4.2.7.Final.jar +netty-transport-classes-epoll/4.1.118.Final//netty-transport-classes-epoll-4.1.118.Final.jar +netty-transport-classes-epoll/4.2.7.Final//netty-transport-classes-epoll-4.2.7.Final.jar +netty-transport-classes-kqueue/4.1.118.Final//netty-transport-classes-kqueue-4.1.118.Final.jar +netty-transport-classes-kqueue/4.2.7.Final//netty-transport-classes-kqueue-4.2.7.Final.jar +netty-transport-native-epoll/4.1.118.Final/linux-aarch_64/netty-transport-native-epoll-4.1.118.Final-linux-aarch_64.jar +netty-transport-native-epoll/4.1.118.Final/linux-riscv64/netty-transport-native-epoll-4.1.118.Final-linux-riscv64.jar +netty-transport-native-epoll/4.1.118.Final/linux-x86_64/netty-transport-native-epoll-4.1.118.Final-linux-x86_64.jar +netty-transport-native-epoll/4.2.7.Final/linux-aarch_64/netty-transport-native-epoll-4.2.7.Final-linux-aarch_64.jar +netty-transport-native-epoll/4.2.7.Final/linux-riscv64/netty-transport-native-epoll-4.2.7.Final-linux-riscv64.jar +netty-transport-native-epoll/4.2.7.Final/linux-x86_64/netty-transport-native-epoll-4.2.7.Final-linux-x86_64.jar +netty-transport-native-kqueue/4.1.118.Final/osx-aarch_64/netty-transport-native-kqueue-4.1.118.Final-osx-aarch_64.jar +netty-transport-native-kqueue/4.1.118.Final/osx-x86_64/netty-transport-native-kqueue-4.1.118.Final-osx-x86_64.jar +netty-transport-native-kqueue/4.2.7.Final/osx-aarch_64/netty-transport-native-kqueue-4.2.7.Final-osx-aarch_64.jar +netty-transport-native-kqueue/4.2.7.Final/osx-x86_64/netty-transport-native-kqueue-4.2.7.Final-osx-x86_64.jar +netty-transport-native-unix-common/4.1.118.Final//netty-transport-native-unix-common-4.1.118.Final.jar +netty-transport-native-unix-common/4.2.7.Final//netty-transport-native-unix-common-4.2.7.Final.jar +netty-transport-rxtx/4.1.118.Final//netty-transport-rxtx-4.1.118.Final.jar +netty-transport-sctp/4.1.118.Final//netty-transport-sctp-4.1.118.Final.jar +netty-transport-udt/4.1.118.Final//netty-transport-udt-4.1.118.Final.jar +netty-transport/4.1.118.Final//netty-transport-4.1.118.Final.jar +netty-transport/4.2.7.Final//netty-transport-4.2.7.Final.jar +paranamer/2.8.3//paranamer-2.8.3.jar +paranamer/2.8//paranamer-2.8.jar +protobuf-java/3.25.5//protobuf-java-3.25.5.jar +scala-library/2.13.17//scala-library-2.13.17.jar +scala-reflect/2.13.17//scala-reflect-2.13.17.jar +slf4j-api/1.7.36//slf4j-api-1.7.36.jar +snakeyaml/2.2//snakeyaml-2.2.jar +snappy-java/1.1.10.5//snappy-java-1.1.10.5.jar +zstd-jni/1.5.7-6//zstd-jni-1.5.7-6.jar diff --git a/dev/reformat b/dev/reformat index 5c702536a5b..b6fe643e049 100755 --- a/dev/reformat +++ b/dev/reformat @@ -35,6 +35,7 @@ else ${PROJECT_DIR}/build/mvn spotless:apply -Pspark-3.3 ${PROJECT_DIR}/build/mvn spotless:apply -Pspark-3.5 ${PROJECT_DIR}/build/mvn spotless:apply -Pspark-4.0 + ${PROJECT_DIR}/build/mvn spotless:apply -Pspark-4.1 ${PROJECT_DIR}/build/mvn spotless:apply -Paws ${PROJECT_DIR}/build/mvn spotless:apply -Paliyun ${PROJECT_DIR}/build/mvn spotless:apply -Pmr diff --git a/docs/developers/sbt.md b/docs/developers/sbt.md index 390e0764630..60617f86afa 100644 --- a/docs/developers/sbt.md +++ b/docs/developers/sbt.md @@ -36,6 +36,7 @@ The following table indicates the compatibility of Celeborn Spark and Flink clie | Spark 3.4 | ❌ | ✔ | ✔ | ✔ | ✔ | ✔ | ✔ | | Spark 3.5 | ❌ | ✔ | ✔ | ✔ | ✔ | ✔ | ✔ | | Spark 4.0 | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✔ | +| Spark 4.1 | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✔ | | Flink 1.16 | ❌ | ✔ | ✔ | ❌ | ❌ | ❌ | ❌ | | Flink 1.17 | ❌ | ✔ | ✔ | ❌ | ❌ | ❌ | ❌ | | Flink 1.18 | ❌ | ✔ | ✔ | ❌ | ❌ | ❌ | ❌ | diff --git a/pom.xml b/pom.xml index 5ef3d5f7804..5cc4b54b915 100644 --- a/pom.xml +++ b/pom.xml @@ -1581,6 +1581,25 @@ + + spark-4.1 + + client-spark/common + client-spark/spark-3 + client-spark/spark-3-columnar-common + client-spark/spark-4-columnar-shuffle + client-spark/spark-4-shaded + tests/spark-it + + + 1.8.0 + 2.13.17 + 2.13 + 4.1.0 + 1.5.7-6 + + + jdk-8 diff --git a/project/CelebornBuild.scala b/project/CelebornBuild.scala index 94392eca2d2..05f33da83b3 100644 --- a/project/CelebornBuild.scala +++ b/project/CelebornBuild.scala @@ -494,6 +494,7 @@ object Utils { case Some("spark-3.4") => Some(Spark34) case Some("spark-3.5") => Some(Spark35) case Some("spark-4.0") => Some(Spark40) + case Some("spark-4.1") => Some(Spark41) case _ => None } @@ -942,6 +943,23 @@ object Spark40 extends SparkClientProjects { override val sparkColumnarShuffleVersion: String = "4" } +object Spark41 extends SparkClientProjects { + + val sparkClientProjectPath = "client-spark/spark-3" + val sparkClientProjectName = "celeborn-client-spark-4" + val sparkClientShadedProjectPath = "client-spark/spark-4-shaded" + val sparkClientShadedProjectName = "celeborn-client-spark-4-shaded" + + val lz4JavaVersion = "1.8.0" + val sparkProjectScalaVersion = "2.13.17" + + val sparkVersion = "4.1.0" + val zstdJniVersion = "1.5.7-6" + val scalaBinaryVersion = "2.13" + + override val sparkColumnarShuffleVersion: String = "4" +} + trait SparkClientProjects { val sparkClientProjectPath: String diff --git a/tests/spark-it/pom.xml b/tests/spark-it/pom.xml index c0a37ea6c33..1b6f6019ece 100644 --- a/tests/spark-it/pom.xml +++ b/tests/spark-it/pom.xml @@ -328,5 +328,23 @@ + + spark-4.1 + + + org.apache.celeborn + celeborn-client-spark-3_${scala.binary.version} + ${project.version} + test + + + org.apache.celeborn + celeborn-client-spark-3_${scala.binary.version} + ${project.version} + test-jar + test + + +