From 8dc89f5cc20cd7f0fff28140d0bbc567da07127f Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Wed, 15 Jul 2026 22:53:22 -0700 Subject: [PATCH 01/12] [SPARK-58200][BUILD] Add a scala-3 build profile and make the version switch reversible Adds the plumbing needed to build the tree against Scala 3 (3.3.8) alongside 2.13, using the mechanism Spark already used to cross-build 2.12/2.13: rewrite the POMs with dev/change-scala-version.sh, then select the target with a Maven profile. Both Maven and sbt follow, the latter through the existing -P passthrough into sbt-pom-reader. Nothing here is expected to build the whole reactor under Scala 3 yet. The point is to have a build target so progress can be measured module by module, which is how the scala-2.13 profile was introduced in SPARK-29296 roughly two years before 2.13 worked. Version handling. scala.version is now a reference (${scala213.version} or ${scala3.version}) that the switch flips, rather than a literal interpolated from `mvn help:evaluate -Pscala-`. Each full version is therefore defined once and the two cannot drift; the previous scheme required a scala- profile to exist purely to be evaluated, and the commented-out scala-2.13 profile it read had drifted to 2.13.11. Coordinates that must not follow scala.binary.version get their own property, which also keeps the artifactId rewrite away from them: mvn-scalafmt (a build-time plugin with no _3 build), chill (SPARK-55152, consumed as chill_2.13 from Scala 3), and the Scala standard library itself, whose coordinate differs between 2 and 3 (scala-library vs scala3-library_3). The scala-3 profile keeps scala-library managed even though it no longer names it, or a transitive 2.13 stdlib silently downgrades the one underneath scala3-library_3. Fixes to dev/change-scala-version.sh, most of which are latent today rather than new: - the artifactId rewrite matched any line containing the text "artifactId", so it also corrupted property declarations whose name merely ends in it; it is now anchored to the element - FROM_VERSION was hardcoded to 2.12, which made a switch to any newer version one-way - the scala.binary.version pattern required a dot and so could never match a bare 3 - the scaladoc/mima branch injected -Pscala-2.13 when switching *to* 2.13, from when 2.12 was the default; 2.13 is now the default and that profile no longer exists, so running the script corrupted dev/mima and docs/_plugins/build_api_docs.rb genjavadoc is dropped under Scala 3 in the sbt build: it is a Scala 2 compiler plugin published against the full Scala version, and leaving its -P: options behind makes the Scala 3 compiler fail with "bad option". Co-Authored-By: Claude Opus 4.8 (1M context) --- common/tags/pom.xml | 2 +- common/unsafe/pom.xml | 2 +- connector/kafka-0-10-assembly/pom.xml | 2 +- core/pom.xml | 4 +- dev/change-scala-version.sh | 33 ++++++++------ examples/pom.xml | 2 +- pom.xml | 65 +++++++++++++++++++++++++-- project/SparkBuild.scala | 23 +++++++++- repl/pom.xml | 2 +- sql/connect/common/pom.xml | 2 +- sql/connect/shims/pom.xml | 2 +- sql/pipelines/pom.xml | 2 +- streaming/pom.xml | 2 +- udf/worker/core/pom.xml | 2 +- udf/worker/grpc/pom.xml | 2 +- udf/worker/proto/pom.xml | 2 +- 16 files changed, 117 insertions(+), 32 deletions(-) diff --git a/common/tags/pom.xml b/common/tags/pom.xml index cad844bbef6ab..e86ebcb032f3e 100644 --- a/common/tags/pom.xml +++ b/common/tags/pom.xml @@ -37,7 +37,7 @@ org.scala-lang - scala-library + ${scala.library.artifact} ${scala.version} diff --git a/common/unsafe/pom.xml b/common/unsafe/pom.xml index faf6857659ce5..630ca111b7972 100644 --- a/common/unsafe/pom.xml +++ b/common/unsafe/pom.xml @@ -72,7 +72,7 @@ com.twitter - chill_${scala.binary.version} + chill_${chill.scala.binary.version} com.esotericsoftware diff --git a/connector/kafka-0-10-assembly/pom.xml b/connector/kafka-0-10-assembly/pom.xml index 7b6c9e7a4a655..eb1798ad40003 100644 --- a/connector/kafka-0-10-assembly/pom.xml +++ b/connector/kafka-0-10-assembly/pom.xml @@ -113,7 +113,7 @@ org.scala-lang - scala-library + ${scala.library.artifact} provided diff --git a/core/pom.xml b/core/pom.xml index 5db9a65233838..8ef5bf3c812d7 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -68,7 +68,7 @@ com.twitter - chill_${scala.binary.version} + chill_${chill.scala.binary.version} com.twitter @@ -281,7 +281,7 @@ org.scala-lang - scala-library + ${scala.library.artifact} org.scala-lang diff --git a/dev/change-scala-version.sh b/dev/change-scala-version.sh index 52087a0c6020c..30a31fb871ad5 100755 --- a/dev/change-scala-version.sh +++ b/dev/change-scala-version.sh @@ -19,7 +19,7 @@ set -e -VALID_VERSIONS=( 2.13 ) +VALID_VERSIONS=( 2.13 3 ) usage() { echo "Usage: $(basename $0) [-h|--help] @@ -45,7 +45,7 @@ check_scala_version() { check_scala_version "$TO_VERSION" if [ $TO_VERSION = "2.13" ]; then - FROM_VERSION="2.12" + FROM_VERSION="3" else FROM_VERSION="2.13" fi @@ -57,7 +57,9 @@ sed_i() { BASEDIR=$(dirname $0)/.. for f in $(find "$BASEDIR" -name 'pom.xml' -not -path '*target*'); do echo $f - sed_i 's/\(artifactId.*\)_'$FROM_VERSION'/\1_'$TO_VERSION'/g' $f + # Anchor to the element: a looser match also corrupts property names + # ending in "artifactId". + sed_i 's|\([^<]*\)_'$FROM_VERSION'<|\1_'$TO_VERSION'<|g' $f sed_i 's/^\([[:space:]]*\)\(\)/\1\2/' $f sed_i 's/^\([[:space:]]*\)$/\1 + 2.13.18 + 3.3.8 + ${scala213.version} 2.13 + + scala-library + + 2.13 + + 2.13 2.2.0 4.9.10 false @@ -488,7 +499,7 @@ com.twitter - chill_${scala.binary.version} + chill_${chill.scala.binary.version} ${chill.version} @@ -1231,7 +1242,7 @@ org.scala-lang - scala-library + ${scala.library.artifact} ${scala.version} @@ -3359,7 +3370,7 @@ org.antipathy - mvn-scalafmt_${scala.binary.version} + mvn-scalafmt_${scalafmt.plugin.scala.binary.version} 1.1.1713302731.c3d0074 ${scalafmt.validateOnly} @@ -3574,6 +3585,52 @@ --> + + + scala-3 + + ${scala3.version} + 3 + scala3-library_3 + + + + + + org.scala-lang + scala-library + ${scala213.version} + + + + + + + + net.alchim31.maven + scala-maven-plugin + + + + -unchecked + -deprecation + -feature + -release + 17 + -Wunused:imports + -source:3.3-migration + + + + + + + + + + org.scala-lang + scala-reflect + ${scala213.version} + + + org.scala-lang + scala-compiler + ${scala213.version} + From 0f295cbea896dd186695bcbfcd3d8e17dcc40f55 Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Wed, 22 Jul 2026 17:04:26 -0700 Subject: [PATCH 03/12] [SPARK-58200][BUILD][FOLLOWUP] Let repl inherit the managed scala-compiler version repl declared scala-compiler and scala-reflect with an explicit ${scala.version}, which overrides dependencyManagement. Under -Pscala-3 that resolved to a nonexistent 3.x artifact even with the managed pin in the scala-3 profile. Drop the explicit versions so repl inherits the managed one, as every other module already does. The 2.13 build is unaffected: the managed version is ${scala.version}, so the effective POM is byte-identical there. Co-Authored-By: Claude Opus 4.8 (1M context) --- repl/pom.xml | 2 -- 1 file changed, 2 deletions(-) diff --git a/repl/pom.xml b/repl/pom.xml index 69c7fc84cfb73..d1cf7f5c6c2a5 100644 --- a/repl/pom.xml +++ b/repl/pom.xml @@ -61,12 +61,10 @@ org.scala-lang scala-compiler - ${scala.version} org.scala-lang scala-reflect - ${scala.version} org.slf4j From 696abb3225f7a6c7a97115234921d447b46d7c8a Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Wed, 22 Jul 2026 17:15:55 -0700 Subject: [PATCH 04/12] [SPARK-58200][BUILD][FOLLOWUP] Pin kafka and classutil to the 2.13 line under scala-3 kafka and classutil publish no Scala 3 build, so kafka_${scala.binary.version} and classutil_${scala.binary.version} resolved to nonexistent _3 artifacts under -Pscala-3. Pin both coordinates the way chill already is. kafka is test scope only; the connectors compile against the Java kafka-clients. Pinning it to 2.13 makes it drag in jackson-module-scala_2.13, which conflicts with the jackson-module-scala_3 that core provides, so exclude it alongside the jackson-core, jackson-databind and jackson-annotations exclusions already there. The exclusion uses the pinned property rather than a literal _2.13 so change-scala-version.sh does not rewrite it to a suffix that matches nothing. The 2.13 build is unaffected: the resolved test classpath of sql-kafka-0-10 is identical before and after, and jackson-module-scala_2.13 remains on it via core. Co-Authored-By: Claude Opus 4.8 (1M context) --- connector/kafka-0-10-sql/pom.xml | 6 +++++- connector/kafka-0-10/pom.xml | 6 +++++- pom.xml | 9 +++++++-- tools/pom.xml | 2 +- 4 files changed, 18 insertions(+), 5 deletions(-) diff --git a/connector/kafka-0-10-sql/pom.xml b/connector/kafka-0-10-sql/pom.xml index d37f582a63258..f12852c2805f2 100644 --- a/connector/kafka-0-10-sql/pom.xml +++ b/connector/kafka-0-10-sql/pom.xml @@ -104,7 +104,7 @@ org.apache.kafka - kafka_${scala.binary.version} + kafka_${kafka.scala.binary.version} ${kafka.version} test @@ -120,6 +120,10 @@ com.fasterxml.jackson.core jackson-annotations + + com.fasterxml.jackson.module + jackson-module-scala_${kafka.scala.binary.version} + commons-logging commons-logging diff --git a/connector/kafka-0-10/pom.xml b/connector/kafka-0-10/pom.xml index aa67d515d15c7..8f56a8f4ab03b 100644 --- a/connector/kafka-0-10/pom.xml +++ b/connector/kafka-0-10/pom.xml @@ -80,7 +80,7 @@ org.apache.kafka - kafka_${scala.binary.version} + kafka_${kafka.scala.binary.version} ${kafka.version} test @@ -96,6 +96,10 @@ com.fasterxml.jackson.core jackson-annotations + + com.fasterxml.jackson.module + jackson-module-scala_${kafka.scala.binary.version} + commons-logging commons-logging diff --git a/pom.xml b/pom.xml index 2830478c740cd..b3a714133b413 100644 --- a/pom.xml +++ b/pom.xml @@ -183,9 +183,14 @@ scala-library 2.13 - + 2.13 + + 2.13 + 2.13 2.2.0 4.9.10 false diff --git a/tools/pom.xml b/tools/pom.xml index 14a372603cd3f..82b32b3b267e7 100644 --- a/tools/pom.xml +++ b/tools/pom.xml @@ -43,7 +43,7 @@ org.clapper - classutil_${scala.binary.version} + classutil_${classutil.scala.binary.version} org.ow2.asm From bef233839fc219f6a0f4866c6eab8cb0cdfc30b8 Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Wed, 22 Jul 2026 21:02:58 -0700 Subject: [PATCH 05/12] [SPARK-58200][CORE][FOLLOWUP] Make ConfigBuilder parse under Scala 3 ### What changes were proposed in this pull request? Two source changes in `ConfigBuilder.scala` so `common/utils` compiles under the `scala-3` profile: - `enum` is a keyword in Scala 3 and cannot be used as a binding name, so `case Some(enum) => enum` becomes `case Some(e) => e`. - Scala 3 requires parentheses around a typed lambda parameter, so `{ v: T => v.toString }` becomes `{ (v: T) => v.toString }`. ### Why are the changes needed? Without them `common/utils` fails with 17 errors under `-Pscala-3`, which also blocks `unsafe`, `kvstore` and `variant` since they depend on it. That leaves only 16 Scala files in the whole reactor actually reaching the Scala 3 compiler, too little for CI to prove anything about the profile. ### Does this PR introduce any user-facing change? No. ### How was this patch tested? Both changes are accepted by Scala 2.13, so the existing build is unaffected: `common-utils/compile` and `common-utils/Test/compile` pass on 2.13. Under `-Pscala-3` the 14 modules below now compile, emitting 112 `.tasty` files (899 classes): tags, common-utils, common-utils-java, unsafe, kvstore, variant, sketch, launcher, network-common, network-shuffle, connect-shims, udf-worker-proto, udf-worker-core, udf-worker-grpc. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../org/apache/spark/internal/config/ConfigBuilder.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/common/utils/src/main/scala/org/apache/spark/internal/config/ConfigBuilder.scala b/common/utils/src/main/scala/org/apache/spark/internal/config/ConfigBuilder.scala index a914a96384593..ec16fee2d0e91 100644 --- a/common/utils/src/main/scala/org/apache/spark/internal/config/ConfigBuilder.scala +++ b/common/utils/src/main/scala/org/apache/spark/internal/config/ConfigBuilder.scala @@ -56,7 +56,7 @@ private object ConfigHelpers { def toEnum[E <: Enum[E]](s: String, enumClass: Class[E], key: String): E = { enumClass.getEnumConstants.find(_.name().equalsIgnoreCase(s.trim)) match { - case Some(enum) => enum + case Some(e) => e case None => throw configOutOfRangeOfOptionsError(key, s, enumClass.getEnumConstants) } } @@ -155,7 +155,7 @@ private[spark] class TypedConfigBuilder[T]( import ConfigHelpers._ def this(parent: ConfigBuilder, converter: String => T) = { - this(parent, converter, { v: T => v.toString }) + this(parent, converter, { (v: T) => v.toString }) } /** Apply a transformation to the user-provided values of the config entry. */ From ff746e342cb010c2347b6222c91703c4d5cb03ee Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Wed, 22 Jul 2026 21:10:50 -0700 Subject: [PATCH 06/12] [SPARK-58200][INFRA][FOLLOWUP] Add a Scala 3 build job ### What changes were proposed in this pull request? Adds a `scala3` job to `build_and_test.yml`, modelled on the existing `java25` job. It runs three checks: 1. `dev/change-scala-version.sh 3` followed by `2.13` must leave the tree unchanged, guarding the reversibility of the version switch. 2. The 14 modules that currently build under Scala 3 are compiled with `-Pscala-3`. 3. Each module that has Scala sources must have emitted `.tasty` files. Only dotty produces those, so this rules out a silent fallback to the Scala 2 compiler. Like `java25`, the job is enabled on forks and disabled on scheduled `apache/spark` runs, so it reports on pull requests. ### Why are the changes needed? The `scala-3` profile added by SPARK-58200 had no CI coverage, so nothing stopped a later change from breaking the switch or the modules that compile. All four build defects fixed in this PR were the kind this job catches. The module list is deliberately explicit rather than derived: the rest of the reactor still needs source migration under SPARK-54150, and an allowlist that has to be extended as subtasks land is clearer than a filter that could pass for the wrong reason. ### Does this PR introduce any user-facing change? No. ### How was this patch tested? Locally, on the merged tree: the round-trip leaves 0 modified files, the 14 modules compile under `-Pscala-3` producing 899 classes and 112 `.tasty` files, and the `.tasty` assertion was checked against negative controls (a Java-only module and a missing directory both correctly fail it). Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/build_and_test.yml | 115 +++++++++++++++++++++++++++ 1 file changed, 115 insertions(+) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 0bcda10e3b6c3..dcafb13a2840d 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -137,6 +137,7 @@ jobs: ui=true docs=true java25=true + scala3=true else pyspark_install=false pyspark_connect_old_client=false @@ -150,6 +151,7 @@ jobs: ui=false docs=false java25=false + scala3=false fi build=`./dev/is-changed.py -m "core,unsafe,kvstore,avro,utils,utils-java,network-common,network-shuffle,repl,launcher,examples,sketch,variant,api,catalyst,hive-thriftserver,mllib-local,mllib,graphx,streaming,sql-kafka-0-10,streaming-kafka-0-10,streaming-kinesis-asl,kubernetes,hadoop-cloud,spark-ganglia-lgpl,profiler,protobuf,yarn,connect,sql,hive,pipelines"` build_core_utils=`./dev/is-changed.py -m "core,unsafe,kvstore,utils,utils-java,network-common,network-shuffle,sketch,variant,launcher"` @@ -166,6 +168,7 @@ jobs: \"docker-integration-tests\": \"$docker\", \"lint\" : \"true\", \"java25\" : \"$java25\", + \"scala3\" : \"$scala3\", \"docs\" : \"$docs\", \"yarn\" : \"$yarn\", \"k8s-integration-tests\" : \"$kubernetes\", @@ -1140,6 +1143,118 @@ jobs: export MAVEN_CLI_OPTS="--no-transfer-progress" ./build/mvn $MAVEN_CLI_OPTS -DskipTests -Pyarn -Pkubernetes -Pvolcano -Phive -Phive-thriftserver -Phadoop-cloud -Pjvm-profiler -Pspark-ganglia-lgpl -Pkinesis-asl clean install + # SPARK-58200: Scala 3 cross-build. Only the modules listed below compile under Scala 3 + # today; the rest of the reactor still needs source migration (SPARK-54150). This job + # guards the two things the scala-3 profile promises: that the version switch is + # reversible, and that the modules that do build keep building. + # + # Scope, deliberately narrow: main sources only, built with SBT. Test sources are not + # migrated yet and do not compile, and the Maven path is verified by hand rather than + # here, so a Maven-only regression would not be caught. + scala3: + needs: [precondition] + if: fromJson(needs.precondition.outputs.required).scala3 == 'true' + name: Scala 3 build + runs-on: ubuntu-latest + timeout-minutes: 60 + env: + # Modules whose main sources compile under Scala 3. Extend as SPARK-54150 + # subtasks land. + SCALA3_MODULES: >- + tags/compile + common-utils/compile + common-utils-java/compile + unsafe/compile + kvstore/compile + variant/compile + sketch/compile + launcher/compile + network-common/compile + network-shuffle/compile + connect-shims/compile + udf-worker-proto/compile + udf-worker-core/compile + udf-worker-grpc/compile + steps: + - name: Checkout Spark repository + uses: actions/checkout@v6 + with: + fetch-depth: 0 + repository: apache/spark + ref: ${{ inputs.branch }} + - name: Sync the current branch with the latest in Apache Spark + if: github.repository != 'apache/spark' + run: | + git fetch https://github.com/$GITHUB_REPOSITORY.git ${GITHUB_REF#refs/heads/} + git -c user.name='Apache Spark Test Account' -c user.email='sparktestacc@gmail.com' merge --no-commit --progress --squash FETCH_HEAD + git -c user.name='Apache Spark Test Account' -c user.email='sparktestacc@gmail.com' commit -m "Merged commit" --allow-empty + - name: Cache SBT and Maven + uses: actions/cache@v5 + with: + path: | + build/apache-maven-* + build/*.jar + ~/.sbt + key: build-${{ runner.os }}-${{ hashFiles('**/pom.xml', 'project/build.properties', 'build/mvn', 'build/sbt', 'build/sbt-launch-lib.bash', 'build/spark-build-info') }} + restore-keys: | + build-${{ runner.os }}- + # Own key: this is the only Scala 3 build, so the shared coursier cache holds + # none of the Scala 3 artifacts and cross-restoring would just save a copy of + # the 2.13 closure under this key. + - name: Cache Coursier local repository + uses: actions/cache@v5 + with: + path: ~/.cache/coursier + key: coursier-scala3-${{ runner.os }}-${{ hashFiles('**/pom.xml', '**/plugins.sbt') }} + restore-keys: | + coursier-scala3-${{ runner.os }}- + - name: Install Java ${{ inputs.java }} + uses: actions/setup-java@v5 + with: + distribution: zulu + java-version: ${{ inputs.java }} + - name: Check that the Scala version switch is reversible + run: | + ./dev/change-scala-version.sh 3 + ./dev/change-scala-version.sh 2.13 + # Switching to Scala 3 and back must be a no-op. If this fails, a POM was + # rewritten in one direction but not the other. + git diff --exit-code + - name: Build with SBT + run: | + ./dev/change-scala-version.sh 3 + ./build/sbt -Pscala-3 $SCALA3_MODULES + - name: Check that the Scala 3 compiler actually ran + run: | + # Only dotty emits .tasty files, so their presence rules out a silent + # fallback to the Scala 2 compiler. The set of modules to check is derived + # from what was built rather than hand-listed, so it cannot drift from + # SCALA3_MODULES. The output directory is pinned to the exact compiler + # version so leftovers from a previous scala3.version cannot satisfy it. + version=$(./build/mvn -q help:evaluate -Dexpression=scala3.version -DforceStdout -N) + echo "Checking target/scala-$version output directories" + checked=0 + for classes in $(find . -type d -path "*/target/scala-$version/classes" | sort); do + module=${classes%%/target/*} + # Java-only modules legitimately produce no .tasty files. + if [ -z "$(find "$module/src/main" -name '*.scala' 2>/dev/null | head -1)" ]; then + continue + fi + count=$(find "$classes" -name '*.tasty' | wc -l) + echo "$module: $count .tasty file(s)" + if [ "$count" -eq 0 ]; then + echo "ERROR: $module has Scala sources but emitted no .tasty files;" + echo " the Scala 3 compiler did not run for it." + exit 1 + fi + checked=$((checked + 1)) + done + if [ "$checked" -eq 0 ]; then + echo "ERROR: no module with Scala sources was verified; the build produced nothing." + exit 1 + fi + echo "Verified $checked module(s) compiled by the Scala 3 compiler." + # Documentation build docs: needs: [precondition, infra-image] From 9dc639957bc7e338067d776fc9bd60e039e17c7d Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Wed, 22 Jul 2026 21:43:35 -0700 Subject: [PATCH 07/12] [SPARK-58200][BUILD][FOLLOWUP] Fix the Maven path under the scala-3 profile ### What changes were proposed in this pull request? Two fixes found by building with Maven rather than SBT: - `dev/change-scala-version.sh` now rewrites `` alongside `` and ``. Previously the stdlib coordinate was only swapped inside the `scala-3` profile, so the rewritten POM claimed `scala.version` 3.3.8 while still naming `scala-library`, and resolution failed on the nonexistent `org.scala-lang:scala-library:3.3.8` unless `-Pscala-3` was also passed. - The `scala-3` profile sets `maven.scaladoc.skip`. `scala-maven-plugin` passes `-doc-format:html` and `-doc-title` to the `doc-jar` goal by default; Scala 3's scaladoc rejects both, and because `-doc-title` is dropped its argument is then read as a source file, failing the build at the first module with Scala sources. ### Why are the changes needed? Without them `./build/mvn -Pscala-3 ... install` fails immediately, even though the same modules build with SBT. The CI job added in this PR uses SBT, so neither defect would have been caught there. ### Does this PR introduce any user-facing change? No. `maven.scaladoc.skip` stays `false` outside the profile, so scaladoc still runs for Scala 2.13. ### How was this patch tested? `./dev/change-scala-version.sh 3` followed by `2.13` still leaves the tree unchanged. After switching to Scala 3, `dependency:resolve` without any profile now resolves `scala3-library_3:3.3.8` and `scala-library:2.13.18`. With `-Pscala-3 -Dmaven.test.skip=true`, all 14 modules that build under Scala 3 install successfully. Test sources are not migrated yet, so `-DskipTests` alone still fails in `udf-worker-core`; that belongs to SPARK-54150. Co-Authored-By: Claude Opus 4.8 (1M context) --- dev/change-scala-version.sh | 12 ++++++++++++ pom.xml | 5 +++++ 2 files changed, 17 insertions(+) diff --git a/dev/change-scala-version.sh b/dev/change-scala-version.sh index 30a31fb871ad5..e133cca6303f3 100755 --- a/dev/change-scala-version.sh +++ b/dev/change-scala-version.sh @@ -86,6 +86,18 @@ sed_i '1,//s|[^<]*|[0-9][0-9.]*[0-9][0-9.]*'$TO_VERSION'/s|>[^<]*|>'"$SCALA_LIBRARY_ARTIFACT"'|' \ + "$BASEDIR/pom.xml" + # Update source of scaladocs. 2.13 is the default and needs no flag; only Scala 3 is # selected with -Pscala-3, so this tracks the non-default version rather than the target. echo "$BASEDIR/docs/_plugins/build_api_docs.rb" diff --git a/pom.xml b/pom.xml index 38a24008a83eb..29637e2afbb58 100644 --- a/pom.xml +++ b/pom.xml @@ -3598,6 +3598,11 @@ ${scala3.version} 3 scala3-library_3 + + true From b254cd3b83f95ab1df971456bc13f58d641b4eb5 Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Wed, 22 Jul 2026 21:50:15 -0700 Subject: [PATCH 08/12] [SPARK-58200][BUILD][FOLLOWUP] Manage scala-compiler and scala-reflect on the 2.13 line ### What changes were proposed in this pull request? Points the root `dependencyManagement` entries for `scala-compiler` and `scala-reflect` at `${scala213.version}` instead of `${scala.version}`, and drops the duplicate declarations the `scala-3` profile added for them. ### Why are the changes needed? Neither artifact is published for Scala 3, so `${scala.version}` resolves to a nonexistent 3.x coordinate once the version switch has run. Pinning them in the profile worked, but re-declaring the same groupId:artifactId there left two managed entries for one dependency, and the profile copy carried no `` element, so whether the base `scala-xml` exclusion survived depended on Maven's profile-injection merge semantics. Managing them on the 2.13 line in the base POM is unambiguous, keeps the exclusion attached to the single declaration that has it, and makes the switched POM resolve these artifacts with or without `-Pscala-3`. ### Does this PR introduce any user-facing change? No. Under Scala 2.13 `scala.version` is `${scala213.version}`, so the managed versions are unchanged. ### How was this patch tested? The effective POM on Scala 2.13 is identical before and after apart from the removed profile entries; both managed versions still resolve to 2.13.18. After `./dev/change-scala-version.sh 3`, `scala-compiler` and `scala-reflect` resolve to 2.13.18 and `scala3-library_3` to 3.3.8, both with and without `-Pscala-3`. `-Pscala-3 -Dmaven.test.skip=true install` of the 14 Scala 3 modules still succeeds. Co-Authored-By: Claude Opus 4.8 (1M context) --- pom.xml | 21 ++++++--------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/pom.xml b/pom.xml index 29637e2afbb58..4267ce493e700 100644 --- a/pom.xml +++ b/pom.xml @@ -1232,7 +1232,11 @@ org.scala-lang scala-compiler - ${scala.version} + + ${scala213.version} org.scala-lang.modules @@ -1243,7 +1247,7 @@ org.scala-lang scala-reflect - ${scala.version} + ${scala213.version} org.scala-lang @@ -3613,19 +3617,6 @@ scala-library ${scala213.version} - - - org.scala-lang - scala-reflect - ${scala213.version} - - - org.scala-lang - scala-compiler - ${scala213.version} - From be8cee091f77e0e06e07e4fccaaf28664d2632dc Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Wed, 22 Jul 2026 21:50:15 -0700 Subject: [PATCH 09/12] [SPARK-58200][INFRA][FOLLOWUP] Cover the Maven path in the Scala 3 build job ### What changes were proposed in this pull request? Turns the `scala3` job into a matrix over `sbt` and `maven`, with `fail-fast` disabled so one build tool failing still reports the other. The Maven leg runs `-Pscala-3 -Dmaven.test.skip=true ... install` over the same 14 modules and caches `~/.m2/repository` under its own key. `-Dmaven.test.skip` rather than `-DskipTests`, since the latter still compiles test sources, which are not migrated to Scala 3 yet. The `.tasty` verification now picks the output directory per build tool: SBT writes to `target/scala-`, Maven to `target/scala-`. ### Why are the changes needed? The two paths diverge, and the SBT leg alone does not cover the difference. Testing Maven by hand found two defects that an SBT-only job could not have caught: the switched POM not resolving without `-Pscala-3`, and the `scala-maven-plugin` scaladoc options that Scala 3 rejects. Both are fixed in this PR, and both lived in configuration SBT never reads. ### Does this PR introduce any user-facing change? No. ### How was this patch tested? Both legs were run locally on the merged tree. SBT compiles all 14 modules; Maven installs all 14 plus the parent POM. The `.tasty` verification was exercised against both output layouts and reports the same 4 modules with Scala sources (`tags`, `common-utils`, `connect-shims`, `udf-worker-core`). Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/build_and_test.yml | 57 ++++++++++++++++++++++------ 1 file changed, 46 insertions(+), 11 deletions(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index dcafb13a2840d..910c2563b27a2 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -1148,18 +1148,27 @@ jobs: # guards the two things the scala-3 profile promises: that the version switch is # reversible, and that the modules that do build keep building. # - # Scope, deliberately narrow: main sources only, built with SBT. Test sources are not - # migrated yet and do not compile, and the Maven path is verified by hand rather than - # here, so a Maven-only regression would not be caught. + # Scope, deliberately narrow: main sources only. Test sources are not migrated yet and + # do not compile, so SBT builds `compile` and Maven runs with -Dmaven.test.skip=true. + # Both build tools are covered because the two paths diverge: the scala-3 profile's + # compiler arguments and scaladoc settings apply only to Maven, and SBT resolves the + # module graph itself rather than through the reactor. scala3: needs: [precondition] if: fromJson(needs.precondition.outputs.required).scala3 == 'true' - name: Scala 3 build + name: Scala 3 build (${{ matrix.build-tool }}) runs-on: ubuntu-latest timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + build-tool: + - sbt + - maven env: # Modules whose main sources compile under Scala 3. Extend as SPARK-54150 - # subtasks land. + # subtasks land. The two lists must name the same modules; SBT uses project + # names and Maven uses directories. SCALA3_MODULES: >- tags/compile common-utils/compile @@ -1175,6 +1184,7 @@ jobs: udf-worker-proto/compile udf-worker-core/compile udf-worker-grpc/compile + SCALA3_MAVEN_MODULES: "common/tags,common/utils,common/utils-java,common/unsafe,common/kvstore,common/variant,common/sketch,launcher,common/network-common,common/network-shuffle,sql/connect/shims,udf/worker/proto,udf/worker/core,udf/worker/grpc" steps: - name: Checkout Spark repository uses: actions/checkout@v6 @@ -1198,16 +1208,25 @@ jobs: key: build-${{ runner.os }}-${{ hashFiles('**/pom.xml', 'project/build.properties', 'build/mvn', 'build/sbt', 'build/sbt-launch-lib.bash', 'build/spark-build-info') }} restore-keys: | build-${{ runner.os }}- - # Own key: this is the only Scala 3 build, so the shared coursier cache holds - # none of the Scala 3 artifacts and cross-restoring would just save a copy of - # the 2.13 closure under this key. + # Own keys: this is the only Scala 3 build, so the shared caches hold none of the + # Scala 3 artifacts and cross-restoring would just save a copy of the 2.13 closure + # under these keys. - name: Cache Coursier local repository + if: matrix.build-tool == 'sbt' uses: actions/cache@v5 with: path: ~/.cache/coursier key: coursier-scala3-${{ runner.os }}-${{ hashFiles('**/pom.xml', '**/plugins.sbt') }} restore-keys: | coursier-scala3-${{ runner.os }}- + - name: Cache Maven local repository + if: matrix.build-tool == 'maven' + uses: actions/cache@v5 + with: + path: ~/.m2/repository + key: maven-scala3-${{ runner.os }}-java${{ inputs.java }}-${{ hashFiles('**/pom.xml') }} + restore-keys: | + maven-scala3-${{ runner.os }}-java${{ inputs.java }}- - name: Install Java ${{ inputs.java }} uses: actions/setup-java@v5 with: @@ -1221,17 +1240,33 @@ jobs: # rewritten in one direction but not the other. git diff --exit-code - name: Build with SBT + if: matrix.build-tool == 'sbt' run: | ./dev/change-scala-version.sh 3 ./build/sbt -Pscala-3 $SCALA3_MODULES + - name: Build with Maven + if: matrix.build-tool == 'maven' + run: | + ./dev/change-scala-version.sh 3 + export MAVEN_OPTS="-Xss64m -Xmx4g -Xms4g -XX:ReservedCodeCacheSize=128m -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN" + # -Dmaven.test.skip, not -DskipTests: the latter still compiles test sources, + # which are not migrated to Scala 3 yet. -am pulls in the parent POM. + ./build/mvn --no-transfer-progress -Pscala-3 -Dmaven.test.skip=true \ + -pl "$SCALA3_MAVEN_MODULES" -am install - name: Check that the Scala 3 compiler actually ran run: | # Only dotty emits .tasty files, so their presence rules out a silent # fallback to the Scala 2 compiler. The set of modules to check is derived # from what was built rather than hand-listed, so it cannot drift from - # SCALA3_MODULES. The output directory is pinned to the exact compiler - # version so leftovers from a previous scala3.version cannot satisfy it. - version=$(./build/mvn -q help:evaluate -Dexpression=scala3.version -DforceStdout -N) + # SCALA3_MODULES. + if [ '${{ matrix.build-tool }}' = 'sbt' ]; then + # SBT writes to target/scala-. Pinning to the exact + # version stops leftovers from a previous scala3.version satisfying the check. + version=$(./build/mvn -q help:evaluate -Dexpression=scala3.version -DforceStdout -N) + else + # Maven's outputDirectory follows scala.binary.version, which is just "3". + version=3 + fi echo "Checking target/scala-$version output directories" checked=0 for classes in $(find . -type d -path "*/target/scala-$version/classes" | sort); do From 242974a6c633eb36d74ca27bc56757b81e5b5d83 Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Wed, 22 Jul 2026 22:10:56 -0700 Subject: [PATCH 10/12] [SPARK-58200][INFRA][FOLLOWUP] List the Scala 3 Maven modules one per line ### What changes were proposed in this pull request? Reformats `SCALA3_MAVEN_MODULES` from a single long comma-separated string to one module per line, matching how `SCALA3_MODULES` is already written, and joins the entries with commas where Maven's `-pl` consumes them. The list has to be space separated in YAML: a folded scalar replaces newlines with spaces, so keeping the commas in the value would produce `a, b` rather than `a,b` and Maven would not match the module names. ### Why are the changes needed? The two module lists cover the same 14 modules and are easier to compare, and to extend as SPARK-54150 subtasks land, when they are formatted the same way. ### Does this PR introduce any user-facing change? No. ### How was this patch tested? `-Pscala-3 -Dmaven.test.skip=true -pl "$(echo $SCALA3_MAVEN_MODULES | tr ' ' ',')" -am install` was run with the reformatted value and installs all 14 modules plus the parent POM. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/build_and_test.yml | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 910c2563b27a2..8ebb70d0b14d4 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -1184,7 +1184,22 @@ jobs: udf-worker-proto/compile udf-worker-core/compile udf-worker-grpc/compile - SCALA3_MAVEN_MODULES: "common/tags,common/utils,common/utils-java,common/unsafe,common/kvstore,common/variant,common/sketch,launcher,common/network-common,common/network-shuffle,sql/connect/shims,udf/worker/proto,udf/worker/core,udf/worker/grpc" + # Space separated here for readability; joined with commas for Maven's -pl below. + SCALA3_MAVEN_MODULES: >- + common/tags + common/utils + common/utils-java + common/unsafe + common/kvstore + common/variant + common/sketch + launcher + common/network-common + common/network-shuffle + sql/connect/shims + udf/worker/proto + udf/worker/core + udf/worker/grpc steps: - name: Checkout Spark repository uses: actions/checkout@v6 @@ -1252,7 +1267,7 @@ jobs: # -Dmaven.test.skip, not -DskipTests: the latter still compiles test sources, # which are not migrated to Scala 3 yet. -am pulls in the parent POM. ./build/mvn --no-transfer-progress -Pscala-3 -Dmaven.test.skip=true \ - -pl "$SCALA3_MAVEN_MODULES" -am install + -pl "$(echo $SCALA3_MAVEN_MODULES | tr ' ' ',')" -am install - name: Check that the Scala 3 compiler actually ran run: | # Only dotty emits .tasty files, so their presence rules out a silent From 247e857f38c33e508f23d4f36194987db6630ab8 Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Wed, 22 Jul 2026 22:15:42 -0700 Subject: [PATCH 11/12] [SPARK-58200][INFRA][FOLLOWUP] Derive the SBT project names from the module list ### What changes were proposed in this pull request? Replaces the two parallel module lists in the `scala3` job with one, given as directories. Maven consumes it directly; the SBT project names are read out of each module POM's `` instead of being listed separately. If a POM does not declare the property, the step fails and names it. SBT would otherwise fall back to the artifactId and build the wrong project, or nothing, while the job still reported success. ### Why are the changes needed? SBT addresses projects by `` and Maven by directory, and neither can be derived from the other by rule, so the two lists had to be kept in sync by hand with nothing detecting drift. Adding a module to one and not the other would have quietly narrowed what the job covers. ### Does this PR introduce any user-facing change? No. ### How was this patch tested? The derived command list is identical to the hand-written one it replaces, and `./build/sbt -Pscala-3` with the derived commands compiles all 14 modules with no errors. The failure path was checked against a POM with no ``; every module in the tree currently declares it, so the guard is for modules added later. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/build_and_test.yml | 42 ++++++++++++++-------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 8ebb70d0b14d4..4302776ed27f6 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -1166,26 +1166,12 @@ jobs: - sbt - maven env: - # Modules whose main sources compile under Scala 3. Extend as SPARK-54150 - # subtasks land. The two lists must name the same modules; SBT uses project - # names and Maven uses directories. + # Modules whose main sources compile under Scala 3, given as directories. + # Extend as SPARK-54150 subtasks land; this is the only list to update. Maven + # takes these directly, and the SBT project names are read out of each POM + # below, since SBT addresses projects by rather than by path + # and the two cannot be derived from one another by rule. SCALA3_MODULES: >- - tags/compile - common-utils/compile - common-utils-java/compile - unsafe/compile - kvstore/compile - variant/compile - sketch/compile - launcher/compile - network-common/compile - network-shuffle/compile - connect-shims/compile - udf-worker-proto/compile - udf-worker-core/compile - udf-worker-grpc/compile - # Space separated here for readability; joined with commas for Maven's -pl below. - SCALA3_MAVEN_MODULES: >- common/tags common/utils common/utils-java @@ -1258,7 +1244,21 @@ jobs: if: matrix.build-tool == 'sbt' run: | ./dev/change-scala-version.sh 3 - ./build/sbt -Pscala-3 $SCALA3_MODULES + # Read the SBT project name out of each module POM. Falling back to the + # artifactId when the property is missing would silently build the wrong + # project, or nothing at all, so treat that as an error instead. + sbt_commands="" + for module in $SCALA3_MODULES; do + name=$(grep -o '[^<]*' "$module/pom.xml" | head -1 | cut -d'>' -f2) + if [ -z "$name" ]; then + echo "ERROR: $module/pom.xml declares no , so the SBT" + echo " project name cannot be derived. Add the property to that POM." + exit 1 + fi + sbt_commands="$sbt_commands $name/compile" + done + echo "SBT commands:$sbt_commands" + ./build/sbt -Pscala-3 $sbt_commands - name: Build with Maven if: matrix.build-tool == 'maven' run: | @@ -1267,7 +1267,7 @@ jobs: # -Dmaven.test.skip, not -DskipTests: the latter still compiles test sources, # which are not migrated to Scala 3 yet. -am pulls in the parent POM. ./build/mvn --no-transfer-progress -Pscala-3 -Dmaven.test.skip=true \ - -pl "$(echo $SCALA3_MAVEN_MODULES | tr ' ' ',')" -am install + -pl "$(echo $SCALA3_MODULES | tr ' ' ',')" -am install - name: Check that the Scala 3 compiler actually ran run: | # Only dotty emits .tasty files, so their presence rules out a silent From 1fb811c6a4aaceda0524f1416d28bf631822123f Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Wed, 22 Jul 2026 22:37:05 -0700 Subject: [PATCH 12/12] [SPARK-58200][INFRA][FOLLOWUP] Build only main sources in the Scala 3 Maven leg ### What changes were proposed in this pull request? Splits the Maven leg of the `scala3` job into two passes: one that packages the test-jars the module set depends on, and one that builds main sources only. ### Why are the changes needed? The single-pass `-Dmaven.test.skip=true` build failed in CI resolving `spark-tags_3:jar:tests`. Maven resolves test-scope dependencies even when it is not compiling tests, and `maven.test.skip` suppresses test-jar packaging along with test compilation, so nothing produced the jar. It passed locally only because a stale artifact from an earlier `-DskipTests` run was still in `~/.m2`. Only three modules publish test-jars the others depend on, and all three have Java test sources only, so packaging them cannot invoke the Scala 3 compiler. Building those first leaves the second pass free to skip test compilation entirely, which matches what the SBT leg compiles: main sources for the same 14 modules. The alternative, `-DskipTests` throughout, would compile every module's test sources. Those are not migrated to Scala 3 yet and fail in `udf-worker-core`. The second pass also passes `-DskipTests`, because scalatest-maven-plugin reads that property rather than `maven.test.skip` and would otherwise run any test classes left in the build directory. ### Does this PR introduce any user-facing change? No. ### How was this patch tested? Ran the step with both `~/.m2` and every `target/scala-3` directory removed first, since earlier runs had been passing against leftover state. Both passes report `BUILD SUCCESS`, all 14 modules plus the parent POM install, no tests execute, and the `.tasty` verification finds the same 4 modules with Scala sources. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/build_and_test.yml | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 4302776ed27f6..da28c0848f8eb 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -1186,6 +1186,15 @@ jobs: udf/worker/proto udf/worker/core udf/worker/grpc + # Modules above that publish a test-jar the others depend on. Maven resolves + # test-scope dependencies even when it is not compiling tests, so these have to + # be packaged before the main-sources-only pass, and -Dmaven.test.skip suppresses + # test-jar packaging along with test compilation. Their test sources are Java + # only, so this pass never invokes the Scala 3 compiler. + SCALA3_TEST_JAR_MODULES: >- + common/tags + common/network-common + common/utils-java steps: - name: Checkout Spark repository uses: actions/checkout@v6 @@ -1264,9 +1273,19 @@ jobs: run: | ./dev/change-scala-version.sh 3 export MAVEN_OPTS="-Xss64m -Xmx4g -Xms4g -XX:ReservedCodeCacheSize=128m -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN" - # -Dmaven.test.skip, not -DskipTests: the latter still compiles test sources, - # which are not migrated to Scala 3 yet. -am pulls in the parent POM. - ./build/mvn --no-transfer-progress -Pscala-3 -Dmaven.test.skip=true \ + export MVN="./build/mvn --no-transfer-progress -Pscala-3" + # Package the test-jars the module set depends on. -DskipTests, not + # -Dmaven.test.skip, because the jars have to be built; only their execution is + # skipped. These modules have Java test sources only. + $MVN -DskipTests \ + -pl "$(echo $SCALA3_TEST_JAR_MODULES | tr ' ' ',')" -am install + # Then build main sources only, matching what the SBT leg compiles. Test sources + # are not migrated to Scala 3 yet, so -Dmaven.test.skip keeps them out; the + # test-jars they would otherwise have produced came from the pass above. + # -DskipTests as well because scalatest-maven-plugin reads that rather than + # maven.test.skip, and would otherwise run whatever test classes it finds. + # -am pulls in the parent POM. + $MVN -Dmaven.test.skip=true -DskipTests \ -pl "$(echo $SCALA3_MODULES | tr ' ' ',')" -am install - name: Check that the Scala 3 compiler actually ran run: |