DRILL-8540: Update Jetty to Version 12

.github/workflows/ci.yml

@@ -33,8 +33,8 @@ jobs:
     timeout-minutes: 150
     strategy:
       matrix:
-        # Java versions to run unit tests
-        java: [ '11', '17', '21' ]
+        # Java versions to run unit tests (Jetty 12 requires Java 17+)
+        java: [ '17', '21' ]
         profile: ['default-hadoop']
       fail-fast: false
     steps:
@@ -79,7 +79,7 @@ jobs:
         uses: actions/setup-java@v4
         with:
           distribution: 'temurin'
-          java-version: '11'
+          java-version: '17'
           cache: 'maven'
       # Caches built protobuf library
       - name: Cache protobufs

.github/workflows/codeql-analysis.yml

@@ -59,6 +59,13 @@ jobs:
     - name: Checkout repository
       uses: actions/checkout@v4
 
+    - name: Setup java
+      uses: actions/setup-java@v4
+      with:
+        distribution: 'temurin'
+        java-version: '17'
+        cache: 'maven'
+
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
       uses: github/codeql-action/init@v2

.github/workflows/publish-snapshot.yml

@@ -31,6 +31,12 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v4
+      - name: Setup java
+        uses: actions/setup-java@v4
+        with:
+          distribution: 'temurin'
+          java-version: '17'
+          cache: 'maven'
       - name: Cache Maven Repository
         uses: actions/cache@v4
         with:

contrib/storage-phoenix/pom.xml

@@ -43,6 +43,17 @@
       <groupId>org.apache.drill.exec</groupId>
       <artifactId>drill-java-exec</artifactId>
       <version>${project.version}</version>
+      <exclusions>
+        <!-- Exclude Jetty 9 from Hadoop transitives to avoid conflicts with Jetty 12 -->
+        <exclusion>
+          <groupId>org.eclipse.jetty</groupId>
+          <artifactId>jetty-webapp</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.eclipse.jetty.websocket</groupId>
+          <artifactId>*</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
     <dependency>
       <groupId>org.apache.drill.exec</groupId>
@@ -128,6 +139,15 @@
           <groupId>org.apache.hadoop</groupId>
           <artifactId>hadoop-yarn-server-tests</artifactId>
         </exclusion>
+        <!-- Exclude Jetty 9 from Hadoop to avoid conflicts with Jetty 12 -->
+        <exclusion>
+          <groupId>org.eclipse.jetty</groupId>
+          <artifactId>*</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.eclipse.jetty.websocket</groupId>
+          <artifactId>*</artifactId>
+        </exclusion>
       </exclusions>
     </dependency>
 
@@ -136,6 +156,17 @@
       <artifactId>hbase-server</artifactId>
       <version>${hbase.version}</version>
       <scope>test</scope>
+      <exclusions>
+        <!-- Exclude Jetty 9 from Hadoop to avoid conflicts with Jetty 12 -->
+        <exclusion>
+          <groupId>org.eclipse.jetty</groupId>
+          <artifactId>*</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.eclipse.jetty.websocket</groupId>
+          <artifactId>*</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
     <dependency>
       <groupId>org.apache.hbase</groupId>
@@ -290,6 +321,14 @@
       <version>1.78.1</version>
       <scope>test</scope>
     </dependency>
+
+    <!-- Jakarta Bean Validation API needed by HBase shaded Jetty/Jersey -->
+    <dependency>
+      <groupId>javax.validation</groupId>
+      <artifactId>validation-api</artifactId>
+      <version>2.0.1.Final</version>
+      <scope>test</scope>
+    </dependency>
   </dependencies>
 
   <build>

docs/dev/DevDocs.md

@@ -19,3 +19,7 @@ For more info about generating and using javadocs see [Javadocs.md](Javadocs.md)
 ## Building with Maven
 
 For more info about the use of maven see [Maven.md](Maven.md)
+
+## Jetty 12 Migration
+
+For information about the Jetty 12 upgrade, known limitations, and developer guidelines see [Jetty12Migration.md](Jetty12Migration.md)

docs/dev/Jetty12Migration.md

@@ -0,0 +1,183 @@
+# Jetty 12 Migration Guide
+
+## Overview
+
+Apache Drill has been upgraded from Jetty 9 to Jetty 12 to address security vulnerabilities and maintain compatibility with modern Java versions. This document describes the changes made, known limitations, and guidance for developers.
+
+## What Changed
+
+### Core API Changes
+
+Jetty 12 introduced significant API changes as part of the Jakarta EE 10 migration:
+
+1. **Servlet API Migration**: `javax.servlet.*` → `jakarta.servlet.*`
+2. **Package Restructuring**: Servlet components moved to `org.eclipse.jetty.ee10.servlet.*`
+3. **Handler API Redesign**: New `org.eclipse.jetty.server.Handler` interface
+4. **Resource Loading**: New `ResourceFactory` API replaces old `Resource` API
+5. **Authentication APIs**: `LoginService.login()` signature changed
+
+### Modified Files
+
+#### Production Code
+
+- **exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/WebServer.java**
+  - Updated to use `ResourceFactory.root()` for resource loading
+  - Fixed null pointer issues when HTTP server is disabled
+
+- **drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/http/WebServer.java**
+  - Updated all imports to `org.eclipse.jetty.ee10.servlet.*`
+  - Modified `LoginService.login()` to new signature with `Function<Boolean, Session>` parameter
+  - Changed to use `IdentityService.newUserIdentity()` for user identity creation
+  - Updated `ResourceFactory` API usage
+  - Updated `SessionAuthentication` constants
+  - Fixed `ServletContextHandler` constructor usage
+
+#### Test Code
+
+The following test classes are temporarily disabled (see Known Limitations below):
+- `TestImpersonationDisabledWithMiniDFS.java`
+- `TestImpersonationMetadata.java`
+- `TestImpersonationQueries.java`
+- `TestInboundImpersonation.java`
+
+## Known Limitations
+
+### Hadoop MiniDFSCluster Test Incompatibility
+
+**Issue**: Tests using Hadoop's MiniDFSCluster cannot run due to Jetty version conflicts.
+
+**Root Cause**: Apache Hadoop 3.x depends on Jetty 9, while Drill now uses Jetty 12. When tests attempt to start both:
+- Drill's embedded web server (Jetty 12)
+- Hadoop's MiniDFSCluster (Jetty 9)
+
+The conflicting Jetty versions on the classpath cause `NoClassDefFoundError` exceptions.
+
+**Affected Tests**:
+- Impersonation tests with HDFS
+- Any tests requiring MiniDFSCluster with Drill's HTTP server enabled
+
+**Resolution Timeline**: These tests will be re-enabled when:
+- Apache Hadoop 4.x is released with Jetty 12 support
+- A Hadoop 3.x maintenance release upgrades to Jetty 12 (tracked in [HADOOP-19625](https://issues.apache.org/jira/browse/HADOOP-19625))
+
+**Current Status**: HADOOP-19625 is open and targets Jetty 12 EE10, but requires Java 17 baseline (tracked in HADOOP-17177). No specific release version or timeline is available yet.
+
+### Why Alternative Solutions Failed
+
+Several approaches were attempted to resolve the Jetty conflict:
+
+1. **Dual Jetty versions in test scope**: Failed because Maven cannot have two different versions of the same artifact on the classpath simultaneously, and the Jetty BOM forces all Jetty artifacts to the same version.
+
+2. **Disabling Drill's HTTP server in tests**: Failed because drill-java-exec classes are compiled against Jetty 12, and the bytecode contains hard references to Jetty 12 classes that fail to load even when the HTTP server is disabled.
+
+3. **Separate test module with Jetty 9**: Failed because depending on the drill-java-exec JAR (compiled with Jetty 12) brings Jetty 12 class references into the test classpath.
+
+### Workarounds for Developers
+
+If you need to test HDFS impersonation functionality:
+
+1. **Integration tests**: Use a real Hadoop cluster instead of MiniDFSCluster
+2. **Manual testing**: Test in HDFS-enabled environments
+3. **Alternative tests**: Use tests with local filesystem instead of MiniDFSCluster (see other impersonation tests that don't require HDFS)
+
+## Developer Guidelines
+
+### Writing New Web Server Code
+
+When adding new HTTP/servlet functionality:
+
+1. Use Jakarta EE 10 imports:
+   ```java
+   import jakarta.servlet.http.HttpServletRequest;
+   import jakarta.servlet.http.HttpServletResponse;
+   ```
+
+2. Use Jetty 12 EE10 servlet packages:
+   ```java
+   import org.eclipse.jetty.ee10.servlet.ServletContextHandler;
+   import org.eclipse.jetty.ee10.servlet.ServletHolder;
+   ```
+
+3. Use `ResourceFactory.root()` for resource loading:
+   ```java
+   ResourceFactory rf = ResourceFactory.root();
+   InputStream stream = rf.newClassLoaderResource("/path/to/resource").newInputStream();
+   ```
+
+4. Use new `ServletContextHandler` constructor pattern:
+   ```java
+   ServletContextHandler handler = new ServletContextHandler(ServletContextHandler.SESSIONS);
+   handler.setContextPath("/");
+   ```
+
+### Writing Tests
+
+1. Tests that start Drill's HTTP server should **not** use Hadoop MiniDFSCluster
+2. If HDFS testing is required, use local filesystem or mark test with `@Ignore` and add comprehensive documentation
+3. When adding `@Ignore` for Jetty conflicts, reference `TestImpersonationDisabledWithMiniDFS` for standard explanation
+
+### Debugging Jetty Issues
+
+Common issues and solutions:
+
+- **NoClassDefFoundError for Jetty classes**: Check that all Jetty dependencies are Jetty 12, not Jetty 9
+- **ClassNotFoundException for javax.servlet**: Should be `jakarta.servlet` with Jetty 12
+- **NullPointerException in ResourceFactory**: Use `ResourceFactory.root()` instead of `ResourceFactory.of(server)`
+- **Incompatible types in ServletContextHandler**: Use new constructor pattern with `SESSIONS` constant
+
+## Dependency Management
+
+### Maven BOM
+
+Drill's parent POM includes the Jetty 12 BOM:
+
+```xml
+<dependencyManagement>
+  <dependencies>
+    <dependency>
+      <groupId>org.eclipse.jetty</groupId>
+      <artifactId>jetty-bom</artifactId>
+      <version>12.0.16</version>
+      <type>pom</type>
+      <scope>import</scope>
+    </dependency>
+  </dependencies>
+</dependencyManagement>
+```
+
+This ensures all Jetty dependencies use version 12.0.16.
+
+### Key Dependencies
+
+Production dependencies include:
+- `jetty-server` - Core server functionality
+- `jetty-ee10-servlet` - Servlet support
+- `jetty-ee10-servlets` - Standard servlet implementations
+- `jetty-security` - Security handlers
+- `jetty-util` - Utility classes
+
+## Migration Checklist for Future Updates
+
+When upgrading Jetty versions in the future:
+
+- [ ] Check Jetty release notes for API changes
+- [ ] Update Jetty BOM version in parent POM
+- [ ] Run full test suite including integration tests
+- [ ] Check for deprecation warnings in web server code
+- [ ] Verify checkstyle compliance
+- [ ] Check HADOOP-19625 status to see if MiniDFSCluster tests can be re-enabled
+- [ ] Update this document with any new changes or limitations
+
+## References
+
+- [Jetty 12 Migration Guide](https://eclipse.dev/jetty/documentation/jetty-12/migration-guide/index.html)
+- [Jakarta EE 10 Documentation](https://jakarta.ee/specifications/platform/10/)
+- [HADOOP-19625: Upgrade Jetty to 12.x](https://issues.apache.org/jira/browse/HADOOP-19625)
+- [HADOOP-17177: Java 17 Support](https://issues.apache.org/jira/browse/HADOOP-17177)
+
+## Support
+
+For questions or issues related to Jetty 12 migration:
+1. Check existing test classes for examples
+2. Review this document and referenced Jetty documentation
+3. File an issue on the Apache Drill JIRA with component "Web Server"

drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/http/AmRestApi.java

@@ -22,15 +22,15 @@
 import java.util.List;
 import java.util.Map;
 
-import javax.annotation.security.PermitAll;
-import javax.ws.rs.DefaultValue;
-import javax.ws.rs.GET;
-import javax.ws.rs.POST;
-import javax.ws.rs.Path;
-import javax.ws.rs.PathParam;
-import javax.ws.rs.Produces;
-import javax.ws.rs.QueryParam;
-import javax.ws.rs.core.MediaType;
+import jakarta.annotation.security.PermitAll;
+import jakarta.ws.rs.DefaultValue;
+import jakarta.ws.rs.GET;
+import jakarta.ws.rs.POST;
+import jakarta.ws.rs.Path;
+import jakarta.ws.rs.PathParam;
+import jakarta.ws.rs.Produces;
+import jakarta.ws.rs.QueryParam;
+import jakarta.ws.rs.core.MediaType;
 
 import org.apache.drill.yarn.appMaster.Dispatcher;
 import org.apache.drill.yarn.appMaster.http.AbstractTasksModel.TaskModel;

drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/http/AuthDynamicFeature.java

@@ -22,17 +22,17 @@
 import org.apache.drill.yarn.appMaster.http.WebUiPageTree.LogInLogOutPages;
 import org.glassfish.jersey.server.model.AnnotatedMethod;
 
-import javax.annotation.Priority;
-import javax.annotation.security.PermitAll;
-import javax.annotation.security.RolesAllowed;
-import javax.ws.rs.Priorities;
-import javax.ws.rs.container.ContainerRequestContext;
-import javax.ws.rs.container.ContainerRequestFilter;
-import javax.ws.rs.container.DynamicFeature;
-import javax.ws.rs.container.ResourceInfo;
-import javax.ws.rs.core.FeatureContext;
-import javax.ws.rs.core.Response;
-import javax.ws.rs.core.SecurityContext;
+import jakarta.annotation.Priority;
+import jakarta.annotation.security.PermitAll;
+import jakarta.annotation.security.RolesAllowed;
+import jakarta.ws.rs.Priorities;
+import jakarta.ws.rs.container.ContainerRequestContext;
+import jakarta.ws.rs.container.ContainerRequestFilter;
+import jakarta.ws.rs.container.DynamicFeature;
+import jakarta.ws.rs.container.ResourceInfo;
+import jakarta.ws.rs.core.FeatureContext;
+import jakarta.ws.rs.core.Response;
+import jakarta.ws.rs.core.SecurityContext;
 import java.net.URI;
 import java.net.URLEncoder;
 

drill-yarn/src/main/java/org/apache/drill/yarn/appMaster/http/CsrfTokenInjectFilter.java

@@ -17,15 +17,15 @@
  */
 package org.apache.drill.yarn.appMaster.http;
 
-import javax.servlet.Filter;
-import javax.servlet.FilterChain;
-import javax.servlet.FilterConfig;
-import javax.servlet.ServletException;
-import javax.servlet.ServletRequest;
-import javax.servlet.ServletResponse;
-import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpSession;
-import javax.ws.rs.HttpMethod;
+import jakarta.servlet.Filter;
+import jakarta.servlet.FilterChain;
+import jakarta.servlet.FilterConfig;
+import jakarta.servlet.ServletException;
+import jakarta.servlet.ServletRequest;
+import jakarta.servlet.ServletResponse;
+import jakarta.servlet.http.HttpServletRequest;
+import jakarta.servlet.http.HttpSession;
+import jakarta.ws.rs.HttpMethod;
 import java.io.IOException;
 
 /**