feat: DH-20578: Groovy remote file sourcing

engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java

@@ -0,0 +1,254 @@
+//
+// Copyright (c) 2016-2026 Deephaven Data Labs and Patent Pending
+//
+package io.deephaven.engine.util;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.net.URLConnection;
+import java.net.URLStreamHandler;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * A custom ClassLoader that fetches source files from remote clients via registered RemoteFileSourceProvider instances.
+ * This is designed to support Groovy script imports where the source files are provided by remote clients.
+ *
+ * <p>
+ * When a resource is requested (e.g., for a Groovy import), this class loader:
+ * <ol>
+ * <li>Checks registered providers to see if they can source the resource</li>
+ * <li>Returns a custom URL with protocol "remotefile://" if a provider can handle it</li>
+ * <li>When that URL is opened, fetches the resource bytes from the provider</li>
+ * </ol>
+ */
+public class RemoteFileSourceClassLoader extends ClassLoader {
+    private static final long RESOURCE_TIMEOUT_SECONDS = 5;
+
+    private static volatile RemoteFileSourceClassLoader instance;
+    private final CopyOnWriteArrayList<RemoteFileSourceProvider> providers = new CopyOnWriteArrayList<>();
+
+
+    /**
+     * Constructs a new RemoteFileSourceClassLoader with the specified parent class loader.
+     *
+     * @param parent the parent class loader for delegation
+     */
+    private RemoteFileSourceClassLoader(ClassLoader parent) {
+        super(parent);
+    }
+
+    /**
+     * Initializes the singleton RemoteFileSourceClassLoader instance with the specified parent class loader.
+     *
+     * <p>
+     * This method must be called exactly once before any calls to {@link #getInstance()}. The method is synchronized to
+     * prevent race conditions when multiple threads attempt initialization.
+     *
+     * @param parent the parent class loader for delegation
+     * @return the newly created singleton instance
+     * @throws IllegalStateException if the instance has already been initialized
+     */
+    public static synchronized RemoteFileSourceClassLoader initialize(ClassLoader parent) {
+        if (instance != null) {
+            throw new IllegalStateException("RemoteFileSourceClassLoader is already initialized");
+        }
+
+        instance = new RemoteFileSourceClassLoader(parent);
+        return instance;
+    }
+
+    /**
+     * Returns the singleton instance of the RemoteFileSourceClassLoader.
+     *
+     * <p>
+     * This method requires that {@link #initialize(ClassLoader)} has been called first.
+     *
+     * @return the singleton instance
+     * @throws IllegalStateException if the instance has not yet been initialized via {@link #initialize(ClassLoader)}
+     */
+    public static RemoteFileSourceClassLoader getInstance() {
+        if (instance == null) {
+            throw new IllegalStateException("RemoteFileSourceClassLoader is not yet initialized");
+        }
+        return instance;
+    }
+
+    /**
+     * Registers a new provider that can source remote resources.
+     *
+     * @param provider the provider to register
+     */
+    public void registerProvider(RemoteFileSourceProvider provider) {
+        providers.add(provider);
+    }
+
+    /**
+     * Unregisters a previously registered provider.
+     *
+     * @param provider the provider to unregister
+     */
+    public void unregisterProvider(RemoteFileSourceProvider provider) {
+        providers.remove(provider);
+    }
+
+    /**
+     * Returns whether there are any active providers with non-empty resource paths configured. This indicates that
+     * remote sources are actually configured, not just that the execution context is set.
+     *
+     * @return true if any provider is active and has resource paths configured, false otherwise
+     */
+    public boolean hasConfiguredRemoteSources() {
+        for (RemoteFileSourceProvider candidate : providers) {
+            if (candidate.isActive() && candidate.hasConfiguredResources()) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    /**
+     * Returns whether the current execution context is dirty, indicating that remote sources have changed and the cache
+     * should be cleared. This method is used by GroovyDeephavenSession to determine when to refresh the class cache.
+     *
+     * @return true if there is a dirty execution context, false otherwise
+     */
+    public boolean isDirty() {
+        for (RemoteFileSourceProvider candidate : providers) {
+            if (candidate.isActive() && candidate.isDirty()) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    /**
+     * Gets the resource with the specified name by checking registered providers.
+     *
+     * <p>
+     * This method iterates through all registered providers to see if any can source the requested resource. If a
+     * provider can handle the resource, a custom URL with protocol "remotefile://" is returned. If no provider can
+     * handle the resource, the request is delegated to the parent class loader.
+     *
+     * @param name the resource name
+     * @return a URL for reading the resource, or null if the resource could not be found
+     */
+    @Override
+    public URL getResource(String name) {
+        RemoteFileSourceProvider provider = null;
+        for (RemoteFileSourceProvider candidate : providers) {
+            if (candidate.isActive() && candidate.canSourceResource(name)) {
+                provider = candidate;
+                break;
+            }
+        }
+
+        if (provider != null) {
+            try {
+                return new URL(null, "remotefile://" + name, new RemoteFileURLStreamHandler(provider, name));
+            } catch (MalformedURLException e) {
+                // Fall through to parent if URL creation fails
+            }
+        }
+
+        return super.getResource(name);
+    }
+
+    /**
+     * URLStreamHandler that delegates to a RemoteFileSourceProvider to fetch resource bytes.
+     */
+    private static class RemoteFileURLStreamHandler extends URLStreamHandler {
+        private final RemoteFileSourceProvider provider;
+        private final String resourceName;
+
+        /**
+         * Constructs a new RemoteFileURLStreamHandler for the specified provider and resource.
+         *
+         * @param provider the provider that will source the resource
+         * @param resourceName the name of the resource to fetch
+         */
+        RemoteFileURLStreamHandler(RemoteFileSourceProvider provider, String resourceName) {
+            this.provider = provider;
+            this.resourceName = resourceName;
+        }
+
+        /**
+         * Opens a connection to the resource referenced by this URL.
+         *
+         * @param url the URL to open a connection to
+         * @return a URLConnection to the specified URL
+         */
+        @Override
+        protected URLConnection openConnection(URL url) {
+            return new RemoteFileURLConnection(url, provider, resourceName);
+        }
+    }
+
+    /**
+     * URLConnection that fetches resource bytes from a RemoteFileSourceProvider.
+     */
+    private static class RemoteFileURLConnection extends URLConnection {
+        private final RemoteFileSourceProvider provider;
+        private final String resourceName;
+        private byte[] content;
+
+        /**
+         * Constructs a new RemoteFileURLConnection for the specified URL, provider, and resource.
+         *
+         * @param url the URL to connect to
+         * @param provider the provider that will source the resource
+         * @param resourceName the name of the resource to fetch
+         */
+        RemoteFileURLConnection(URL url, RemoteFileSourceProvider provider, String resourceName) {
+            super(url);
+            this.provider = provider;
+            this.resourceName = resourceName;
+        }
+
+        /**
+         * Opens a connection to the resource by requesting it from the provider.
+         *
+         * <p>
+         * This method fetches the resource bytes from the provider with a timeout of {@value #RESOURCE_TIMEOUT_SECONDS}
+         * seconds. If already connected, this method does nothing.
+         *
+         * @throws IOException if the connection fails or times out
+         */
+        @Override
+        public void connect() throws IOException {
+            if (!connected) {
+                try {
+                    content = provider.requestResource(resourceName)
+                            .orTimeout(RESOURCE_TIMEOUT_SECONDS, TimeUnit.SECONDS)
+                            .get();
+                    connected = true;
+                } catch (Exception e) {
+                    throw new IOException("Failed to fetch remote resource: " + resourceName, e);
+                }
+            }
+        }
+
+        /**
+         * Returns an input stream that reads from this connection's resource.
+         *
+         * <p>
+         * This method calls {@link #connect()} to ensure the connection is established and resource bytes are fetched
+         * from the provider. The method then verifies that content has been successfully downloaded before creating the
+         * input stream.
+         *
+         * @return an input stream that reads from the fetched resource bytes
+         * @throws IOException if the connection or content download fails or if the resource has no content
+         */
+        @Override
+        public InputStream getInputStream() throws IOException {
+            connect();
+            if (content == null || content.length == 0) {
+                throw new IOException("No content for resource: " + resourceName);
+            }
+            return new ByteArrayInputStream(content);
+        }
+    }
+}

engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceProvider.java

@@ -0,0 +1,52 @@
+//
+// Copyright (c) 2016-2026 Deephaven Data Labs and Patent Pending
+//
+package io.deephaven.engine.util;
+
+import java.util.concurrent.CompletableFuture;
+
+/**
+ * Interface for providing remote resources to the ClassLoader. Plugins can implement this interface and register with
+ * RemoteFileSourceClassLoader to provide resources from remote sources.
+ */
+public interface RemoteFileSourceProvider {
+    /**
+     * Check if this provider can source the given resource.
+     *
+     * @param resourceName the name of the resource to check (e.g., "com/example/MyClass.groovy")
+     * @return true if this provider can handle the resource, false otherwise
+     */
+    boolean canSourceResource(String resourceName);
+
+    /**
+     * Check if this provider is currently active and should be used for resource requests.
+     *
+     * @return true if this provider is active, false otherwise
+     */
+    boolean isActive();
+
+    /**
+     * Check if this provider has any resource paths configured. A provider can be active (execution context set) but
+     * have no resource paths configured.
+     *
+     * @return true if this provider has resource paths configured, false otherwise
+     */
+    boolean hasConfiguredResources();
+
+    /**
+     * Check if this provider's execution context is dirty, indicating that remote sources have changed and the cache
+     * should be cleared.
+     *
+     * @return true if this provider is active and dirty, false otherwise
+     */
+    boolean isDirty();
+
+    /**
+     * Request a resource from the remote source.
+     *
+     * @param resourceName the name of the resource to fetch (e.g., "com/example/MyClass.groovy")
+     * @return a CompletableFuture containing the resource bytes, or null if not found
+     */
+    CompletableFuture<byte[]> requestResource(String resourceName);
+}
+

engine/context/src/main/java/io/deephaven/engine/context/ExecutionContext.java

@@ -205,6 +205,21 @@ public ExecutionContext withClassLoader(ClassLoader classLoader) {
                 operationInitializer, classLoader);
     }
 
+    /**
+     * Returns, or creates, an execution context with the given value for {@code queryCompiler} and existing values for
+     * the other members.
+     *
+     * @param queryCompiler the query compiler to use instead
+     * @return the execution context
+     */
+    public ExecutionContext withQueryCompiler(QueryCompiler queryCompiler) {
+        if (queryCompiler == this.queryCompiler) {
+            return this;
+        }
+        return new ExecutionContext(isSystemic, authContext, queryLibrary, queryScope, queryCompiler, updateGraph,
+                operationInitializer);
+    }
+
 
     /**
      * Execute runnable within this execution context.

engine/table/src/main/java/io/deephaven/engine/util/AbstractScriptSession.java

@@ -69,7 +69,8 @@ private static void ensureDirectory(final Path directory) {
     protected final File classCacheDirectory;
     private final ScriptSessionQueryScope queryScope;
 
-    protected final ExecutionContext executionContext;
+    // DH-20578: Not final so it can be updated with fresh QueryCompiler when remote sources are detected
+    protected ExecutionContext executionContext;
 
     private S lastSnapshot;
 
@@ -95,9 +96,26 @@ protected AbstractScriptSession(
         this.classCacheDirectory = classCacheDirectory;
 
         queryScope = new ScriptSessionQueryScope();
+
+        executionContext = createExecutionContext(updateGraph, operationInitializer, parentClassLoader);
+    }
+
+    /**
+     * DH-20578: Creates an ExecutionContext with a QueryCompiler based on the provided ClassLoader. This allows
+     * updating the ExecutionContext when fresh ClassLoaders are needed (e.g., for remote file sources).
+     *
+     * @param updateGraph the update graph for the context
+     * @param operationInitializer the operation initializer for the context
+     * @param parentClassLoader the ClassLoader to use for creating the QueryCompiler
+     * @return a new ExecutionContext with a QueryCompiler based on the provided ClassLoader
+     */
+    protected ExecutionContext createExecutionContext(
+            final UpdateGraph updateGraph,
+            final OperationInitializer operationInitializer,
+            final ClassLoader parentClassLoader) {
         final QueryCompiler compilerContext = QueryCompilerImpl.create(classCacheDirectory, parentClassLoader);
 
-        executionContext = ExecutionContext.newBuilder()
+        return ExecutionContext.newBuilder()
                 .markSystemic()
                 .newQueryLibrary()
                 .setQueryScope(queryScope)