From 9f68c0b09d2d97a705ca4ab5e8a040bf41bf4bcc Mon Sep 17 00:00:00 2001 From: Harry Pierson Date: Fri, 1 May 2026 14:16:52 -0700 Subject: [PATCH 01/29] wip Co-authored-by: Copilot --- .../src/main/java/dev/dbos/transact/DBOS.java | 12 +- .../dev/dbos/transact/config/DBOSConfig.java | 25 ++ .../dev/dbos/transact/database/StepsDAO.java | 4 +- .../transact/execution/ThrowingConsumer.java | 6 + .../transact/execution/ThrowingFunction.java | 6 + .../transact/internal/DBOSIntegration.java | 8 + .../dbos/transact/txstep/JdbcStepFactory.java | 261 ++++++++++++++++++ .../workflow/internal/StepResult.java | 35 ++- 8 files changed, 344 insertions(+), 13 deletions(-) create mode 100644 transact/src/main/java/dev/dbos/transact/execution/ThrowingConsumer.java create mode 100644 transact/src/main/java/dev/dbos/transact/execution/ThrowingFunction.java create mode 100644 transact/src/main/java/dev/dbos/transact/txstep/JdbcStepFactory.java diff --git a/transact/src/main/java/dev/dbos/transact/DBOS.java b/transact/src/main/java/dev/dbos/transact/DBOS.java index a6d1a4ca2..7278a8817 100644 --- a/transact/src/main/java/dev/dbos/transact/DBOS.java +++ b/transact/src/main/java/dev/dbos/transact/DBOS.java @@ -62,9 +62,7 @@ public class DBOS implements AutoCloseable { private final Set lifecycleRegistry = ConcurrentHashMap.newKeySet(); private final DBOSConfig config; private final AtomicReference dbosExecutor = new AtomicReference<>(); - private final DBOSIntegration integration = - new DBOSIntegration( - dbosExecutor::get, this::registerLifecycleListener, this::registerWorkflow); + private final DBOSIntegration integration; private AlertHandler alertHandler; @@ -83,7 +81,13 @@ public DBOS(@NonNull DBOSConfig config) { Objects.requireNonNull(config.dbPassword(), "DBOSConfig.dbPassword must not be null"); } - this.config = config; + this.config = new DBOSConfig(config); + this.integration = + new DBOSIntegration( + this.config, + dbosExecutor::get, + this::registerLifecycleListener, + this::registerWorkflow); } /** diff --git a/transact/src/main/java/dev/dbos/transact/config/DBOSConfig.java b/transact/src/main/java/dev/dbos/transact/config/DBOSConfig.java index ac00eb2eb..79b06be59 100644 --- a/transact/src/main/java/dev/dbos/transact/config/DBOSConfig.java +++ b/transact/src/main/java/dev/dbos/transact/config/DBOSConfig.java @@ -58,6 +58,31 @@ public record DBOSConfig( listenQueues = (listenQueues == null) ? Set.of() : Set.copyOf(listenQueues); } + // Copy constructor + public DBOSConfig(DBOSConfig other) { + this( + other.appName, + other.databaseUrl, + other.dbUser, + other.dbPassword, + other.dataSource, + other.adminServer, + other.adminServerPort, + other.migrate, + other.conductorKey, + other.conductorDomain, + (other.conductorExecutorMetadata == null + ? null + : Map.copyOf(other.conductorExecutorMetadata)), + other.appVersion, + other.executorId, + other.databaseSchema, + other.enablePatching, + (other.listenQueues == null ? null : Set.copyOf(other.listenQueues)), + other.serializer, + other.schedulerPollingInterval); + } + public static @NonNull DBOSConfig defaults(@NonNull String appName) { return new DBOSConfig( appName, null, null, null, null, false, // adminServer diff --git a/transact/src/main/java/dev/dbos/transact/database/StepsDAO.java b/transact/src/main/java/dev/dbos/transact/database/StepsDAO.java index 66af3d263..f5cf67eb9 100644 --- a/transact/src/main/java/dev/dbos/transact/database/StepsDAO.java +++ b/transact/src/main/java/dev/dbos/transact/database/StepsDAO.java @@ -69,7 +69,7 @@ static void recordStepResultTxn( try (PreparedStatement pstmt = connection.prepareStatement(sql)) { pstmt.setString(1, result.workflowId()); pstmt.setInt(2, result.stepId()); - pstmt.setString(3, result.functionName()); + pstmt.setString(3, result.stepName()); if (result.output() != null) { pstmt.setString(4, result.output()); @@ -99,7 +99,7 @@ static void recordStepResultTxn( logger.warn( String.format( "Step output for %s:%d-%s was already recorded", - result.workflowId(), result.stepId(), result.functionName())); + result.workflowId(), result.stepId(), result.stepName())); throw new DBOSWorkflowExecutionConflictException(result.workflowId()); } } diff --git a/transact/src/main/java/dev/dbos/transact/execution/ThrowingConsumer.java b/transact/src/main/java/dev/dbos/transact/execution/ThrowingConsumer.java new file mode 100644 index 000000000..29b4b0692 --- /dev/null +++ b/transact/src/main/java/dev/dbos/transact/execution/ThrowingConsumer.java @@ -0,0 +1,6 @@ +package dev.dbos.transact.execution; + +@FunctionalInterface +public interface ThrowingConsumer { + void execute(P p) throws E; +} diff --git a/transact/src/main/java/dev/dbos/transact/execution/ThrowingFunction.java b/transact/src/main/java/dev/dbos/transact/execution/ThrowingFunction.java new file mode 100644 index 000000000..2c2c1cee7 --- /dev/null +++ b/transact/src/main/java/dev/dbos/transact/execution/ThrowingFunction.java @@ -0,0 +1,6 @@ +package dev.dbos.transact.execution; + +@FunctionalInterface +public interface ThrowingFunction { + T execute(P p) throws E; +} diff --git a/transact/src/main/java/dev/dbos/transact/internal/DBOSIntegration.java b/transact/src/main/java/dev/dbos/transact/internal/DBOSIntegration.java index 32e708cce..5338bedfc 100644 --- a/transact/src/main/java/dev/dbos/transact/internal/DBOSIntegration.java +++ b/transact/src/main/java/dev/dbos/transact/internal/DBOSIntegration.java @@ -1,6 +1,7 @@ package dev.dbos.transact.internal; import dev.dbos.transact.StartWorkflowOptions; +import dev.dbos.transact.config.DBOSConfig; import dev.dbos.transact.database.ExternalState; import dev.dbos.transact.execution.DBOSExecutor; import dev.dbos.transact.execution.DBOSLifecycleListener; @@ -35,14 +36,17 @@ public interface RegisteredWorkflowConsumer { void register(Workflow wfTag, Object target, Method method, String instanceName); } + private final DBOSConfig config; private final Supplier executorSupplier; private final Consumer listenerConsumer; private final RegisteredWorkflowConsumer workflowConsumer; public DBOSIntegration( + @NonNull DBOSConfig config, @NonNull Supplier executorSupplier, @NonNull Consumer lifecycleConsumer, @NonNull RegisteredWorkflowConsumer workflowConsumer) { + this.config = Objects.requireNonNull(config); this.executorSupplier = Objects.requireNonNull(executorSupplier); this.listenerConsumer = Objects.requireNonNull(lifecycleConsumer); this.workflowConsumer = Objects.requireNonNull(workflowConsumer); @@ -57,6 +61,10 @@ private DBOSExecutor executor(String caller) { return exec; } + public DBOSConfig config() { + return this.config; + } + /** * Register a lifecycle listener that receives callbacks when DBOS is launched or shut down * diff --git a/transact/src/main/java/dev/dbos/transact/txstep/JdbcStepFactory.java b/transact/src/main/java/dev/dbos/transact/txstep/JdbcStepFactory.java new file mode 100644 index 000000000..4d0c09367 --- /dev/null +++ b/transact/src/main/java/dev/dbos/transact/txstep/JdbcStepFactory.java @@ -0,0 +1,261 @@ +package dev.dbos.transact.txstep; + +import dev.dbos.transact.DBOS; +import dev.dbos.transact.database.SystemDatabase; +import dev.dbos.transact.execution.ThrowingConsumer; +import dev.dbos.transact.execution.ThrowingFunction; +import dev.dbos.transact.json.DBOSSerializer; +import dev.dbos.transact.json.SerializationUtil; +import dev.dbos.transact.workflow.internal.StepResult; + +import java.sql.Connection; +import java.sql.SQLException; +import java.util.Objects; + +import javax.sql.DataSource; + +import org.jspecify.annotations.Nullable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class JdbcStepFactory { + + private static final Logger logger = LoggerFactory.getLogger(JdbcStepFactory.class); + + private final DBOS dbos; + private final DataSource dataSource; + private final String schema; + private final DBOSSerializer serializer; + + public JdbcStepFactory(DBOS dbos, DataSource dataSource) throws SQLException { + this(dbos, dataSource, null, null); + } + + public JdbcStepFactory(DBOS dbos, DataSource dataSource, String schema, DBOSSerializer serializer) + throws SQLException { + this.dbos = Objects.requireNonNull(dbos); + this.dataSource = Objects.requireNonNull(dataSource); + var config = dbos.integration().config(); + this.schema = SystemDatabase.sanitizeSchema(schema == null ? config.databaseSchema() : schema); + this.serializer = serializer == null ? config.serializer() : serializer; + + createTxOutputTable(dataSource, this.schema); + } + + public static void createTxOutputTable(DataSource dataSource, String schema) + throws SQLException { + try (var conn = dataSource.getConnection()) { + ensureSchema(conn, schema); + ensureTxOutputTable(conn, schema); + } + } + + public static void ensureSchema(Connection conn, String schema) throws SQLException { + Objects.requireNonNull(schema, "schema must not be null"); + var sql = "SELECT schema_name FROM information_schema.schemata WHERE schema_name = ?"; + try (var stmt = conn.prepareStatement(sql)) { + stmt.setString(1, schema); + try (var rs = stmt.executeQuery()) { + if (rs.next()) { + return; + } + } + } + + try (var stmt = conn.createStatement()) { + stmt.execute("CREATE SCHEMA IF NOT EXISTS \"%s\"".formatted(schema)); + } + } + + public static void ensureTxOutputTable(Connection conn, String schema) throws SQLException { + Objects.requireNonNull(schema, "schema must not be null"); + var sql = "SELECT 1 FROM information_schema.tables WHERE table_schema = ? AND table_name = ?"; + try (var stmt = conn.prepareStatement(sql)) { + stmt.setString(1, schema); + stmt.setString(2, "tx_step_outputs"); + try (var rs = stmt.executeQuery()) { + if (rs.next()) { + return; + } + } + } + + try (var stmt = conn.createStatement()) { + var ddlSql = + """ + CREATE TABLE IF NOT EXISTS "%1$s".tx_step_outputs ( + workflow_id TEXT NOT NULL, + step_id INT NOT NULL, + output TEXT, + error TEXT, + serialization TEXT, + created_at BIGINT NOT NULL DEFAULT (EXTRACT(EPOCH FROM now())*1000)::bigint, + PRIMARY KEY (workflow_id, function_num) + )""" + .formatted(schema); + stmt.execute(ddlSql); + } + } + + private static class DBOSSqlException extends RuntimeException { + public DBOSSqlException(SQLException wrappedException) { + super(wrappedException.getMessage(), wrappedException); + } + } + + // helper methods that wrap SQLExceptions in WrappedSqlException to distinguish from app + // exceptions + private Connection getConnection() { + try { + var conn = dataSource.getConnection(); + conn.setAutoCommit(false); + return conn; + } catch (SQLException e) { + throw new DBOSSqlException(e); + } + } + + private static void commit(Connection conn) { + try { + conn.commit(); + } catch (SQLException e) { + throw new DBOSSqlException(e); + } + } + + private static void rollback(Connection conn) { + try { + conn.rollback(); + } catch (SQLException e) { + throw new DBOSSqlException(e); + } + } + + private static void close(Connection conn) { + try { + conn.close(); + } catch (SQLException e) { + throw new DBOSSqlException(e); + } + } + + private @Nullable StepResult checkExecution(String workflowId, int stepId, String stepName) { + var sql = + """ + SELECT output, error, serialization + FROM "%s".tx_step_outputs + WHERE workflow_id = ? AND step_id = ? + """ + .formatted(this.schema); + try (var conn = dataSource.getConnection(); + var stmt = conn.prepareStatement(sql)) { + stmt.setString(1, workflowId); + stmt.setInt(2, stepId); + try (var rs = stmt.executeQuery()) { + if (rs.next()) { + var output = rs.getString("output"); + var error = rs.getString("error"); + var serialization = rs.getString("serialization"); + var result = + new StepResult(workflowId, stepId, stepName, output, error, null, serialization); + return result; + } else { + return null; + } + } + } catch (SQLException e) { + throw new DBOSSqlException(e); + } + } + + private void recordResult( + Connection conn, + String workflowId, + int stepId, + String output, + String error, + String serialization) { + if (output != null && error != null) { + throw new IllegalArgumentException("attempted to record non null output and error result"); + } + + String sql = + """ + INSERT INTO "%s".tx_step_outputs + (workflow_id, step_id, output, error, serialization) + VALUES (?, ?, ?, ?) + ON CONFLICT DO NOTHING + """ + .formatted(schema); + + try (var stmt = conn.prepareStatement(sql)) { + stmt.setString(1, workflowId); + stmt.setInt(2, stepId); + stmt.setString(3, output); + stmt.setString(4, error); + stmt.setString(5, serialization); + stmt.executeUpdate(); + } catch (SQLException e) { + throw new DBOSSqlException(e); + } + } + + private void recordOutput(Connection conn, String workflowId, int stepId, R retVal) { + var value = SerializationUtil.serializeValue(retVal, null, serializer); + recordResult(conn, workflowId, stepId, value.serializedValue(), null, value.serialization()); + } + + private void recordError(String workflowId, int stepId, E exception) { + var value = SerializationUtil.serializeError(exception, null, serializer); + var conn = getConnection(); + try { + recordResult(conn, workflowId, stepId, null, value.serializedValue(), value.serialization()); + } finally { + close(conn); + } + } + + private R txStepInternal( + ThrowingFunction func, String stepName) throws E { + var workflowId = Objects.requireNonNull(DBOS.workflowId()); + int stepId = Objects.requireNonNull(DBOS.stepId()); + + var prevResult = this.checkExecution(workflowId, stepId, stepName); + if (prevResult != null) { + return prevResult.toResult(serializer); + } + + var conn = getConnection(); + try { + var retVal = func.execute(conn); + recordOutput(conn, workflowId, stepId, retVal); + commit(conn); + return retVal; + } catch (Exception e) { + rollback(conn); + recordError(workflowId, stepId, e); + throw e; + } finally { + close(conn); + } + } + + public R txStep(ThrowingFunction func, String stepName) + throws E { + return dbos.runStep( + () -> { + return this.txStepInternal(func, stepName); + }, + stepName); + } + + public void txStep(ThrowingConsumer func, String stepName) + throws E { + txStep( + c -> { + func.execute(c); + return null; + }, + stepName); + } +} diff --git a/transact/src/main/java/dev/dbos/transact/workflow/internal/StepResult.java b/transact/src/main/java/dev/dbos/transact/workflow/internal/StepResult.java index f3e6f3e40..c706e9121 100644 --- a/transact/src/main/java/dev/dbos/transact/workflow/internal/StepResult.java +++ b/transact/src/main/java/dev/dbos/transact/workflow/internal/StepResult.java @@ -1,9 +1,12 @@ package dev.dbos.transact.workflow.internal; +import dev.dbos.transact.json.DBOSSerializer; +import dev.dbos.transact.json.SerializationUtil; + public record StepResult( String workflowId, int stepId, - String functionName, + String stepName, String output, String error, String childWorkflowId, @@ -14,20 +17,38 @@ public StepResult(String workflowId, int stepId, String functionName) { } public StepResult withOutput(String v) { - return new StepResult( - workflowId, stepId, functionName, v, error, childWorkflowId, serialization); + return new StepResult(workflowId, stepId, stepName, v, error, childWorkflowId, serialization); } public StepResult withError(String v) { - return new StepResult( - workflowId, stepId, functionName, output, v, childWorkflowId, serialization); + return new StepResult(workflowId, stepId, stepName, output, v, childWorkflowId, serialization); } public StepResult withChildWorkflowId(String v) { - return new StepResult(workflowId, stepId, functionName, output, error, v, serialization); + return new StepResult(workflowId, stepId, stepName, output, error, v, serialization); } public StepResult withSerialization(String v) { - return new StepResult(workflowId, stepId, functionName, output, error, childWorkflowId, v); + return new StepResult(workflowId, stepId, stepName, output, error, childWorkflowId, v); + } + + @SuppressWarnings("unchecked") + public R toResult(DBOSSerializer serializer) throws E { + if (error != null) { + var t = SerializationUtil.deserializeError(error, serialization, serializer); + if (t instanceof Exception) { + throw (E) t; + } else { + throw new RuntimeException(t.getMessage(), t); + } + } + + if (output != null) { + return (R) SerializationUtil.deserializeValue(output, serialization, serializer); + } + + throw new IllegalStateException( + "Recorded output and error are both null for workflow %s step %d (%s)" + .formatted(workflowId, stepId, stepName)); } } From ccb71791d2302a419f35f663272a26e0b9fbef14 Mon Sep 17 00:00:00 2001 From: Harry Pierson Date: Fri, 1 May 2026 14:17:22 -0700 Subject: [PATCH 02/29] spotless --- .../main/java/dev/dbos/transact/txstep/JdbcStepFactory.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/transact/src/main/java/dev/dbos/transact/txstep/JdbcStepFactory.java b/transact/src/main/java/dev/dbos/transact/txstep/JdbcStepFactory.java index 4d0c09367..672789b9d 100644 --- a/transact/src/main/java/dev/dbos/transact/txstep/JdbcStepFactory.java +++ b/transact/src/main/java/dev/dbos/transact/txstep/JdbcStepFactory.java @@ -42,8 +42,7 @@ public JdbcStepFactory(DBOS dbos, DataSource dataSource, String schema, DBOSSeri createTxOutputTable(dataSource, this.schema); } - public static void createTxOutputTable(DataSource dataSource, String schema) - throws SQLException { + public static void createTxOutputTable(DataSource dataSource, String schema) throws SQLException { try (var conn = dataSource.getConnection()) { ensureSchema(conn, schema); ensureTxOutputTable(conn, schema); From 20114b1496e25c6b922bca7b09ed25b59a320e7b Mon Sep 17 00:00:00 2001 From: Harry Pierson Date: Fri, 1 May 2026 17:18:25 -0700 Subject: [PATCH 03/29] WIP Co-authored-by: Copilot --- .../dbos/transact/txstep/JdbcStepFactory.java | 57 ++++-- .../txstep/JdbcStepFactoryInitTest.java | 162 ++++++++++++++++++ .../transact/txstep/JdbcStepFactoryTest.java | 103 +++++++++++ .../java/dev/dbos/transact/utils/DBUtils.java | 62 +++++++ .../dbos/transact/utils/TxStepOutputRow.java | 23 +++ 5 files changed, 388 insertions(+), 19 deletions(-) create mode 100644 transact/src/test/java/dev/dbos/transact/txstep/JdbcStepFactoryInitTest.java create mode 100644 transact/src/test/java/dev/dbos/transact/txstep/JdbcStepFactoryTest.java create mode 100644 transact/src/test/java/dev/dbos/transact/utils/TxStepOutputRow.java diff --git a/transact/src/main/java/dev/dbos/transact/txstep/JdbcStepFactory.java b/transact/src/main/java/dev/dbos/transact/txstep/JdbcStepFactory.java index 672789b9d..e52014693 100644 --- a/transact/src/main/java/dev/dbos/transact/txstep/JdbcStepFactory.java +++ b/transact/src/main/java/dev/dbos/transact/txstep/JdbcStepFactory.java @@ -27,12 +27,20 @@ public class JdbcStepFactory { private final String schema; private final DBOSSerializer serializer; - public JdbcStepFactory(DBOS dbos, DataSource dataSource) throws SQLException { + public JdbcStepFactory(DBOS dbos, DataSource dataSource) { this(dbos, dataSource, null, null); } - public JdbcStepFactory(DBOS dbos, DataSource dataSource, String schema, DBOSSerializer serializer) - throws SQLException { + public JdbcStepFactory(DBOS dbos, DataSource dataSource, String schema) { + this(dbos, dataSource, schema, null); + } + + public JdbcStepFactory(DBOS dbos, DataSource dataSource, DBOSSerializer serializer) { + this(dbos, dataSource, null, serializer); + } + + public JdbcStepFactory( + DBOS dbos, DataSource dataSource, String schema, DBOSSerializer serializer) { this.dbos = Objects.requireNonNull(dbos); this.dataSource = Objects.requireNonNull(dataSource); var config = dbos.integration().config(); @@ -42,43 +50,54 @@ public JdbcStepFactory(DBOS dbos, DataSource dataSource, String schema, DBOSSeri createTxOutputTable(dataSource, this.schema); } - public static void createTxOutputTable(DataSource dataSource, String schema) throws SQLException { + public static void createTxOutputTable(DataSource dataSource, String schema) { try (var conn = dataSource.getConnection()) { ensureSchema(conn, schema); ensureTxOutputTable(conn, schema); + } catch (SQLException e) { + throw new DBOSSqlException(e); } } - public static void ensureSchema(Connection conn, String schema) throws SQLException { - Objects.requireNonNull(schema, "schema must not be null"); + public static boolean schemaExists(Connection conn, String schema) throws SQLException { var sql = "SELECT schema_name FROM information_schema.schemata WHERE schema_name = ?"; try (var stmt = conn.prepareStatement(sql)) { - stmt.setString(1, schema); + stmt.setString(1, Objects.requireNonNull(schema, "schema must not be null")); try (var rs = stmt.executeQuery()) { if (rs.next()) { - return; + return true; + } else { + return false; } } } + } - try (var stmt = conn.createStatement()) { - stmt.execute("CREATE SCHEMA IF NOT EXISTS \"%s\"".formatted(schema)); + public static void ensureSchema(Connection conn, String schema) throws SQLException { + Objects.requireNonNull(schema, "schema must not be null"); + if (!schemaExists(conn, schema)) { + try (var stmt = conn.createStatement()) { + stmt.execute("CREATE SCHEMA IF NOT EXISTS \"%s\"".formatted(schema)); + } } } - public static void ensureTxOutputTable(Connection conn, String schema) throws SQLException { - Objects.requireNonNull(schema, "schema must not be null"); + public static boolean tableExists(Connection conn, String schema) throws SQLException { var sql = "SELECT 1 FROM information_schema.tables WHERE table_schema = ? AND table_name = ?"; try (var stmt = conn.prepareStatement(sql)) { - stmt.setString(1, schema); - stmt.setString(2, "tx_step_outputs"); + stmt.setString(1, Objects.requireNonNull(schema, "schema must not be null")); + stmt.setString(2, Objects.requireNonNull("tx_step_outputs", "tableName must not be null")); try (var rs = stmt.executeQuery()) { - if (rs.next()) { - return; - } + return rs.next(); } } + } + public static void ensureTxOutputTable(Connection conn, String schema) throws SQLException { + Objects.requireNonNull(schema, "schema must not be null"); + if (tableExists(conn, schema)) { + return; + } try (var stmt = conn.createStatement()) { var ddlSql = """ @@ -89,7 +108,7 @@ public static void ensureTxOutputTable(Connection conn, String schema) throws SQ error TEXT, serialization TEXT, created_at BIGINT NOT NULL DEFAULT (EXTRACT(EPOCH FROM now())*1000)::bigint, - PRIMARY KEY (workflow_id, function_num) + PRIMARY KEY (workflow_id, step_id) )""" .formatted(schema); stmt.execute(ddlSql); @@ -182,7 +201,7 @@ private void recordResult( """ INSERT INTO "%s".tx_step_outputs (workflow_id, step_id, output, error, serialization) - VALUES (?, ?, ?, ?) + VALUES (?, ?, ?, ?, ?) ON CONFLICT DO NOTHING """ .formatted(schema); diff --git a/transact/src/test/java/dev/dbos/transact/txstep/JdbcStepFactoryInitTest.java b/transact/src/test/java/dev/dbos/transact/txstep/JdbcStepFactoryInitTest.java new file mode 100644 index 000000000..15794dae9 --- /dev/null +++ b/transact/src/test/java/dev/dbos/transact/txstep/JdbcStepFactoryInitTest.java @@ -0,0 +1,162 @@ +package dev.dbos.transact.txstep; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import dev.dbos.transact.Constants; +import dev.dbos.transact.DBOS; +import dev.dbos.transact.database.SystemDatabase; +import dev.dbos.transact.utils.DBUtils; +import dev.dbos.transact.utils.PgContainer; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.util.Objects; + +import org.junit.jupiter.api.AutoClose; +import org.junit.jupiter.api.Test; + +public class JdbcStepFactoryInitTest { + @AutoClose final PgContainer pgContainer = new PgContainer(); + + static boolean validateSchema(Connection conn, String schema) throws SQLException { + Objects.requireNonNull(schema); + try (var rs = conn.getMetaData().getSchemas()) { + while (rs.next()) { + if (schema.equalsIgnoreCase(rs.getString("TABLE_SCHEM"))) { + return true; + } + } + } + return false; + } + + static boolean validateTxStepOutputsTable(Connection conn, String schema) throws SQLException { + try (var rs = + conn.getMetaData() + .getTables(null, Objects.requireNonNull(schema), "tx_step_outputs", null)) { + if (rs.next()) { + if (schema.equals(rs.getString("TABLE_SCHEM")) + && "tx_step_outputs".equals(rs.getString("TABLE_NAME")) + && "TABLE".equalsIgnoreCase(rs.getString("TABLE_TYPE"))) { + return true; + } + } + } + return false; + } + + @Test + public void sameDbDefaultSchema() throws Exception { + var config = pgContainer.dbosConfig(); + try (var dbos = new DBOS(config); + var dataSource = pgContainer.dataSource()) { + var schema = Constants.DB_SCHEMA; + + // ensure step factory schema/table do not exist + try (var conn = dataSource.getConnection()) { + assertFalse(validateSchema(conn, schema)); + assertFalse(validateTxStepOutputsTable(conn, schema)); + } + + // create step factory to initialize the app db tables + new JdbcStepFactory(dbos, dataSource); + + // ensure step factory schema/table do exist + try (var conn = dataSource.getConnection()) { + assertTrue(validateSchema(conn, schema)); + assertTrue(validateTxStepOutputsTable(conn, schema)); + } + dbos.launch(); + } + } + + @Test + public void sameDbCustomDbosSchema() throws Exception { + var schema = "custom"; + var config = pgContainer.dbosConfig().withDatabaseSchema(schema); + try (var dbos = new DBOS(config); + var dataSource = pgContainer.dataSource()) { + new JdbcStepFactory(dbos, dataSource); + try (var conn = dataSource.getConnection()) { + assertTrue(validateSchema(conn, schema)); + assertTrue(validateTxStepOutputsTable(conn, schema)); + } + dbos.launch(); + } + } + + @Test + public void sameDbCustomFactorySchema() throws Exception { + var schema = "custom"; + var config = pgContainer.dbosConfig(); + try (var dbos = new DBOS(config); + var dataSource = pgContainer.dataSource()) { + new JdbcStepFactory(dbos, dataSource, schema); + try (var conn = dataSource.getConnection()) { + assertFalse(validateSchema(conn, Constants.DB_SCHEMA)); + assertTrue(validateSchema(conn, schema)); + assertTrue(validateTxStepOutputsTable(conn, schema)); + } + dbos.launch(); + try (var conn = dataSource.getConnection()) { + assertTrue(validateSchema(conn, Constants.DB_SCHEMA)); + } + } + } + + @Test + public void sameDbCustomDbosAndFactorySchema() throws Exception { + var dbosSchema = "custom_a"; + var factorySchema = "custom_b"; + var config = pgContainer.dbosConfig().withDatabaseSchema(dbosSchema); + try (var dbos = new DBOS(config); + var dataSource = pgContainer.dataSource()) { + new JdbcStepFactory(dbos, dataSource, factorySchema); + try (var conn = dataSource.getConnection()) { + assertFalse(validateSchema(conn, dbosSchema)); + assertTrue(validateSchema(conn, factorySchema)); + assertTrue(validateTxStepOutputsTable(conn, factorySchema)); + } + dbos.launch(); + try (var conn = dataSource.getConnection()) { + assertTrue(validateSchema(conn, dbosSchema)); + } + } + } + + @Test + public void separateDBs() throws Exception { + // create a 2nd database in the container's PG instance + var appDbName = "factory_test_db"; + try (var conn = + DriverManager.getConnection( + pgContainer.jdbcUrl(), pgContainer.username(), pgContainer.password()); + var stmt = conn.createStatement()) { + stmt.execute("CREATE DATABASE " + appDbName); + } + var appDbJdbcUrl = pgContainer.jdbcUrl().replaceFirst("/[^/]+$", "/" + appDbName); + + var config = pgContainer.dbosConfig(); + try (var dbos = new DBOS(config); + var dataSource = + SystemDatabase.createDataSource( + appDbJdbcUrl, pgContainer.username(), pgContainer.password())) { + new JdbcStepFactory(dbos, dataSource); + dbos.launch(); + + var appDbTables = DBUtils.getTables(dataSource, "dbos"); + assertEquals(1, appDbTables.size()); + assertTrue(appDbTables.contains("tx_step_outputs")); + + var sysDbTables = DBUtils.getTables(pgContainer, "dbos"); + assertTrue(sysDbTables.size() >= 10); + assertFalse(sysDbTables.contains("tx_step_outputs")); + assertTrue(sysDbTables.contains("dbos_migrations")); + assertTrue(sysDbTables.contains("workflow_status")); + assertTrue(sysDbTables.contains("operation_outputs")); + } + } +} diff --git a/transact/src/test/java/dev/dbos/transact/txstep/JdbcStepFactoryTest.java b/transact/src/test/java/dev/dbos/transact/txstep/JdbcStepFactoryTest.java new file mode 100644 index 000000000..18af4e5b6 --- /dev/null +++ b/transact/src/test/java/dev/dbos/transact/txstep/JdbcStepFactoryTest.java @@ -0,0 +1,103 @@ +package dev.dbos.transact.txstep; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import dev.dbos.transact.DBOS; +import dev.dbos.transact.config.DBOSConfig; +import dev.dbos.transact.utils.DBUtils; +import dev.dbos.transact.utils.PgContainer; +import dev.dbos.transact.workflow.Workflow; + +import java.sql.Connection; +import java.sql.SQLException; +import java.util.Objects; + +import com.zaxxer.hikari.HikariDataSource; +import org.junit.jupiter.api.AutoClose; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +interface FactoryTestService { + int greet(String user) throws SQLException; +} + +class FactoryTestServiceImpl implements FactoryTestService { + + private final DBOS dbos; + private final JdbcStepFactory stepFactory; + + public FactoryTestServiceImpl(DBOS dbos, JdbcStepFactory stepFactory) { + this.dbos = dbos; + this.stepFactory = stepFactory; + } + + int insertGreeting(Connection conn, String user) throws SQLException { + var sql = + """ + INSERT INTO greetings(name, greet_count) + VALUES (?, 1) + ON CONFLICT(name) + DO UPDATE SET greet_count = greetings.greet_count + 1 + RETURNING greet_count + """; + + try (var stmt = conn.prepareStatement(sql)) { + stmt.setString(1, Objects.requireNonNull(user)); + try (var rs = stmt.executeQuery()) { + if (rs.next()) { + return rs.getInt("greet_count"); + } else { + return 0; + } + } + } + } + + @Override + @Workflow + public int greet(String user) throws SQLException { + return stepFactory.txStep((Connection c) -> insertGreeting(c, user), "insertGreeting"); + } +} + +public class JdbcStepFactoryTest { + @AutoClose final PgContainer pgContainer = new PgContainer(); + + DBOSConfig dbosConfig; + @AutoClose DBOS dbos; + @AutoClose HikariDataSource dataSource; + JdbcStepFactory stepFactory; + + @BeforeEach + void beforeEach() { + dbosConfig = pgContainer.dbosConfig(); + dataSource = pgContainer.dataSource(); + dbos = new DBOS(dbosConfig); + stepFactory = new JdbcStepFactory(dbos, dataSource); + } + + @Test + public void foo() throws Exception { + { + var sql = + "CREATE TABLE greetings(name text NOT NULL, greet_count integer DEFAULT 0, PRIMARY KEY(name))"; + try (var conn = dataSource.getConnection(); + var stmt = conn.createStatement()) { + stmt.execute(sql); + } + } + var proxy = + dbos.registerProxy(FactoryTestService.class, new FactoryTestServiceImpl(dbos, stepFactory)); + + dbos.launch(); + assertEquals(1, proxy.greet("Luna")); + assertEquals(2, proxy.greet("Luna")); + assertEquals(3, proxy.greet("Luna")); + assertEquals(4, proxy.greet("Luna")); + assertEquals(5, proxy.greet("Luna")); + + var txStepRows = DBUtils.getTxStepRows(dataSource); + assertEquals(5, txStepRows.size()); + + } +} diff --git a/transact/src/test/java/dev/dbos/transact/utils/DBUtils.java b/transact/src/test/java/dev/dbos/transact/utils/DBUtils.java index 450b64554..13e801cd2 100644 --- a/transact/src/test/java/dev/dbos/transact/utils/DBUtils.java +++ b/transact/src/test/java/dev/dbos/transact/utils/DBUtils.java @@ -15,7 +15,10 @@ import java.sql.Statement; import java.time.Instant; import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; import java.util.List; +import java.util.Map; import javax.sql.DataSource; @@ -444,4 +447,63 @@ public static List getStreamEntries(DataSource ds, String workflowId, } } } + + public static List getTxStepRows(DataSource ds) throws SQLException { + return getTxStepRows(ds, null); + } + + public static List getTxStepRows(DataSource ds, String schema) + throws SQLException { + schema = SystemDatabase.sanitizeSchema(schema); + var sql = "SELECT * FROM \"%s\".tx_step_outputs ORDER BY created_at".formatted(schema); + try (var conn = ds.getConnection(); + var stmt = conn.createStatement(); + var rs = stmt.executeQuery(sql)) { + List rows = new ArrayList<>(); + while (rs.next()) { + rows.add(new TxStepOutputRow(rs)); + } + return rows; + } + } + + public static List> dumpResultSet(ResultSet rs) throws SQLException { + List> results = new ArrayList<>(); + var metaData = rs.getMetaData(); + var columnCount = metaData.getColumnCount(); + while (rs.next()) { + Map map = new HashMap<>(); + for (var i = 1; i <= columnCount; i++) { + map.put(metaData.getColumnLabel(i), rs.getObject(i)); + } + results.add(map); + } + return results; + } + + public static Collection getTables(PgContainer pg, String schema) throws SQLException { + try (var ds = pg.dataSource()) { + return getTables(ds, schema); + } + } + + public static Collection getTables(DataSource ds, String schema) throws SQLException { + try (var conn = ds.getConnection()) { + return getTables(conn, schema); + } + } + + public static Collection getTables(Connection conn, String schema) throws SQLException { + List tables = new ArrayList<>(); + try (var rs = conn.getMetaData().getTables(null, schema, null, null)) { + while (rs.next()) { + var name = rs.getString("TABLE_NAME"); + var type = rs.getString("TABLE_TYPE"); + if ("TABLE".equals(type)) { + tables.add(name); + } + } + } + return tables; + } } diff --git a/transact/src/test/java/dev/dbos/transact/utils/TxStepOutputRow.java b/transact/src/test/java/dev/dbos/transact/utils/TxStepOutputRow.java new file mode 100644 index 000000000..7472fe314 --- /dev/null +++ b/transact/src/test/java/dev/dbos/transact/utils/TxStepOutputRow.java @@ -0,0 +1,23 @@ +package dev.dbos.transact.utils; + +import java.sql.ResultSet; +import java.sql.SQLException; + +public record TxStepOutputRow( + String workflowId, + int stepId, + String output, + String error, + String serialization, + Long createdAt) { + + public TxStepOutputRow(ResultSet rs) throws SQLException { + this( + rs.getString("workflow_id"), + rs.getInt("step_id"), + rs.getString("output"), + rs.getString("error"), + rs.getString("serialization"), + rs.getObject("created_at", Long.class)); + } +} From 8c2b17f493b396dff0bc555d6a7b5800b0175030 Mon Sep 17 00:00:00 2001 From: Harry Pierson Date: Fri, 1 May 2026 17:42:24 -0700 Subject: [PATCH 04/29] spotless --- .../test/java/dev/dbos/transact/txstep/JdbcStepFactoryTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/transact/src/test/java/dev/dbos/transact/txstep/JdbcStepFactoryTest.java b/transact/src/test/java/dev/dbos/transact/txstep/JdbcStepFactoryTest.java index 18af4e5b6..63d7ebff6 100644 --- a/transact/src/test/java/dev/dbos/transact/txstep/JdbcStepFactoryTest.java +++ b/transact/src/test/java/dev/dbos/transact/txstep/JdbcStepFactoryTest.java @@ -98,6 +98,5 @@ public void foo() throws Exception { var txStepRows = DBUtils.getTxStepRows(dataSource); assertEquals(5, txStepRows.size()); - } } From 782038ca944015026b67a6a00eb7feb771010e3f Mon Sep 17 00:00:00 2001 From: Harry Pierson Date: Mon, 4 May 2026 12:35:56 -0700 Subject: [PATCH 05/29] moar tests --- .../dbos/transact/txstep/JdbcStepFactory.java | 6 + .../transact/txstep/JdbcStepFactoryTest.java | 399 ++++++++++++++++-- .../java/dev/dbos/transact/utils/DBUtils.java | 35 +- 3 files changed, 399 insertions(+), 41 deletions(-) diff --git a/transact/src/main/java/dev/dbos/transact/txstep/JdbcStepFactory.java b/transact/src/main/java/dev/dbos/transact/txstep/JdbcStepFactory.java index e52014693..14cad37bc 100644 --- a/transact/src/main/java/dev/dbos/transact/txstep/JdbcStepFactory.java +++ b/transact/src/main/java/dev/dbos/transact/txstep/JdbcStepFactory.java @@ -98,6 +98,7 @@ public static void ensureTxOutputTable(Connection conn, String schema) throws SQ if (tableExists(conn, schema)) { return; } + logger.debug("Creating tx_step_outputs table in schema={}", schema); try (var stmt = conn.createStatement()) { var ddlSql = """ @@ -238,20 +239,25 @@ private R txStepInternal( var workflowId = Objects.requireNonNull(DBOS.workflowId()); int stepId = Objects.requireNonNull(DBOS.stepId()); + logger.debug("txStep starting: workflowId={} stepId={} stepName={}", workflowId, stepId, stepName); var prevResult = this.checkExecution(workflowId, stepId, stepName); if (prevResult != null) { + logger.debug("txStep cache hit: workflowId={} stepId={} stepName={}", workflowId, stepId, stepName); return prevResult.toResult(serializer); } + logger.debug("txStep executing: workflowId={} stepId={} stepName={}", workflowId, stepId, stepName); var conn = getConnection(); try { var retVal = func.execute(conn); recordOutput(conn, workflowId, stepId, retVal); commit(conn); + logger.debug("txStep succeeded: workflowId={} stepId={} stepName={}", workflowId, stepId, stepName); return retVal; } catch (Exception e) { rollback(conn); recordError(workflowId, stepId, e); + logger.debug("txStep failed: workflowId={} stepId={} stepName={} error={}", workflowId, stepId, stepName, e.getMessage()); throw e; } finally { close(conn); diff --git a/transact/src/test/java/dev/dbos/transact/txstep/JdbcStepFactoryTest.java b/transact/src/test/java/dev/dbos/transact/txstep/JdbcStepFactoryTest.java index 63d7ebff6..61db3688e 100644 --- a/transact/src/test/java/dev/dbos/transact/txstep/JdbcStepFactoryTest.java +++ b/transact/src/test/java/dev/dbos/transact/txstep/JdbcStepFactoryTest.java @@ -1,12 +1,19 @@ package dev.dbos.transact.txstep; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import dev.dbos.transact.DBOS; import dev.dbos.transact.config.DBOSConfig; +import dev.dbos.transact.context.WorkflowOptions; +import dev.dbos.transact.json.SerializationUtil; import dev.dbos.transact.utils.DBUtils; import dev.dbos.transact.utils.PgContainer; import dev.dbos.transact.workflow.Workflow; +import dev.dbos.transact.workflow.WorkflowHandle; import java.sql.Connection; import java.sql.SQLException; @@ -18,46 +25,89 @@ import org.junit.jupiter.api.Test; interface FactoryTestService { - int greet(String user) throws SQLException; + record TestResult(String user, int greetCount) {} + + TestResult insertWorkflow(String user) throws SQLException; + + TestResult errorWorkflow(String user) throws SQLException; + + TestResult readWorkflow(String user) throws SQLException; + + TestResult insertThenReadWorkflow(String user) throws SQLException; } class FactoryTestServiceImpl implements FactoryTestService { - private final DBOS dbos; private final JdbcStepFactory stepFactory; - public FactoryTestServiceImpl(DBOS dbos, JdbcStepFactory stepFactory) { - this.dbos = dbos; + public FactoryTestServiceImpl(JdbcStepFactory stepFactory) { this.stepFactory = stepFactory; } - int insertGreeting(Connection conn, String user) throws SQLException { + TestResult insertGreeting(Connection conn, String user) throws SQLException { var sql = """ - INSERT INTO greetings(name, greet_count) - VALUES (?, 1) - ON CONFLICT(name) - DO UPDATE SET greet_count = greetings.greet_count + 1 - RETURNING greet_count - """; + INSERT INTO greetings(name, greet_count) + VALUES (?, 1) + ON CONFLICT(name) + DO UPDATE SET greet_count = greetings.greet_count + 1 + RETURNING greet_count + """; try (var stmt = conn.prepareStatement(sql)) { stmt.setString(1, Objects.requireNonNull(user)); try (var rs = stmt.executeQuery()) { - if (rs.next()) { - return rs.getInt("greet_count"); - } else { - return 0; - } + var greetCount = rs.next() ? rs.getInt("greet_count") : 0; + return new TestResult(user, greetCount); + } + } + } + + TestResult errorGreeting(Connection conn, String user) throws SQLException { + insertGreeting(conn, user); + throw new RuntimeException("Test Exception %d".formatted(System.currentTimeMillis())); + } + + TestResult readGreeting(Connection conn, String user) throws SQLException { + var sql = + """ + SELECT greet_count + FROM greetings + WHERE name = ? + """; + try (var stmt = conn.prepareStatement(sql)) { + stmt.setString(1, Objects.requireNonNull(user)); + try (var rs = stmt.executeQuery()) { + var greetCount = rs.next() ? rs.getInt("greet_count") : 0; + return new TestResult(user, greetCount); } } } @Override @Workflow - public int greet(String user) throws SQLException { + public TestResult insertWorkflow(String user) throws SQLException { return stepFactory.txStep((Connection c) -> insertGreeting(c, user), "insertGreeting"); } + + @Override + @Workflow + public TestResult errorWorkflow(String user) throws SQLException { + return stepFactory.txStep((Connection c) -> errorGreeting(c, user), "errorGreeting"); + } + + @Override + @Workflow + public TestResult readWorkflow(String user) throws SQLException { + return stepFactory.txStep((Connection c) -> readGreeting(c, user), "readGreeting"); + } + + @Override + @Workflow + public TestResult insertThenReadWorkflow(String user) throws SQLException { + stepFactory.txStep((Connection c) -> insertGreeting(c, user), "insertGreeting"); + return stepFactory.txStep((Connection c) -> readGreeting(c, user), "readGreeting"); + } } public class JdbcStepFactoryTest { @@ -67,36 +117,313 @@ public class JdbcStepFactoryTest { @AutoClose DBOS dbos; @AutoClose HikariDataSource dataSource; JdbcStepFactory stepFactory; + FactoryTestService proxy; + FactoryTestServiceImpl impl; @BeforeEach - void beforeEach() { + void beforeEach() throws SQLException { + dbosConfig = pgContainer.dbosConfig(); dataSource = pgContainer.dataSource(); + + try (var conn = dataSource.getConnection(); + var stmt = conn.createStatement()) { + stmt.execute( + "CREATE TABLE greetings(name text NOT NULL, greet_count integer DEFAULT 0, PRIMARY KEY(name))"); + } + dbos = new DBOS(dbosConfig); stepFactory = new JdbcStepFactory(dbos, dataSource); + + impl = new FactoryTestServiceImpl(stepFactory); + proxy = dbos.registerProxy(FactoryTestService.class, impl); + + dbos.launch(); } - @Test - public void foo() throws Exception { - { - var sql = - "CREATE TABLE greetings(name text NOT NULL, greet_count integer DEFAULT 0, PRIMARY KEY(name))"; - try (var conn = dataSource.getConnection(); - var stmt = conn.createStatement()) { - stmt.execute(sql); + private int getGreetCount(String user) throws SQLException { + var sql = "SELECT greet_count FROM greetings WHERE name = ?"; + try (var conn = dataSource.getConnection(); + var stmt = conn.prepareStatement(sql)) { + stmt.setString(1, user); + try (var rs = stmt.executeQuery()) { + return rs.next() ? rs.getInt("greet_count") : 0; } } - var proxy = - dbos.registerProxy(FactoryTestService.class, new FactoryTestServiceImpl(dbos, stepFactory)); + } + + @Test + public void testInsert() throws Exception { + var wfid = "wf1"; + var user = "testUser"; + try (var _o = new WorkflowOptions(wfid).setContext()) { + var result = proxy.insertWorkflow(user); + assertEquals(1, result.greetCount()); + assertEquals(user, result.user()); + } + + var rows = DBUtils.getTxStepRows(dataSource, wfid); + assertEquals(1, rows.size()); + var row = rows.get(0); + assertEquals(wfid, row.workflowId()); + assertEquals(0, row.stepId()); + assertNotNull(row.output()); + assertNull(row.error()); + assertEquals(SerializationUtil.NATIVE, row.serialization()); + var output = SerializationUtil.deserializeValue(row.output(), row.serialization(), null); + assertEquals(new FactoryTestService.TestResult(user, 1), output); + + assertEquals(1, getGreetCount(user)); + } + + @Test + public void testError() throws Exception { + var wfid = "wf1"; + var user = "testUser"; + try (var _o = new WorkflowOptions(wfid).setContext()) { + assertThrows(RuntimeException.class, () -> proxy.errorWorkflow(user)); + } + + // Transaction rolled back — no greeting inserted + var rows = DBUtils.getTxStepRows(dataSource, wfid); + assertEquals(1, rows.size()); + var row = rows.get(0); + assertEquals(wfid, row.workflowId()); + assertEquals(0, row.stepId()); + assertNull(row.output()); + assertNotNull(row.error()); + + assertEquals(0, getGreetCount(user)); + } + + @Test + public void testRead() throws Exception { + var insertWfid = "wf1"; + var readWfid = "wf2"; + var user = "testUser"; + + try (var _o = new WorkflowOptions(insertWfid).setContext()) { + proxy.insertWorkflow(user); + } + + try (var _o = new WorkflowOptions(readWfid).setContext()) { + var result = proxy.readWorkflow(user); + assertEquals(1, result.greetCount()); + assertEquals(user, result.user()); + } + + var rows = DBUtils.getTxStepRows(dataSource, readWfid); + assertEquals(1, rows.size()); + var row = rows.get(0); + assertEquals(readWfid, row.workflowId()); + assertEquals(0, row.stepId()); + assertNotNull(row.output()); + assertNull(row.error()); + assertEquals(SerializationUtil.NATIVE, row.serialization()); + var output = SerializationUtil.deserializeValue(row.output(), row.serialization(), null); + assertEquals(new FactoryTestService.TestResult(user, 1), output); + + assertEquals(1, getGreetCount(user)); + } + + @Test + public void testIdempotency() throws Exception { + var wfid = "wf1"; + var user = "testUser"; + + try (var _o = new WorkflowOptions(wfid).setContext()) { + var result = proxy.insertWorkflow(user); + assertEquals(1, result.greetCount()); + assertEquals(user, result.user()); + } + + // Second call with same wfid — txStep output is cached, insert not re-executed + try (var _o = new WorkflowOptions(wfid).setContext()) { + var result = proxy.insertWorkflow(user); + assertEquals(1, result.greetCount()); + assertEquals(user, result.user()); + } + + assertEquals(1, getGreetCount(user)); + assertEquals(1, DBUtils.getTxStepRows(dataSource, wfid).size()); + } + + @Test + public void testRetryError() throws Exception { + var wfid = "wf1"; + var user = "testUser"; + + try (var _o = new WorkflowOptions(wfid).setContext()) { + assertThrows(RuntimeException.class, () -> proxy.errorWorkflow(user)); + } + assertEquals(0, getGreetCount(user)); + dbos.close(); + + try (var conn = dataSource.getConnection(); + var stmt = + conn.prepareStatement("DELETE FROM dbos.operation_outputs WHERE workflow_uuid = ?")) { + stmt.setString(1, wfid); + stmt.executeUpdate(); + } + DBUtils.setWorkflowState(dataSource, wfid, "PENDING"); + + assertEquals(0, DBUtils.getStepRows(dataSource, wfid).size()); + assertEquals(1, DBUtils.getTxStepRows(dataSource, wfid).size()); + + dbos.launch(); + WorkflowHandle handle = + dbos.retrieveWorkflow(wfid); + assertThrows(RuntimeException.class, handle::getResult); + + // Cached error replayed — insert still not committed + assertEquals(0, getGreetCount(user)); + var txSteps = DBUtils.getTxStepRows(dataSource, wfid); + assertEquals(1, txSteps.size()); + assertNull(txSteps.get(0).output()); + assertNotNull(txSteps.get(0).error()); + } + + @Test + public void testMultipleTxSteps() throws Exception { + var wfid = "wf1"; + var user = "testUser"; + + try (var _o = new WorkflowOptions(wfid).setContext()) { + var result = proxy.insertThenReadWorkflow(user); + assertEquals(1, result.greetCount()); + assertEquals(user, result.user()); + } + + assertEquals(1, getGreetCount(user)); + + var rows = DBUtils.getTxStepRows(dataSource, wfid); + assertEquals(2, rows.size()); + assertEquals(0, rows.get(0).stepId()); + assertNotNull(rows.get(0).output()); + assertNull(rows.get(0).error()); + assertEquals(1, rows.get(1).stepId()); + assertNotNull(rows.get(1).output()); + assertNull(rows.get(1).error()); + } + + @Test + public void testDistinctWorkflows() throws Exception { + var wfid1 = "wf1"; + var wfid2 = "wf2"; + var user = "testUser"; + + try (var _o = new WorkflowOptions(wfid1).setContext()) { + var result = proxy.insertWorkflow(user); + assertEquals(1, result.greetCount()); + } + + try (var _o = new WorkflowOptions(wfid2).setContext()) { + var result = proxy.insertWorkflow(user); + assertEquals(2, result.greetCount()); + } + + assertEquals(2, getGreetCount(user)); + assertEquals(1, DBUtils.getTxStepRows(dataSource, wfid1).size()); + assertEquals(1, DBUtils.getTxStepRows(dataSource, wfid2).size()); + } + + @Test + public void testRetryPartialMultipleSteps() throws Exception { + var wfid = "wf1"; + var user = "testUser"; + try (var _o = new WorkflowOptions(wfid).setContext()) { + var result = proxy.insertThenReadWorkflow(user); + assertEquals(1, result.greetCount()); + assertEquals(user, result.user()); + } + assertEquals(1, getGreetCount(user)); + dbos.close(); + + // Simulate crash after step 0 wrote tx_step_outputs but before step 1 ran: + // both operation_outputs rows are gone, and step 1 has no tx_step_outputs entry + try (var conn = dataSource.getConnection(); + var stmt = + conn.prepareStatement("DELETE FROM dbos.operation_outputs WHERE workflow_uuid = ?")) { + stmt.setString(1, wfid); + stmt.executeUpdate(); + } + try (var conn = dataSource.getConnection(); + var stmt = + conn.prepareStatement( + "DELETE FROM dbos.tx_step_outputs WHERE workflow_id = ? AND step_id = 1")) { + stmt.setString(1, wfid); + stmt.executeUpdate(); + } + DBUtils.setWorkflowState(dataSource, wfid, "PENDING"); + + assertEquals(0, DBUtils.getStepRows(dataSource, wfid).size()); + assertEquals(1, DBUtils.getTxStepRows(dataSource, wfid).size()); + + var relaunchTimestamp = System.currentTimeMillis(); dbos.launch(); - assertEquals(1, proxy.greet("Luna")); - assertEquals(2, proxy.greet("Luna")); - assertEquals(3, proxy.greet("Luna")); - assertEquals(4, proxy.greet("Luna")); - assertEquals(5, proxy.greet("Luna")); - - var txStepRows = DBUtils.getTxStepRows(dataSource); - assertEquals(5, txStepRows.size()); + WorkflowHandle handle = + dbos.retrieveWorkflow(wfid); + var result = (FactoryTestService.TestResult) handle.getResult(); + assertEquals(1, result.greetCount()); + assertEquals(user, result.user()); + + // Step 0 cache hit — insert not re-executed + assertEquals(1, getGreetCount(user)); + + var txSteps = DBUtils.getTxStepRows(dataSource, wfid); + assertEquals(2, txSteps.size()); + assertTrue(txSteps.get(0).createdAt() < relaunchTimestamp); // step 0: original run + assertTrue(txSteps.get(1).createdAt() >= relaunchTimestamp); // step 1: re-executed on retry + } + + @Test + public void testRetryInsert() throws Exception { + var timestamp = System.currentTimeMillis(); + + var wfid = "wf1"; + var user = "testUser"; + try (var _o = new WorkflowOptions(wfid).setContext()) { + var result = proxy.insertWorkflow(user); + assertEquals(1, result.greetCount()); + assertEquals(user, result.user()); + } + dbos.close(); + + try (var conn = dataSource.getConnection(); + var stmt = + conn.prepareStatement("DELETE FROM dbos.operation_outputs WHERE workflow_uuid = ?")) { + stmt.setString(1, wfid); + stmt.executeUpdate(); + } + DBUtils.setWorkflowState(dataSource, wfid, "PENDING"); + + assertEquals(0, DBUtils.getStepRows(dataSource, wfid).size()); + assertEquals(1, DBUtils.getTxStepRows(dataSource, wfid).size()); + + var relaunchTimestamp = System.currentTimeMillis(); + dbos.launch(); + var handle = dbos.retrieveWorkflow(wfid); + var result = (FactoryTestService.TestResult) handle.getResult(); + assertEquals(1, result.greetCount()); + assertEquals(user, result.user()); + + var steps = DBUtils.getStepRows(dataSource, wfid); + var txSteps = DBUtils.getTxStepRows(dataSource, wfid); + assertEquals(1, steps.size()); + assertEquals(1, txSteps.size()); + + var step = steps.get(0); + var txStep = txSteps.get(0); + assertEquals(step.output(), txStep.output()); + assertEquals(step.error(), txStep.error()); + + assertTrue(txStep.createdAt() < step.startedAt()); + assertTrue(timestamp < txStep.createdAt()); + assertTrue(txStep.createdAt() < relaunchTimestamp); + assertTrue(relaunchTimestamp < step.startedAt()); + + // Retry reads from tx_step_outputs cache — insert not re-executed + assertEquals(1, getGreetCount(user)); } } diff --git a/transact/src/test/java/dev/dbos/transact/utils/DBUtils.java b/transact/src/test/java/dev/dbos/transact/utils/DBUtils.java index 13e801cd2..b84eb80b2 100644 --- a/transact/src/test/java/dev/dbos/transact/utils/DBUtils.java +++ b/transact/src/test/java/dev/dbos/transact/utils/DBUtils.java @@ -19,6 +19,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import javax.sql.DataSource; @@ -448,14 +449,15 @@ public static List getStreamEntries(DataSource ds, String workflowId, } } - public static List getTxStepRows(DataSource ds) throws SQLException { - return getTxStepRows(ds, null); + public static List getAllTxStepRows(DataSource ds) throws SQLException { + return getAllTxStepRows(ds, null); } - public static List getTxStepRows(DataSource ds, String schema) + public static List getAllTxStepRows(DataSource ds, String schema) throws SQLException { - schema = SystemDatabase.sanitizeSchema(schema); - var sql = "SELECT * FROM \"%s\".tx_step_outputs ORDER BY created_at".formatted(schema); + var sql = + "SELECT * FROM \"%s\".tx_step_outputs ORDER BY created_at" + .formatted(SystemDatabase.sanitizeSchema(schema)); try (var conn = ds.getConnection(); var stmt = conn.createStatement(); var rs = stmt.executeQuery(sql)) { @@ -467,6 +469,29 @@ public static List getTxStepRows(DataSource ds, String schema) } } + public static List getTxStepRows(DataSource ds, String workflowId) + throws SQLException { + return getTxStepRows(ds, workflowId, null); + } + + public static List getTxStepRows(DataSource ds, String workflowId, String schema) + throws SQLException { + var sql = + "SELECT * FROM \"%s\".tx_step_outputs WHERE workflow_id = ? ORDER BY step_id" + .formatted(SystemDatabase.sanitizeSchema(schema)); + try (var conn = ds.getConnection(); + var stmt = conn.prepareStatement(sql)) { + stmt.setString(1, Objects.requireNonNull(workflowId)); + try (var rs = stmt.executeQuery()) { + List rows = new ArrayList<>(); + while (rs.next()) { + rows.add(new TxStepOutputRow(rs)); + } + return rows; + } + } + } + public static List> dumpResultSet(ResultSet rs) throws SQLException { List> results = new ArrayList<>(); var metaData = rs.getMetaData(); From b551165e45373020abff76bc1b0a258b57be2281 Mon Sep 17 00:00:00 2001 From: Harry Pierson Date: Mon, 4 May 2026 12:36:26 -0700 Subject: [PATCH 06/29] spotless --- .../dbos/transact/txstep/JdbcStepFactory.java | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/transact/src/main/java/dev/dbos/transact/txstep/JdbcStepFactory.java b/transact/src/main/java/dev/dbos/transact/txstep/JdbcStepFactory.java index 14cad37bc..6fc8ec4c8 100644 --- a/transact/src/main/java/dev/dbos/transact/txstep/JdbcStepFactory.java +++ b/transact/src/main/java/dev/dbos/transact/txstep/JdbcStepFactory.java @@ -239,25 +239,34 @@ private R txStepInternal( var workflowId = Objects.requireNonNull(DBOS.workflowId()); int stepId = Objects.requireNonNull(DBOS.stepId()); - logger.debug("txStep starting: workflowId={} stepId={} stepName={}", workflowId, stepId, stepName); + logger.debug( + "txStep starting: workflowId={} stepId={} stepName={}", workflowId, stepId, stepName); var prevResult = this.checkExecution(workflowId, stepId, stepName); if (prevResult != null) { - logger.debug("txStep cache hit: workflowId={} stepId={} stepName={}", workflowId, stepId, stepName); + logger.debug( + "txStep cache hit: workflowId={} stepId={} stepName={}", workflowId, stepId, stepName); return prevResult.toResult(serializer); } - logger.debug("txStep executing: workflowId={} stepId={} stepName={}", workflowId, stepId, stepName); + logger.debug( + "txStep executing: workflowId={} stepId={} stepName={}", workflowId, stepId, stepName); var conn = getConnection(); try { var retVal = func.execute(conn); recordOutput(conn, workflowId, stepId, retVal); commit(conn); - logger.debug("txStep succeeded: workflowId={} stepId={} stepName={}", workflowId, stepId, stepName); + logger.debug( + "txStep succeeded: workflowId={} stepId={} stepName={}", workflowId, stepId, stepName); return retVal; } catch (Exception e) { rollback(conn); recordError(workflowId, stepId, e); - logger.debug("txStep failed: workflowId={} stepId={} stepName={} error={}", workflowId, stepId, stepName, e.getMessage()); + logger.debug( + "txStep failed: workflowId={} stepId={} stepName={} error={}", + workflowId, + stepId, + stepName, + e.getMessage()); throw e; } finally { close(conn); From c98168aefba75d6dc10d8c86e01fc575f0ce085c Mon Sep 17 00:00:00 2001 From: Harry Pierson Date: Mon, 4 May 2026 12:52:02 -0700 Subject: [PATCH 07/29] transact-jdbi-step-provider --- gradle/libs.versions.toml | 2 + settings.gradle.kts | 2 +- transact-jdbi-step-provider/build.gradle.kts | 22 + .../dbos/transact/jdbi/JdbiStepFactory.java | 196 ++++++++ .../transact/jdbi/JdbiStepFactoryTest.java | 428 ++++++++++++++++++ .../java/dev/dbos/transact/utils/DBUtils.java | 69 +++ .../transact/utils/OperationOutputRow.java | 27 ++ .../dev/dbos/transact/utils/PgContainer.java | 117 +++++ .../dbos/transact/utils/TxStepOutputRow.java | 23 + 9 files changed, 885 insertions(+), 1 deletion(-) create mode 100644 transact-jdbi-step-provider/build.gradle.kts create mode 100644 transact-jdbi-step-provider/src/main/java/dev/dbos/transact/jdbi/JdbiStepFactory.java create mode 100644 transact-jdbi-step-provider/src/test/java/dev/dbos/transact/jdbi/JdbiStepFactoryTest.java create mode 100644 transact-jdbi-step-provider/src/test/java/dev/dbos/transact/utils/DBUtils.java create mode 100644 transact-jdbi-step-provider/src/test/java/dev/dbos/transact/utils/OperationOutputRow.java create mode 100644 transact-jdbi-step-provider/src/test/java/dev/dbos/transact/utils/PgContainer.java create mode 100644 transact-jdbi-step-provider/src/test/java/dev/dbos/transact/utils/TxStepOutputRow.java diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 59417e198..0f256ee6d 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -4,6 +4,7 @@ aspectj = "1.9.22.1" assertj = "3.27.3" cron-utils = "9.2.1" hikaricp = "7.0.2" +jdbi = "3.47.0" kryo = "5.6.2" jackson = "2.21.2" java-websocket = "1.6.0" @@ -34,6 +35,7 @@ aspectjweaver = { module = "org.aspectj:aspectjweaver", version.ref = "aspectj" assertj-core = { module = "org.assertj:assertj-core", version.ref = "assertj" } cron-utils = { module = "com.cronutils:cron-utils", version.ref = "cron-utils" } hikaricp = { module = "com.zaxxer:HikariCP", version.ref = "hikaricp" } +jdbi-core = { module = "org.jdbi:jdbi3-core", version.ref = "jdbi" } jackson-databind = { module = "com.fasterxml.jackson.core:jackson-databind", version.ref = "jackson" } jackson-jsr310 = { module = "com.fasterxml.jackson.datatype:jackson-datatype-jsr310", version.ref = "jackson" } java-websocket = { module = "org.java-websocket:Java-WebSocket", version.ref = "java-websocket" } diff --git a/settings.gradle.kts b/settings.gradle.kts index ae7fee16b..00e950099 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -1,6 +1,6 @@ rootProject.name = "dbos-transact-java" -include("transact", "transact-cli", "transact-spring-boot-starter") +include("transact", "transact-cli", "transact-spring-boot-starter", "transact-jdbi-step-provider") plugins { id("org.gradle.toolchains.foojay-resolver") version "1.0.0" } diff --git a/transact-jdbi-step-provider/build.gradle.kts b/transact-jdbi-step-provider/build.gradle.kts new file mode 100644 index 000000000..35e623b2f --- /dev/null +++ b/transact-jdbi-step-provider/build.gradle.kts @@ -0,0 +1,22 @@ +plugins { id("java-library") } + +tasks.withType { + options.compilerArgs.add("-Xlint:unchecked") + options.compilerArgs.add("-Xlint:deprecation") + options.compilerArgs.add("-Xlint:rawtypes") + options.compilerArgs.add("-Werror") +} + +dependencies { + api(project(":transact")) + api(libs.jdbi.core) + + testImplementation(platform(libs.junit.bom)) + testImplementation(libs.junit.jupiter) + testRuntimeOnly(libs.junit.platform.launcher) + + testRuntimeOnly(libs.logback.classic) + testImplementation(libs.testcontainers.postgresql) + testImplementation(libs.postgresql) + testImplementation(libs.hikaricp) +} diff --git a/transact-jdbi-step-provider/src/main/java/dev/dbos/transact/jdbi/JdbiStepFactory.java b/transact-jdbi-step-provider/src/main/java/dev/dbos/transact/jdbi/JdbiStepFactory.java new file mode 100644 index 000000000..0be95edd4 --- /dev/null +++ b/transact-jdbi-step-provider/src/main/java/dev/dbos/transact/jdbi/JdbiStepFactory.java @@ -0,0 +1,196 @@ +package dev.dbos.transact.jdbi; + +import dev.dbos.transact.DBOS; +import dev.dbos.transact.database.SystemDatabase; +import dev.dbos.transact.execution.ThrowingConsumer; +import dev.dbos.transact.execution.ThrowingFunction; +import dev.dbos.transact.json.DBOSSerializer; +import dev.dbos.transact.json.SerializationUtil; +import dev.dbos.transact.txstep.JdbcStepFactory; +import dev.dbos.transact.workflow.internal.StepResult; + +import java.sql.SQLException; +import java.util.Objects; + +import org.jdbi.v3.core.Handle; +import org.jdbi.v3.core.Jdbi; +import org.jspecify.annotations.Nullable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class JdbiStepFactory { + + private static final Logger logger = LoggerFactory.getLogger(JdbiStepFactory.class); + + private final DBOS dbos; + private final Jdbi jdbi; + private final String schema; + private final DBOSSerializer serializer; + + public JdbiStepFactory(DBOS dbos, Jdbi jdbi) { + this(dbos, jdbi, null, null); + } + + public JdbiStepFactory(DBOS dbos, Jdbi jdbi, String schema) { + this(dbos, jdbi, schema, null); + } + + public JdbiStepFactory(DBOS dbos, Jdbi jdbi, DBOSSerializer serializer) { + this(dbos, jdbi, null, serializer); + } + + public JdbiStepFactory(DBOS dbos, Jdbi jdbi, String schema, DBOSSerializer serializer) { + this.dbos = Objects.requireNonNull(dbos); + this.jdbi = Objects.requireNonNull(jdbi); + var config = dbos.integration().config(); + this.schema = SystemDatabase.sanitizeSchema(schema == null ? config.databaseSchema() : schema); + this.serializer = serializer == null ? config.serializer() : serializer; + + try { + jdbi.useHandle( + handle -> { + try { + JdbcStepFactory.ensureSchema(handle.getConnection(), this.schema); + JdbcStepFactory.ensureTxOutputTable(handle.getConnection(), this.schema); + } catch (SQLException e) { + throw new RuntimeException(e.getMessage(), e); + } + }); + } catch (Exception e) { + if (e instanceof RuntimeException) { + throw (RuntimeException) e; + } + throw new RuntimeException(e); + } + } + + private @Nullable StepResult checkExecution(String workflowId, int stepId, String stepName) { + var sql = + """ + SELECT output, error, serialization + FROM "%s".tx_step_outputs + WHERE workflow_id = ? AND step_id = ? + """ + .formatted(this.schema); + return jdbi.withHandle( + handle -> + handle + .createQuery(sql) + .bind(0, workflowId) + .bind(1, stepId) + .map( + (rs, ctx) -> + new StepResult( + workflowId, + stepId, + stepName, + rs.getString("output"), + rs.getString("error"), + null, + rs.getString("serialization"))) + .findFirst() + .orElse(null)); + } + + private void recordResult( + Handle handle, + String workflowId, + int stepId, + String output, + String error, + String serialization) { + if (output != null && error != null) { + throw new IllegalArgumentException("attempted to record non null output and error result"); + } + + String sql = + """ + INSERT INTO "%s".tx_step_outputs + (workflow_id, step_id, output, error, serialization) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT DO NOTHING + """ + .formatted(schema); + + handle + .createUpdate(sql) + .bind(0, workflowId) + .bind(1, stepId) + .bind(2, output) + .bind(3, error) + .bind(4, serialization) + .execute(); + } + + private void recordOutput(Handle handle, String workflowId, int stepId, R retVal) { + var value = SerializationUtil.serializeValue(retVal, null, serializer); + recordResult(handle, workflowId, stepId, value.serializedValue(), null, value.serialization()); + } + + private void recordError(String workflowId, int stepId, E exception) { + var value = SerializationUtil.serializeError(exception, null, serializer); + jdbi.useHandle( + handle -> + recordResult( + handle, workflowId, stepId, null, value.serializedValue(), value.serialization())); + } + + private R txStepInternal( + ThrowingFunction func, String stepName) throws E { + var workflowId = Objects.requireNonNull(DBOS.workflowId()); + int stepId = Objects.requireNonNull(DBOS.stepId()); + + logger.debug( + "txStep starting: workflowId={} stepId={} stepName={}", workflowId, stepId, stepName); + var prevResult = this.checkExecution(workflowId, stepId, stepName); + if (prevResult != null) { + logger.debug( + "txStep cache hit: workflowId={} stepId={} stepName={}", workflowId, stepId, stepName); + return prevResult.toResult(serializer); + } + + logger.debug( + "txStep executing: workflowId={} stepId={} stepName={}", workflowId, stepId, stepName); + var handle = jdbi.open(); + try { + handle.begin(); + var retVal = func.execute(handle); + recordOutput(handle, workflowId, stepId, retVal); + handle.commit(); + logger.debug( + "txStep succeeded: workflowId={} stepId={} stepName={}", workflowId, stepId, stepName); + return retVal; + } catch (Exception e) { + handle.rollback(); + recordError(workflowId, stepId, e); + logger.debug( + "txStep failed: workflowId={} stepId={} stepName={} error={}", + workflowId, + stepId, + stepName, + e.getMessage()); + throw e; + } finally { + handle.close(); + } + } + + public R txStep(ThrowingFunction func, String stepName) + throws E { + return dbos.runStep( + () -> { + return this.txStepInternal(func, stepName); + }, + stepName); + } + + public void txStep(ThrowingConsumer func, String stepName) + throws E { + txStep( + h -> { + func.execute(h); + return null; + }, + stepName); + } +} diff --git a/transact-jdbi-step-provider/src/test/java/dev/dbos/transact/jdbi/JdbiStepFactoryTest.java b/transact-jdbi-step-provider/src/test/java/dev/dbos/transact/jdbi/JdbiStepFactoryTest.java new file mode 100644 index 000000000..7f9699e37 --- /dev/null +++ b/transact-jdbi-step-provider/src/test/java/dev/dbos/transact/jdbi/JdbiStepFactoryTest.java @@ -0,0 +1,428 @@ +package dev.dbos.transact.jdbi; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import dev.dbos.transact.DBOS; +import dev.dbos.transact.config.DBOSConfig; +import dev.dbos.transact.context.WorkflowOptions; +import dev.dbos.transact.json.SerializationUtil; +import dev.dbos.transact.utils.DBUtils; +import dev.dbos.transact.utils.PgContainer; +import dev.dbos.transact.workflow.Workflow; +import dev.dbos.transact.workflow.WorkflowHandle; + +import java.sql.SQLException; +import java.util.Objects; + +import com.zaxxer.hikari.HikariDataSource; +import org.jdbi.v3.core.Handle; +import org.jdbi.v3.core.Jdbi; +import org.junit.jupiter.api.AutoClose; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +interface FactoryTestService { + record TestResult(String user, int greetCount) {} + + TestResult insertWorkflow(String user); + + TestResult errorWorkflow(String user); + + TestResult readWorkflow(String user); + + TestResult insertThenReadWorkflow(String user); +} + +class FactoryTestServiceImpl implements FactoryTestService { + + private final JdbiStepFactory stepFactory; + + public FactoryTestServiceImpl(JdbiStepFactory stepFactory) { + this.stepFactory = stepFactory; + } + + FactoryTestService.TestResult insertGreeting(Handle handle, String user) { + var sql = + """ + INSERT INTO greetings(name, greet_count) + VALUES (?, 1) + ON CONFLICT(name) + DO UPDATE SET greet_count = greetings.greet_count + 1 + RETURNING greet_count + """; + return handle + .createQuery(sql) + .bind(0, Objects.requireNonNull(user)) + .map((rs, ctx) -> new FactoryTestService.TestResult(user, rs.getInt("greet_count"))) + .findFirst() + .orElse(new FactoryTestService.TestResult(user, 0)); + } + + FactoryTestService.TestResult errorGreeting(Handle handle, String user) { + insertGreeting(handle, user); + throw new RuntimeException("Test Exception %d".formatted(System.currentTimeMillis())); + } + + FactoryTestService.TestResult readGreeting(Handle handle, String user) { + var sql = + """ + SELECT greet_count + FROM greetings + WHERE name = ? + """; + return handle + .createQuery(sql) + .bind(0, Objects.requireNonNull(user)) + .map((rs, ctx) -> new FactoryTestService.TestResult(user, rs.getInt("greet_count"))) + .findFirst() + .orElse(new FactoryTestService.TestResult(user, 0)); + } + + @Override + @Workflow + public FactoryTestService.TestResult insertWorkflow(String user) { + return stepFactory.txStep((Handle h) -> insertGreeting(h, user), "insertGreeting"); + } + + @Override + @Workflow + public FactoryTestService.TestResult errorWorkflow(String user) { + return stepFactory.txStep((Handle h) -> errorGreeting(h, user), "errorGreeting"); + } + + @Override + @Workflow + public FactoryTestService.TestResult readWorkflow(String user) { + return stepFactory.txStep((Handle h) -> readGreeting(h, user), "readGreeting"); + } + + @Override + @Workflow + public FactoryTestService.TestResult insertThenReadWorkflow(String user) { + stepFactory.txStep((Handle h) -> insertGreeting(h, user), "insertGreeting"); + return stepFactory.txStep((Handle h) -> readGreeting(h, user), "readGreeting"); + } +} + +public class JdbiStepFactoryTest { + @AutoClose final PgContainer pgContainer = new PgContainer(); + + DBOSConfig dbosConfig; + @AutoClose DBOS dbos; + @AutoClose HikariDataSource dataSource; + JdbiStepFactory stepFactory; + FactoryTestService proxy; + FactoryTestServiceImpl impl; + + @BeforeEach + void beforeEach() throws SQLException { + + dbosConfig = pgContainer.dbosConfig(); + dataSource = pgContainer.dataSource(); + + try (var conn = dataSource.getConnection(); + var stmt = conn.createStatement()) { + stmt.execute( + "CREATE TABLE greetings(name text NOT NULL, greet_count integer DEFAULT 0, PRIMARY KEY(name))"); + } + + dbos = new DBOS(dbosConfig); + Jdbi jdbi = Jdbi.create(dataSource); + stepFactory = new JdbiStepFactory(dbos, jdbi); + + impl = new FactoryTestServiceImpl(stepFactory); + proxy = dbos.registerProxy(FactoryTestService.class, impl); + + dbos.launch(); + } + + private int getGreetCount(String user) throws SQLException { + var sql = "SELECT greet_count FROM greetings WHERE name = ?"; + try (var conn = dataSource.getConnection(); + var stmt = conn.prepareStatement(sql)) { + stmt.setString(1, user); + try (var rs = stmt.executeQuery()) { + return rs.next() ? rs.getInt("greet_count") : 0; + } + } + } + + @Test + public void testInsert() throws Exception { + var wfid = "wf1"; + var user = "testUser"; + try (var _o = new WorkflowOptions(wfid).setContext()) { + var result = proxy.insertWorkflow(user); + assertEquals(1, result.greetCount()); + assertEquals(user, result.user()); + } + + var rows = DBUtils.getTxStepRows(dataSource, wfid); + assertEquals(1, rows.size()); + var row = rows.get(0); + assertEquals(wfid, row.workflowId()); + assertEquals(0, row.stepId()); + assertNotNull(row.output()); + assertNull(row.error()); + assertEquals(SerializationUtil.NATIVE, row.serialization()); + var output = SerializationUtil.deserializeValue(row.output(), row.serialization(), null); + assertEquals(new FactoryTestService.TestResult(user, 1), output); + + assertEquals(1, getGreetCount(user)); + } + + @Test + public void testError() throws Exception { + var wfid = "wf1"; + var user = "testUser"; + try (var _o = new WorkflowOptions(wfid).setContext()) { + assertThrows(RuntimeException.class, () -> proxy.errorWorkflow(user)); + } + + // Transaction rolled back — no greeting inserted + var rows = DBUtils.getTxStepRows(dataSource, wfid); + assertEquals(1, rows.size()); + var row = rows.get(0); + assertEquals(wfid, row.workflowId()); + assertEquals(0, row.stepId()); + assertNull(row.output()); + assertNotNull(row.error()); + + assertEquals(0, getGreetCount(user)); + } + + @Test + public void testRead() throws Exception { + var insertWfid = "wf1"; + var readWfid = "wf2"; + var user = "testUser"; + + try (var _o = new WorkflowOptions(insertWfid).setContext()) { + proxy.insertWorkflow(user); + } + + try (var _o = new WorkflowOptions(readWfid).setContext()) { + var result = proxy.readWorkflow(user); + assertEquals(1, result.greetCount()); + assertEquals(user, result.user()); + } + + var rows = DBUtils.getTxStepRows(dataSource, readWfid); + assertEquals(1, rows.size()); + var row = rows.get(0); + assertEquals(readWfid, row.workflowId()); + assertEquals(0, row.stepId()); + assertNotNull(row.output()); + assertNull(row.error()); + assertEquals(SerializationUtil.NATIVE, row.serialization()); + var output = SerializationUtil.deserializeValue(row.output(), row.serialization(), null); + assertEquals(new FactoryTestService.TestResult(user, 1), output); + + assertEquals(1, getGreetCount(user)); + } + + @Test + public void testIdempotency() throws Exception { + var wfid = "wf1"; + var user = "testUser"; + + try (var _o = new WorkflowOptions(wfid).setContext()) { + var result = proxy.insertWorkflow(user); + assertEquals(1, result.greetCount()); + assertEquals(user, result.user()); + } + + // Second call with same wfid — txStep output is cached, insert not re-executed + try (var _o = new WorkflowOptions(wfid).setContext()) { + var result = proxy.insertWorkflow(user); + assertEquals(1, result.greetCount()); + assertEquals(user, result.user()); + } + + assertEquals(1, getGreetCount(user)); + assertEquals(1, DBUtils.getTxStepRows(dataSource, wfid).size()); + } + + @Test + public void testRetryError() throws Exception { + var wfid = "wf1"; + var user = "testUser"; + + try (var _o = new WorkflowOptions(wfid).setContext()) { + assertThrows(RuntimeException.class, () -> proxy.errorWorkflow(user)); + } + assertEquals(0, getGreetCount(user)); + dbos.close(); + + try (var conn = dataSource.getConnection(); + var stmt = + conn.prepareStatement("DELETE FROM dbos.operation_outputs WHERE workflow_uuid = ?")) { + stmt.setString(1, wfid); + stmt.executeUpdate(); + } + DBUtils.setWorkflowState(dataSource, wfid, "PENDING"); + + assertEquals(0, DBUtils.getStepRows(dataSource, wfid).size()); + assertEquals(1, DBUtils.getTxStepRows(dataSource, wfid).size()); + + dbos.launch(); + WorkflowHandle handle = + dbos.retrieveWorkflow(wfid); + assertThrows(RuntimeException.class, handle::getResult); + + // Cached error replayed — insert still not committed + assertEquals(0, getGreetCount(user)); + var txSteps = DBUtils.getTxStepRows(dataSource, wfid); + assertEquals(1, txSteps.size()); + assertNull(txSteps.get(0).output()); + assertNotNull(txSteps.get(0).error()); + } + + @Test + public void testMultipleTxSteps() throws Exception { + var wfid = "wf1"; + var user = "testUser"; + + try (var _o = new WorkflowOptions(wfid).setContext()) { + var result = proxy.insertThenReadWorkflow(user); + assertEquals(1, result.greetCount()); + assertEquals(user, result.user()); + } + + assertEquals(1, getGreetCount(user)); + + var rows = DBUtils.getTxStepRows(dataSource, wfid); + assertEquals(2, rows.size()); + assertEquals(0, rows.get(0).stepId()); + assertNotNull(rows.get(0).output()); + assertNull(rows.get(0).error()); + assertEquals(1, rows.get(1).stepId()); + assertNotNull(rows.get(1).output()); + assertNull(rows.get(1).error()); + } + + @Test + public void testDistinctWorkflows() throws Exception { + var wfid1 = "wf1"; + var wfid2 = "wf2"; + var user = "testUser"; + + try (var _o = new WorkflowOptions(wfid1).setContext()) { + var result = proxy.insertWorkflow(user); + assertEquals(1, result.greetCount()); + } + + try (var _o = new WorkflowOptions(wfid2).setContext()) { + var result = proxy.insertWorkflow(user); + assertEquals(2, result.greetCount()); + } + + assertEquals(2, getGreetCount(user)); + assertEquals(1, DBUtils.getTxStepRows(dataSource, wfid1).size()); + assertEquals(1, DBUtils.getTxStepRows(dataSource, wfid2).size()); + } + + @Test + public void testRetryPartialMultipleSteps() throws Exception { + var wfid = "wf1"; + var user = "testUser"; + + try (var _o = new WorkflowOptions(wfid).setContext()) { + var result = proxy.insertThenReadWorkflow(user); + assertEquals(1, result.greetCount()); + assertEquals(user, result.user()); + } + assertEquals(1, getGreetCount(user)); + dbos.close(); + + // Simulate crash after step 0 wrote tx_step_outputs but before step 1 ran: + // both operation_outputs rows are gone, and step 1 has no tx_step_outputs entry + try (var conn = dataSource.getConnection(); + var stmt = + conn.prepareStatement("DELETE FROM dbos.operation_outputs WHERE workflow_uuid = ?")) { + stmt.setString(1, wfid); + stmt.executeUpdate(); + } + try (var conn = dataSource.getConnection(); + var stmt = + conn.prepareStatement( + "DELETE FROM dbos.tx_step_outputs WHERE workflow_id = ? AND step_id = 1")) { + stmt.setString(1, wfid); + stmt.executeUpdate(); + } + DBUtils.setWorkflowState(dataSource, wfid, "PENDING"); + + assertEquals(0, DBUtils.getStepRows(dataSource, wfid).size()); + assertEquals(1, DBUtils.getTxStepRows(dataSource, wfid).size()); + + var relaunchTimestamp = System.currentTimeMillis(); + dbos.launch(); + WorkflowHandle handle = + dbos.retrieveWorkflow(wfid); + var result = (FactoryTestService.TestResult) handle.getResult(); + assertEquals(1, result.greetCount()); + assertEquals(user, result.user()); + + // Step 0 cache hit — insert not re-executed + assertEquals(1, getGreetCount(user)); + + var txSteps = DBUtils.getTxStepRows(dataSource, wfid); + assertEquals(2, txSteps.size()); + assertTrue(txSteps.get(0).createdAt() < relaunchTimestamp); // step 0: original run + assertTrue(txSteps.get(1).createdAt() >= relaunchTimestamp); // step 1: re-executed on retry + } + + @Test + public void testRetryInsert() throws Exception { + var timestamp = System.currentTimeMillis(); + + var wfid = "wf1"; + var user = "testUser"; + try (var _o = new WorkflowOptions(wfid).setContext()) { + var result = proxy.insertWorkflow(user); + assertEquals(1, result.greetCount()); + assertEquals(user, result.user()); + } + dbos.close(); + + try (var conn = dataSource.getConnection(); + var stmt = + conn.prepareStatement("DELETE FROM dbos.operation_outputs WHERE workflow_uuid = ?")) { + stmt.setString(1, wfid); + stmt.executeUpdate(); + } + DBUtils.setWorkflowState(dataSource, wfid, "PENDING"); + + assertEquals(0, DBUtils.getStepRows(dataSource, wfid).size()); + assertEquals(1, DBUtils.getTxStepRows(dataSource, wfid).size()); + + var relaunchTimestamp = System.currentTimeMillis(); + dbos.launch(); + var handle = dbos.retrieveWorkflow(wfid); + var result = (FactoryTestService.TestResult) handle.getResult(); + assertEquals(1, result.greetCount()); + assertEquals(user, result.user()); + + var steps = DBUtils.getStepRows(dataSource, wfid); + var txSteps = DBUtils.getTxStepRows(dataSource, wfid); + assertEquals(1, steps.size()); + assertEquals(1, txSteps.size()); + + var step = steps.get(0); + var txStep = txSteps.get(0); + assertEquals(step.output(), txStep.output()); + assertEquals(step.error(), txStep.error()); + + assertTrue(txStep.createdAt() < step.startedAt()); + assertTrue(timestamp < txStep.createdAt()); + assertTrue(txStep.createdAt() < relaunchTimestamp); + assertTrue(relaunchTimestamp < step.startedAt()); + + // Retry reads from tx_step_outputs cache — insert not re-executed + assertEquals(1, getGreetCount(user)); + } +} diff --git a/transact-jdbi-step-provider/src/test/java/dev/dbos/transact/utils/DBUtils.java b/transact-jdbi-step-provider/src/test/java/dev/dbos/transact/utils/DBUtils.java new file mode 100644 index 000000000..7f16faa15 --- /dev/null +++ b/transact-jdbi-step-provider/src/test/java/dev/dbos/transact/utils/DBUtils.java @@ -0,0 +1,69 @@ +package dev.dbos.transact.utils; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import dev.dbos.transact.database.SystemDatabase; + +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; + +import javax.sql.DataSource; + +public class DBUtils { + + public static List getTxStepRows(DataSource ds, String workflowId) + throws SQLException { + var sql = + "SELECT * FROM \"%s\".tx_step_outputs WHERE workflow_id = ? ORDER BY step_id" + .formatted(SystemDatabase.sanitizeSchema(null)); + try (var conn = ds.getConnection(); + var stmt = conn.prepareStatement(sql)) { + stmt.setString(1, Objects.requireNonNull(workflowId)); + try (var rs = stmt.executeQuery()) { + List rows = new ArrayList<>(); + while (rs.next()) { + rows.add(new TxStepOutputRow(rs)); + } + return rows; + } + } + } + + public static List getStepRows(DataSource ds, String workflowId) + throws SQLException { + var sql = + "SELECT * FROM \"%s\".operation_outputs WHERE workflow_uuid = ? ORDER BY function_id" + .formatted(SystemDatabase.sanitizeSchema(null)); + try (var conn = ds.getConnection(); + var stmt = conn.prepareStatement(sql)) { + stmt.setString(1, workflowId); + try (var rs = stmt.executeQuery()) { + List rows = new ArrayList<>(); + while (rs.next()) { + rows.add(new OperationOutputRow(rs)); + } + return rows; + } + } + } + + public static void setWorkflowState(DataSource ds, String workflowId, String newState) + throws SQLException { + String sql = + "UPDATE dbos.workflow_status SET status = ?, updated_at = ? WHERE workflow_uuid = ?"; + + try (var connection = ds.getConnection(); + PreparedStatement pstmt = connection.prepareStatement(sql)) { + pstmt.setString(1, newState); + pstmt.setLong(2, Instant.now().toEpochMilli()); + pstmt.setString(3, workflowId); + + int rowsAffected = pstmt.executeUpdate(); + assertEquals(1, rowsAffected); + } + } +} diff --git a/transact-jdbi-step-provider/src/test/java/dev/dbos/transact/utils/OperationOutputRow.java b/transact-jdbi-step-provider/src/test/java/dev/dbos/transact/utils/OperationOutputRow.java new file mode 100644 index 000000000..c1ac2938b --- /dev/null +++ b/transact-jdbi-step-provider/src/test/java/dev/dbos/transact/utils/OperationOutputRow.java @@ -0,0 +1,27 @@ +package dev.dbos.transact.utils; + +import java.sql.ResultSet; +import java.sql.SQLException; + +public record OperationOutputRow( + String workflowId, + int functionId, + String output, + String error, + String functionName, + String childWorkflowId, + Long startedAt, + Long completedAt) { + + public OperationOutputRow(ResultSet rs) throws SQLException { + this( + rs.getString("workflow_uuid"), + rs.getInt("function_id"), + rs.getString("output"), + rs.getString("error"), + rs.getString("function_name"), + rs.getString("child_workflow_id"), + rs.getObject("started_at_epoch_ms", Long.class), + rs.getObject("completed_at_epoch_ms", Long.class)); + } +} diff --git a/transact-jdbi-step-provider/src/test/java/dev/dbos/transact/utils/PgContainer.java b/transact-jdbi-step-provider/src/test/java/dev/dbos/transact/utils/PgContainer.java new file mode 100644 index 000000000..2c1431dd5 --- /dev/null +++ b/transact-jdbi-step-provider/src/test/java/dev/dbos/transact/utils/PgContainer.java @@ -0,0 +1,117 @@ +package dev.dbos.transact.utils; + +import dev.dbos.transact.DBOSClient; +import dev.dbos.transact.config.DBOSConfig; +import dev.dbos.transact.database.SystemDatabase; + +import java.sql.DriverManager; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Objects; +import java.util.UUID; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.Semaphore; + +import com.zaxxer.hikari.HikariDataSource; +import org.testcontainers.postgresql.PostgreSQLContainer; + +public class PgContainer implements AutoCloseable { + + private static final int SIZE = Runtime.getRuntime().availableProcessors(); + private static final BlockingQueue POOL = new ArrayBlockingQueue<>(SIZE); + private static final Semaphore PERMITS = new Semaphore(SIZE); + + static { + Runtime.getRuntime() + .addShutdownHook( + new Thread( + () -> { + var containers = new ArrayList(); + POOL.drainTo(containers); + containers.forEach(PostgreSQLContainer::stop); + })); + } + + static PostgreSQLContainer acquire() { + try { + PERMITS.acquire(); + var container = POOL.poll(); + if (container == null) { + container = new PostgreSQLContainer("postgres:18"); + container.start(); + } + return container; + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + + static void release(PostgreSQLContainer c) { + POOL.offer(c); + PERMITS.release(); + } + + private final PostgreSQLContainer pgContainer; + private final String jdbcUrl; + private final String dbName; + + public PgContainer() { + // take a container from the pool and create a new database for it + pgContainer = acquire(); + dbName = "test_" + UUID.randomUUID().toString().replace("-", ""); + jdbcUrl = pgContainer.getJdbcUrl().replaceFirst("/[^/]+$", "/" + dbName); + + try (var conn = + DriverManager.getConnection( + pgContainer.getJdbcUrl(), pgContainer.getUsername(), pgContainer.getPassword()); + var stmt = conn.createStatement()) { + stmt.execute("CREATE DATABASE " + dbName); + } catch (SQLException e) { + throw new RuntimeException(e); + } + } + + @Override + public void close() throws Exception { + // drop the database we created and return the container too the pool + var _jdbcUrl = pgContainer.getJdbcUrl(); + try (var conn = DriverManager.getConnection(_jdbcUrl, username(), password()); + var stmt = conn.createStatement()) { + var sql = "DROP DATABASE IF EXISTS %s WITH (FORCE)".formatted(dbName); + stmt.execute(sql); + } + release(pgContainer); + } + + public String jdbcUrl() { + return jdbcUrl; + } + + public String username() { + return pgContainer.getUsername(); + } + + public String password() { + return pgContainer.getPassword(); + } + + public DBOSConfig dbosConfig() { + return dbosConfig(null); + } + + public DBOSConfig dbosConfig(String appName) { + return DBOSConfig.defaults(Objects.requireNonNullElse(appName, "transact-java-test")) + .withDatabaseUrl(jdbcUrl()) + .withDbUser(username()) + .withDbPassword(password()); + } + + public HikariDataSource dataSource() { + return SystemDatabase.createDataSource(jdbcUrl(), username(), password()); + } + + public DBOSClient dbosClient() { + return new DBOSClient(jdbcUrl(), username(), password()); + } +} diff --git a/transact-jdbi-step-provider/src/test/java/dev/dbos/transact/utils/TxStepOutputRow.java b/transact-jdbi-step-provider/src/test/java/dev/dbos/transact/utils/TxStepOutputRow.java new file mode 100644 index 000000000..7472fe314 --- /dev/null +++ b/transact-jdbi-step-provider/src/test/java/dev/dbos/transact/utils/TxStepOutputRow.java @@ -0,0 +1,23 @@ +package dev.dbos.transact.utils; + +import java.sql.ResultSet; +import java.sql.SQLException; + +public record TxStepOutputRow( + String workflowId, + int stepId, + String output, + String error, + String serialization, + Long createdAt) { + + public TxStepOutputRow(ResultSet rs) throws SQLException { + this( + rs.getString("workflow_id"), + rs.getInt("step_id"), + rs.getString("output"), + rs.getString("error"), + rs.getString("serialization"), + rs.getObject("created_at", Long.class)); + } +} From 8e592bcddaf7d1f82f49a89168a1000e60f61208 Mon Sep 17 00:00:00 2001 From: Harry Pierson Date: Mon, 4 May 2026 12:53:29 -0700 Subject: [PATCH 08/29] fix jdbi package name --- settings.gradle.kts | 2 +- .../build.gradle.kts | 0 .../src/main/java/dev/dbos/transact/jdbi/JdbiStepFactory.java | 0 .../test/java/dev/dbos/transact/jdbi/JdbiStepFactoryTest.java | 0 .../src/test/java/dev/dbos/transact/utils/DBUtils.java | 0 .../test/java/dev/dbos/transact/utils/OperationOutputRow.java | 0 .../src/test/java/dev/dbos/transact/utils/PgContainer.java | 0 .../src/test/java/dev/dbos/transact/utils/TxStepOutputRow.java | 0 8 files changed, 1 insertion(+), 1 deletion(-) rename {transact-jdbi-step-provider => transact-jdbi-step-factory}/build.gradle.kts (100%) rename {transact-jdbi-step-provider => transact-jdbi-step-factory}/src/main/java/dev/dbos/transact/jdbi/JdbiStepFactory.java (100%) rename {transact-jdbi-step-provider => transact-jdbi-step-factory}/src/test/java/dev/dbos/transact/jdbi/JdbiStepFactoryTest.java (100%) rename {transact-jdbi-step-provider => transact-jdbi-step-factory}/src/test/java/dev/dbos/transact/utils/DBUtils.java (100%) rename {transact-jdbi-step-provider => transact-jdbi-step-factory}/src/test/java/dev/dbos/transact/utils/OperationOutputRow.java (100%) rename {transact-jdbi-step-provider => transact-jdbi-step-factory}/src/test/java/dev/dbos/transact/utils/PgContainer.java (100%) rename {transact-jdbi-step-provider => transact-jdbi-step-factory}/src/test/java/dev/dbos/transact/utils/TxStepOutputRow.java (100%) diff --git a/settings.gradle.kts b/settings.gradle.kts index 00e950099..6c95ecdcd 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -1,6 +1,6 @@ rootProject.name = "dbos-transact-java" -include("transact", "transact-cli", "transact-spring-boot-starter", "transact-jdbi-step-provider") +include("transact", "transact-cli", "transact-spring-boot-starter", "transact-jdbi-step-factory") plugins { id("org.gradle.toolchains.foojay-resolver") version "1.0.0" } diff --git a/transact-jdbi-step-provider/build.gradle.kts b/transact-jdbi-step-factory/build.gradle.kts similarity index 100% rename from transact-jdbi-step-provider/build.gradle.kts rename to transact-jdbi-step-factory/build.gradle.kts diff --git a/transact-jdbi-step-provider/src/main/java/dev/dbos/transact/jdbi/JdbiStepFactory.java b/transact-jdbi-step-factory/src/main/java/dev/dbos/transact/jdbi/JdbiStepFactory.java similarity index 100% rename from transact-jdbi-step-provider/src/main/java/dev/dbos/transact/jdbi/JdbiStepFactory.java rename to transact-jdbi-step-factory/src/main/java/dev/dbos/transact/jdbi/JdbiStepFactory.java diff --git a/transact-jdbi-step-provider/src/test/java/dev/dbos/transact/jdbi/JdbiStepFactoryTest.java b/transact-jdbi-step-factory/src/test/java/dev/dbos/transact/jdbi/JdbiStepFactoryTest.java similarity index 100% rename from transact-jdbi-step-provider/src/test/java/dev/dbos/transact/jdbi/JdbiStepFactoryTest.java rename to transact-jdbi-step-factory/src/test/java/dev/dbos/transact/jdbi/JdbiStepFactoryTest.java diff --git a/transact-jdbi-step-provider/src/test/java/dev/dbos/transact/utils/DBUtils.java b/transact-jdbi-step-factory/src/test/java/dev/dbos/transact/utils/DBUtils.java similarity index 100% rename from transact-jdbi-step-provider/src/test/java/dev/dbos/transact/utils/DBUtils.java rename to transact-jdbi-step-factory/src/test/java/dev/dbos/transact/utils/DBUtils.java diff --git a/transact-jdbi-step-provider/src/test/java/dev/dbos/transact/utils/OperationOutputRow.java b/transact-jdbi-step-factory/src/test/java/dev/dbos/transact/utils/OperationOutputRow.java similarity index 100% rename from transact-jdbi-step-provider/src/test/java/dev/dbos/transact/utils/OperationOutputRow.java rename to transact-jdbi-step-factory/src/test/java/dev/dbos/transact/utils/OperationOutputRow.java diff --git a/transact-jdbi-step-provider/src/test/java/dev/dbos/transact/utils/PgContainer.java b/transact-jdbi-step-factory/src/test/java/dev/dbos/transact/utils/PgContainer.java similarity index 100% rename from transact-jdbi-step-provider/src/test/java/dev/dbos/transact/utils/PgContainer.java rename to transact-jdbi-step-factory/src/test/java/dev/dbos/transact/utils/PgContainer.java diff --git a/transact-jdbi-step-provider/src/test/java/dev/dbos/transact/utils/TxStepOutputRow.java b/transact-jdbi-step-factory/src/test/java/dev/dbos/transact/utils/TxStepOutputRow.java similarity index 100% rename from transact-jdbi-step-provider/src/test/java/dev/dbos/transact/utils/TxStepOutputRow.java rename to transact-jdbi-step-factory/src/test/java/dev/dbos/transact/utils/TxStepOutputRow.java From 1cc68539cae7040a1a9ae7e18561798e0e603c7a Mon Sep 17 00:00:00 2001 From: Harry Pierson Date: Mon, 4 May 2026 14:08:12 -0700 Subject: [PATCH 09/29] factored common code into PostgresStepFactory --- .../dbos/transact/jdbi/JdbiStepFactory.java | 158 ++++--------- .../dbos/transact/txstep/JdbcStepFactory.java | 215 +++--------------- .../transact/txstep/PostgresStepFactory.java | 197 ++++++++++++++++ .../txstep/JdbcStepFactoryInitTest.java | 11 + 4 files changed, 285 insertions(+), 296 deletions(-) create mode 100644 transact/src/main/java/dev/dbos/transact/txstep/PostgresStepFactory.java diff --git a/transact-jdbi-step-factory/src/main/java/dev/dbos/transact/jdbi/JdbiStepFactory.java b/transact-jdbi-step-factory/src/main/java/dev/dbos/transact/jdbi/JdbiStepFactory.java index 0be95edd4..99790abec 100644 --- a/transact-jdbi-step-factory/src/main/java/dev/dbos/transact/jdbi/JdbiStepFactory.java +++ b/transact-jdbi-step-factory/src/main/java/dev/dbos/transact/jdbi/JdbiStepFactory.java @@ -1,31 +1,19 @@ package dev.dbos.transact.jdbi; import dev.dbos.transact.DBOS; -import dev.dbos.transact.database.SystemDatabase; -import dev.dbos.transact.execution.ThrowingConsumer; -import dev.dbos.transact.execution.ThrowingFunction; import dev.dbos.transact.json.DBOSSerializer; -import dev.dbos.transact.json.SerializationUtil; -import dev.dbos.transact.txstep.JdbcStepFactory; +import dev.dbos.transact.txstep.PostgresStepFactory; import dev.dbos.transact.workflow.internal.StepResult; import java.sql.SQLException; -import java.util.Objects; import org.jdbi.v3.core.Handle; import org.jdbi.v3.core.Jdbi; import org.jspecify.annotations.Nullable; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -public class JdbiStepFactory { +public class JdbiStepFactory extends PostgresStepFactory { - private static final Logger logger = LoggerFactory.getLogger(JdbiStepFactory.class); - - private final DBOS dbos; private final Jdbi jdbi; - private final String schema; - private final DBOSSerializer serializer; public JdbiStepFactory(DBOS dbos, Jdbi jdbi) { this(dbos, jdbi, null, null); @@ -40,38 +28,57 @@ public JdbiStepFactory(DBOS dbos, Jdbi jdbi, DBOSSerializer serializer) { } public JdbiStepFactory(DBOS dbos, Jdbi jdbi, String schema, DBOSSerializer serializer) { - this.dbos = Objects.requireNonNull(dbos); - this.jdbi = Objects.requireNonNull(jdbi); - var config = dbos.integration().config(); - this.schema = SystemDatabase.sanitizeSchema(schema == null ? config.databaseSchema() : schema); - this.serializer = serializer == null ? config.serializer() : serializer; - + super(dbos, schema, serializer); + this.jdbi = jdbi; try { jdbi.useHandle( handle -> { try { - JdbcStepFactory.ensureSchema(handle.getConnection(), this.schema); - JdbcStepFactory.ensureTxOutputTable(handle.getConnection(), this.schema); + PostgresStepFactory.ensurePostgres(handle.getConnection()); + PostgresStepFactory.ensureSchema(handle.getConnection(), this.schema); + PostgresStepFactory.ensureTxOutputTable(handle.getConnection(), this.schema); } catch (SQLException e) { throw new RuntimeException(e.getMessage(), e); } }); } catch (Exception e) { - if (e instanceof RuntimeException) { - throw (RuntimeException) e; + if (e instanceof RuntimeException re) { + throw re; } throw new RuntimeException(e); } } - private @Nullable StepResult checkExecution(String workflowId, int stepId, String stepName) { - var sql = - """ - SELECT output, error, serialization - FROM "%s".tx_step_outputs - WHERE workflow_id = ? AND step_id = ? - """ - .formatted(this.schema); + @Override + protected Handle openTransaction() { + var handle = jdbi.open(); + handle.begin(); + return handle; + } + + @Override + protected Handle openConnection() { + return jdbi.open(); + } + + @Override + protected void commit(Handle handle) { + handle.commit(); + } + + @Override + protected void rollback(Handle handle) { + handle.rollback(); + } + + @Override + protected void close(Handle handle) { + handle.close(); + } + + @Override + protected @Nullable StepResult checkExecution(String workflowId, int stepId, String stepName) { + var sql = CHECK_SQL_TEMPLATE.formatted(this.schema); return jdbi.withHandle( handle -> handle @@ -92,7 +99,8 @@ public JdbiStepFactory(DBOS dbos, Jdbi jdbi, String schema, DBOSSerializer seria .orElse(null)); } - private void recordResult( + @Override + protected void recordResult( Handle handle, String workflowId, int stepId, @@ -102,18 +110,8 @@ private void recordResult( if (output != null && error != null) { throw new IllegalArgumentException("attempted to record non null output and error result"); } - - String sql = - """ - INSERT INTO "%s".tx_step_outputs - (workflow_id, step_id, output, error, serialization) - VALUES (?, ?, ?, ?, ?) - ON CONFLICT DO NOTHING - """ - .formatted(schema); - handle - .createUpdate(sql) + .createUpdate(UPSERT_SQL_TEMPLATE.formatted(schema)) .bind(0, workflowId) .bind(1, stepId) .bind(2, output) @@ -121,76 +119,4 @@ private void recordResult( .bind(4, serialization) .execute(); } - - private void recordOutput(Handle handle, String workflowId, int stepId, R retVal) { - var value = SerializationUtil.serializeValue(retVal, null, serializer); - recordResult(handle, workflowId, stepId, value.serializedValue(), null, value.serialization()); - } - - private void recordError(String workflowId, int stepId, E exception) { - var value = SerializationUtil.serializeError(exception, null, serializer); - jdbi.useHandle( - handle -> - recordResult( - handle, workflowId, stepId, null, value.serializedValue(), value.serialization())); - } - - private R txStepInternal( - ThrowingFunction func, String stepName) throws E { - var workflowId = Objects.requireNonNull(DBOS.workflowId()); - int stepId = Objects.requireNonNull(DBOS.stepId()); - - logger.debug( - "txStep starting: workflowId={} stepId={} stepName={}", workflowId, stepId, stepName); - var prevResult = this.checkExecution(workflowId, stepId, stepName); - if (prevResult != null) { - logger.debug( - "txStep cache hit: workflowId={} stepId={} stepName={}", workflowId, stepId, stepName); - return prevResult.toResult(serializer); - } - - logger.debug( - "txStep executing: workflowId={} stepId={} stepName={}", workflowId, stepId, stepName); - var handle = jdbi.open(); - try { - handle.begin(); - var retVal = func.execute(handle); - recordOutput(handle, workflowId, stepId, retVal); - handle.commit(); - logger.debug( - "txStep succeeded: workflowId={} stepId={} stepName={}", workflowId, stepId, stepName); - return retVal; - } catch (Exception e) { - handle.rollback(); - recordError(workflowId, stepId, e); - logger.debug( - "txStep failed: workflowId={} stepId={} stepName={} error={}", - workflowId, - stepId, - stepName, - e.getMessage()); - throw e; - } finally { - handle.close(); - } - } - - public R txStep(ThrowingFunction func, String stepName) - throws E { - return dbos.runStep( - () -> { - return this.txStepInternal(func, stepName); - }, - stepName); - } - - public void txStep(ThrowingConsumer func, String stepName) - throws E { - txStep( - h -> { - func.execute(h); - return null; - }, - stepName); - } } diff --git a/transact/src/main/java/dev/dbos/transact/txstep/JdbcStepFactory.java b/transact/src/main/java/dev/dbos/transact/txstep/JdbcStepFactory.java index 6fc8ec4c8..1fd334a1c 100644 --- a/transact/src/main/java/dev/dbos/transact/txstep/JdbcStepFactory.java +++ b/transact/src/main/java/dev/dbos/transact/txstep/JdbcStepFactory.java @@ -1,11 +1,7 @@ package dev.dbos.transact.txstep; import dev.dbos.transact.DBOS; -import dev.dbos.transact.database.SystemDatabase; -import dev.dbos.transact.execution.ThrowingConsumer; -import dev.dbos.transact.execution.ThrowingFunction; import dev.dbos.transact.json.DBOSSerializer; -import dev.dbos.transact.json.SerializationUtil; import dev.dbos.transact.workflow.internal.StepResult; import java.sql.Connection; @@ -15,17 +11,10 @@ import javax.sql.DataSource; import org.jspecify.annotations.Nullable; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -public class JdbcStepFactory { +public class JdbcStepFactory extends PostgresStepFactory { - private static final Logger logger = LoggerFactory.getLogger(JdbcStepFactory.class); - - private final DBOS dbos; private final DataSource dataSource; - private final String schema; - private final DBOSSerializer serializer; public JdbcStepFactory(DBOS dbos, DataSource dataSource) { this(dbos, dataSource, null, null); @@ -41,17 +30,14 @@ public JdbcStepFactory(DBOS dbos, DataSource dataSource, DBOSSerializer serializ public JdbcStepFactory( DBOS dbos, DataSource dataSource, String schema, DBOSSerializer serializer) { - this.dbos = Objects.requireNonNull(dbos); + super(dbos, schema, serializer); this.dataSource = Objects.requireNonNull(dataSource); - var config = dbos.integration().config(); - this.schema = SystemDatabase.sanitizeSchema(schema == null ? config.databaseSchema() : schema); - this.serializer = serializer == null ? config.serializer() : serializer; - createTxOutputTable(dataSource, this.schema); } public static void createTxOutputTable(DataSource dataSource, String schema) { try (var conn = dataSource.getConnection()) { + ensurePostgres(conn); ensureSchema(conn, schema); ensureTxOutputTable(conn, schema); } catch (SQLException e) { @@ -59,72 +45,14 @@ public static void createTxOutputTable(DataSource dataSource, String schema) { } } - public static boolean schemaExists(Connection conn, String schema) throws SQLException { - var sql = "SELECT schema_name FROM information_schema.schemata WHERE schema_name = ?"; - try (var stmt = conn.prepareStatement(sql)) { - stmt.setString(1, Objects.requireNonNull(schema, "schema must not be null")); - try (var rs = stmt.executeQuery()) { - if (rs.next()) { - return true; - } else { - return false; - } - } - } - } - - public static void ensureSchema(Connection conn, String schema) throws SQLException { - Objects.requireNonNull(schema, "schema must not be null"); - if (!schemaExists(conn, schema)) { - try (var stmt = conn.createStatement()) { - stmt.execute("CREATE SCHEMA IF NOT EXISTS \"%s\"".formatted(schema)); - } - } - } - - public static boolean tableExists(Connection conn, String schema) throws SQLException { - var sql = "SELECT 1 FROM information_schema.tables WHERE table_schema = ? AND table_name = ?"; - try (var stmt = conn.prepareStatement(sql)) { - stmt.setString(1, Objects.requireNonNull(schema, "schema must not be null")); - stmt.setString(2, Objects.requireNonNull("tx_step_outputs", "tableName must not be null")); - try (var rs = stmt.executeQuery()) { - return rs.next(); - } - } - } - - public static void ensureTxOutputTable(Connection conn, String schema) throws SQLException { - Objects.requireNonNull(schema, "schema must not be null"); - if (tableExists(conn, schema)) { - return; - } - logger.debug("Creating tx_step_outputs table in schema={}", schema); - try (var stmt = conn.createStatement()) { - var ddlSql = - """ - CREATE TABLE IF NOT EXISTS "%1$s".tx_step_outputs ( - workflow_id TEXT NOT NULL, - step_id INT NOT NULL, - output TEXT, - error TEXT, - serialization TEXT, - created_at BIGINT NOT NULL DEFAULT (EXTRACT(EPOCH FROM now())*1000)::bigint, - PRIMARY KEY (workflow_id, step_id) - )""" - .formatted(schema); - stmt.execute(ddlSql); - } - } - private static class DBOSSqlException extends RuntimeException { public DBOSSqlException(SQLException wrappedException) { super(wrappedException.getMessage(), wrappedException); } } - // helper methods that wrap SQLExceptions in WrappedSqlException to distinguish from app - // exceptions - private Connection getConnection() { + @Override + protected Connection openTransaction() { try { var conn = dataSource.getConnection(); conn.setAutoCommit(false); @@ -134,7 +62,17 @@ private Connection getConnection() { } } - private static void commit(Connection conn) { + @Override + protected Connection openConnection() { + try { + return dataSource.getConnection(); + } catch (SQLException e) { + throw new DBOSSqlException(e); + } + } + + @Override + protected void commit(Connection conn) { try { conn.commit(); } catch (SQLException e) { @@ -142,7 +80,8 @@ private static void commit(Connection conn) { } } - private static void rollback(Connection conn) { + @Override + protected void rollback(Connection conn) { try { conn.rollback(); } catch (SQLException e) { @@ -150,7 +89,8 @@ private static void rollback(Connection conn) { } } - private static void close(Connection conn) { + @Override + protected void close(Connection conn) { try { conn.close(); } catch (SQLException e) { @@ -158,36 +98,33 @@ private static void close(Connection conn) { } } - private @Nullable StepResult checkExecution(String workflowId, int stepId, String stepName) { - var sql = - """ - SELECT output, error, serialization - FROM "%s".tx_step_outputs - WHERE workflow_id = ? AND step_id = ? - """ - .formatted(this.schema); + @Override + protected @Nullable StepResult checkExecution(String workflowId, int stepId, String stepName) { + var sql = CHECK_SQL_TEMPLATE.formatted(this.schema); try (var conn = dataSource.getConnection(); var stmt = conn.prepareStatement(sql)) { stmt.setString(1, workflowId); stmt.setInt(2, stepId); try (var rs = stmt.executeQuery()) { if (rs.next()) { - var output = rs.getString("output"); - var error = rs.getString("error"); - var serialization = rs.getString("serialization"); - var result = - new StepResult(workflowId, stepId, stepName, output, error, null, serialization); - return result; - } else { - return null; + return new StepResult( + workflowId, + stepId, + stepName, + rs.getString("output"), + rs.getString("error"), + null, + rs.getString("serialization")); } + return null; } } catch (SQLException e) { throw new DBOSSqlException(e); } } - private void recordResult( + @Override + protected void recordResult( Connection conn, String workflowId, int stepId, @@ -197,16 +134,7 @@ private void recordResult( if (output != null && error != null) { throw new IllegalArgumentException("attempted to record non null output and error result"); } - - String sql = - """ - INSERT INTO "%s".tx_step_outputs - (workflow_id, step_id, output, error, serialization) - VALUES (?, ?, ?, ?, ?) - ON CONFLICT DO NOTHING - """ - .formatted(schema); - + var sql = UPSERT_SQL_TEMPLATE.formatted(schema); try (var stmt = conn.prepareStatement(sql)) { stmt.setString(1, workflowId); stmt.setInt(2, stepId); @@ -218,77 +146,4 @@ private void recordResult( throw new DBOSSqlException(e); } } - - private void recordOutput(Connection conn, String workflowId, int stepId, R retVal) { - var value = SerializationUtil.serializeValue(retVal, null, serializer); - recordResult(conn, workflowId, stepId, value.serializedValue(), null, value.serialization()); - } - - private void recordError(String workflowId, int stepId, E exception) { - var value = SerializationUtil.serializeError(exception, null, serializer); - var conn = getConnection(); - try { - recordResult(conn, workflowId, stepId, null, value.serializedValue(), value.serialization()); - } finally { - close(conn); - } - } - - private R txStepInternal( - ThrowingFunction func, String stepName) throws E { - var workflowId = Objects.requireNonNull(DBOS.workflowId()); - int stepId = Objects.requireNonNull(DBOS.stepId()); - - logger.debug( - "txStep starting: workflowId={} stepId={} stepName={}", workflowId, stepId, stepName); - var prevResult = this.checkExecution(workflowId, stepId, stepName); - if (prevResult != null) { - logger.debug( - "txStep cache hit: workflowId={} stepId={} stepName={}", workflowId, stepId, stepName); - return prevResult.toResult(serializer); - } - - logger.debug( - "txStep executing: workflowId={} stepId={} stepName={}", workflowId, stepId, stepName); - var conn = getConnection(); - try { - var retVal = func.execute(conn); - recordOutput(conn, workflowId, stepId, retVal); - commit(conn); - logger.debug( - "txStep succeeded: workflowId={} stepId={} stepName={}", workflowId, stepId, stepName); - return retVal; - } catch (Exception e) { - rollback(conn); - recordError(workflowId, stepId, e); - logger.debug( - "txStep failed: workflowId={} stepId={} stepName={} error={}", - workflowId, - stepId, - stepName, - e.getMessage()); - throw e; - } finally { - close(conn); - } - } - - public R txStep(ThrowingFunction func, String stepName) - throws E { - return dbos.runStep( - () -> { - return this.txStepInternal(func, stepName); - }, - stepName); - } - - public void txStep(ThrowingConsumer func, String stepName) - throws E { - txStep( - c -> { - func.execute(c); - return null; - }, - stepName); - } } diff --git a/transact/src/main/java/dev/dbos/transact/txstep/PostgresStepFactory.java b/transact/src/main/java/dev/dbos/transact/txstep/PostgresStepFactory.java new file mode 100644 index 000000000..184e03317 --- /dev/null +++ b/transact/src/main/java/dev/dbos/transact/txstep/PostgresStepFactory.java @@ -0,0 +1,197 @@ +package dev.dbos.transact.txstep; + +import dev.dbos.transact.DBOS; +import dev.dbos.transact.database.SystemDatabase; +import dev.dbos.transact.execution.ThrowingConsumer; +import dev.dbos.transact.execution.ThrowingFunction; +import dev.dbos.transact.json.DBOSSerializer; +import dev.dbos.transact.json.SerializationUtil; +import dev.dbos.transact.workflow.internal.StepResult; + +import java.sql.Connection; +import java.sql.SQLException; +import java.util.Objects; + +import org.jspecify.annotations.Nullable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public abstract class PostgresStepFactory { + + private static final Logger DDL_LOGGER = LoggerFactory.getLogger(PostgresStepFactory.class); + private final Logger logger = LoggerFactory.getLogger(getClass()); + + protected final DBOS dbos; + protected final String schema; + protected final DBOSSerializer serializer; + + protected static final String CHECK_SQL_TEMPLATE = + """ + SELECT output, error, serialization + FROM "%s".tx_step_outputs + WHERE workflow_id = ? AND step_id = ? + """; + + protected static final String UPSERT_SQL_TEMPLATE = + """ + INSERT INTO "%s".tx_step_outputs + (workflow_id, step_id, output, error, serialization) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT DO NOTHING + """; + + public static void ensurePostgres(Connection conn) throws SQLException { + var productName = conn.getMetaData().getDatabaseProductName(); + if (!productName.equalsIgnoreCase("PostgreSQL")) { + throw new IllegalArgumentException( + "PostgresStepFactory requires a PostgreSQL datasource, got: " + productName); + } + } + + public static boolean schemaExists(Connection conn, String schema) throws SQLException { + var sql = "SELECT schema_name FROM information_schema.schemata WHERE schema_name = ?"; + try (var stmt = conn.prepareStatement(sql)) { + stmt.setString(1, Objects.requireNonNull(schema, "schema must not be null")); + try (var rs = stmt.executeQuery()) { + return rs.next(); + } + } + } + + public static void ensureSchema(Connection conn, String schema) throws SQLException { + Objects.requireNonNull(schema, "schema must not be null"); + if (!schemaExists(conn, schema)) { + try (var stmt = conn.createStatement()) { + stmt.execute("CREATE SCHEMA IF NOT EXISTS \"%s\"".formatted(schema)); + } + } + } + + public static boolean tableExists(Connection conn, String schema) throws SQLException { + var sql = "SELECT 1 FROM information_schema.tables WHERE table_schema = ? AND table_name = ?"; + try (var stmt = conn.prepareStatement(sql)) { + stmt.setString(1, Objects.requireNonNull(schema, "schema must not be null")); + stmt.setString(2, Objects.requireNonNull("tx_step_outputs", "tableName must not be null")); + try (var rs = stmt.executeQuery()) { + return rs.next(); + } + } + } + + public static void ensureTxOutputTable(Connection conn, String schema) throws SQLException { + Objects.requireNonNull(schema, "schema must not be null"); + if (tableExists(conn, schema)) { + return; + } + DDL_LOGGER.debug("Creating tx_step_outputs table in schema={}", schema); + try (var stmt = conn.createStatement()) { + var ddlSql = + """ + CREATE TABLE IF NOT EXISTS "%1$s".tx_step_outputs ( + workflow_id TEXT NOT NULL, + step_id INT NOT NULL, + output TEXT, + error TEXT, + serialization TEXT, + created_at BIGINT NOT NULL DEFAULT (EXTRACT(EPOCH FROM now())*1000)::bigint, + PRIMARY KEY (workflow_id, step_id) + )""" + .formatted(schema); + stmt.execute(ddlSql); + } + } + + protected PostgresStepFactory( + DBOS dbos, @Nullable String rawSchema, @Nullable DBOSSerializer rawSerializer) { + this.dbos = Objects.requireNonNull(dbos); + var config = dbos.integration().config(); + this.schema = + SystemDatabase.sanitizeSchema(rawSchema == null ? config.databaseSchema() : rawSchema); + this.serializer = rawSerializer == null ? config.serializer() : rawSerializer; + } + + protected abstract C openTransaction(); + + protected abstract C openConnection(); + + protected abstract void commit(C conn); + + protected abstract void rollback(C conn); + + protected abstract void close(C conn); + + protected abstract @Nullable StepResult checkExecution( + String workflowId, int stepId, String stepName); + + protected abstract void recordResult( + C conn, String workflowId, int stepId, String output, String error, String serialization); + + protected final void recordOutput(C conn, String workflowId, int stepId, R retVal) { + var value = SerializationUtil.serializeValue(retVal, null, serializer); + recordResult(conn, workflowId, stepId, value.serializedValue(), null, value.serialization()); + } + + protected final void recordError( + String workflowId, int stepId, E exception) { + var value = SerializationUtil.serializeError(exception, null, serializer); + var conn = openConnection(); + try { + recordResult(conn, workflowId, stepId, null, value.serializedValue(), value.serialization()); + } finally { + close(conn); + } + } + + protected final R txStepInternal( + ThrowingFunction func, String stepName) throws E { + var workflowId = Objects.requireNonNull(DBOS.workflowId()); + int stepId = Objects.requireNonNull(DBOS.stepId()); + + logger.debug( + "txStep starting: workflowId={} stepId={} stepName={}", workflowId, stepId, stepName); + var prevResult = this.checkExecution(workflowId, stepId, stepName); + if (prevResult != null) { + logger.debug( + "txStep cache hit: workflowId={} stepId={} stepName={}", workflowId, stepId, stepName); + return prevResult.toResult(serializer); + } + + logger.debug( + "txStep executing: workflowId={} stepId={} stepName={}", workflowId, stepId, stepName); + var conn = openTransaction(); + try { + var retVal = func.execute(conn); + recordOutput(conn, workflowId, stepId, retVal); + commit(conn); + logger.debug( + "txStep succeeded: workflowId={} stepId={} stepName={}", workflowId, stepId, stepName); + return retVal; + } catch (Exception e) { + rollback(conn); + recordError(workflowId, stepId, e); + logger.debug( + "txStep failed: workflowId={} stepId={} stepName={} error={}", + workflowId, + stepId, + stepName, + e.getMessage()); + throw e; + } finally { + close(conn); + } + } + + public R txStep(ThrowingFunction func, String stepName) + throws E { + return dbos.runStep(() -> txStepInternal(func, stepName), stepName); + } + + public void txStep(ThrowingConsumer func, String stepName) throws E { + txStep( + c -> { + func.execute(c); + return null; + }, + stepName); + } +} diff --git a/transact/src/test/java/dev/dbos/transact/txstep/JdbcStepFactoryInitTest.java b/transact/src/test/java/dev/dbos/transact/txstep/JdbcStepFactoryInitTest.java index 15794dae9..7458e8df2 100644 --- a/transact/src/test/java/dev/dbos/transact/txstep/JdbcStepFactoryInitTest.java +++ b/transact/src/test/java/dev/dbos/transact/txstep/JdbcStepFactoryInitTest.java @@ -2,6 +2,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import dev.dbos.transact.Constants; @@ -127,6 +128,16 @@ public void sameDbCustomDbosAndFactorySchema() throws Exception { } } + @Test + public void nonPostgresDataSource() throws Exception { + var config = pgContainer.dbosConfig(); + try (var dbos = new DBOS(config)) { + var sqliteDs = new org.sqlite.SQLiteDataSource(); + sqliteDs.setUrl("jdbc:sqlite::memory:"); + assertThrows(IllegalArgumentException.class, () -> new JdbcStepFactory(dbos, sqliteDs)); + } + } + @Test public void separateDBs() throws Exception { // create a 2nd database in the container's PG instance From 6dee4d561c6f660d16f07903e8ce6c7c27c42748 Mon Sep 17 00:00:00 2001 From: Harry Pierson Date: Mon, 4 May 2026 14:20:22 -0700 Subject: [PATCH 10/29] JooqStepFactory --- gradle/libs.versions.toml | 4 + settings.gradle.kts | 2 +- transact-jooq-step-factory/build.gradle.kts | 23 + .../dbos/transact/jooq/JooqStepFactory.java | 127 ++++++ .../transact/jooq/JooqStepFactoryTest.java | 417 ++++++++++++++++++ .../java/dev/dbos/transact/utils/DBUtils.java | 69 +++ .../transact/utils/OperationOutputRow.java | 27 ++ .../dev/dbos/transact/utils/PgContainer.java | 117 +++++ .../dbos/transact/utils/TxStepOutputRow.java | 23 + 9 files changed, 808 insertions(+), 1 deletion(-) create mode 100644 transact-jooq-step-factory/build.gradle.kts create mode 100644 transact-jooq-step-factory/src/main/java/dev/dbos/transact/jooq/JooqStepFactory.java create mode 100644 transact-jooq-step-factory/src/test/java/dev/dbos/transact/jooq/JooqStepFactoryTest.java create mode 100644 transact-jooq-step-factory/src/test/java/dev/dbos/transact/utils/DBUtils.java create mode 100644 transact-jooq-step-factory/src/test/java/dev/dbos/transact/utils/OperationOutputRow.java create mode 100644 transact-jooq-step-factory/src/test/java/dev/dbos/transact/utils/PgContainer.java create mode 100644 transact-jooq-step-factory/src/test/java/dev/dbos/transact/utils/TxStepOutputRow.java diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 0f256ee6d..4d3757127 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -5,6 +5,8 @@ assertj = "3.27.3" cron-utils = "9.2.1" hikaricp = "7.0.2" jdbi = "3.47.0" +jaxb-api = "4.0.2" +jooq = "3.19.15" kryo = "5.6.2" jackson = "2.21.2" java-websocket = "1.6.0" @@ -36,6 +38,8 @@ assertj-core = { module = "org.assertj:assertj-core", version.ref = "assertj" } cron-utils = { module = "com.cronutils:cron-utils", version.ref = "cron-utils" } hikaricp = { module = "com.zaxxer:HikariCP", version.ref = "hikaricp" } jdbi-core = { module = "org.jdbi:jdbi3-core", version.ref = "jdbi" } +jaxb-api = { module = "jakarta.xml.bind:jakarta.xml.bind-api", version.ref = "jaxb-api" } +jooq = { module = "org.jooq:jooq", version.ref = "jooq" } jackson-databind = { module = "com.fasterxml.jackson.core:jackson-databind", version.ref = "jackson" } jackson-jsr310 = { module = "com.fasterxml.jackson.datatype:jackson-datatype-jsr310", version.ref = "jackson" } java-websocket = { module = "org.java-websocket:Java-WebSocket", version.ref = "java-websocket" } diff --git a/settings.gradle.kts b/settings.gradle.kts index 6c95ecdcd..80ef2e6f5 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -1,6 +1,6 @@ rootProject.name = "dbos-transact-java" -include("transact", "transact-cli", "transact-spring-boot-starter", "transact-jdbi-step-factory") +include("transact", "transact-cli", "transact-spring-boot-starter", "transact-jdbi-step-factory", "transact-jooq-step-factory") plugins { id("org.gradle.toolchains.foojay-resolver") version "1.0.0" } diff --git a/transact-jooq-step-factory/build.gradle.kts b/transact-jooq-step-factory/build.gradle.kts new file mode 100644 index 000000000..e83777720 --- /dev/null +++ b/transact-jooq-step-factory/build.gradle.kts @@ -0,0 +1,23 @@ +plugins { id("java-library") } + +tasks.withType { + options.compilerArgs.add("-Xlint:unchecked") + options.compilerArgs.add("-Xlint:deprecation") + options.compilerArgs.add("-Xlint:rawtypes") + options.compilerArgs.add("-Werror") +} + +dependencies { + api(project(":transact")) + api(libs.jooq) + compileOnly(libs.jaxb.api) + + testImplementation(platform(libs.junit.bom)) + testImplementation(libs.junit.jupiter) + testRuntimeOnly(libs.junit.platform.launcher) + + testRuntimeOnly(libs.logback.classic) + testImplementation(libs.testcontainers.postgresql) + testImplementation(libs.postgresql) + testImplementation(libs.hikaricp) +} diff --git a/transact-jooq-step-factory/src/main/java/dev/dbos/transact/jooq/JooqStepFactory.java b/transact-jooq-step-factory/src/main/java/dev/dbos/transact/jooq/JooqStepFactory.java new file mode 100644 index 000000000..514f29660 --- /dev/null +++ b/transact-jooq-step-factory/src/main/java/dev/dbos/transact/jooq/JooqStepFactory.java @@ -0,0 +1,127 @@ +package dev.dbos.transact.jooq; + +import dev.dbos.transact.DBOS; +import dev.dbos.transact.json.DBOSSerializer; +import dev.dbos.transact.txstep.PostgresStepFactory; +import dev.dbos.transact.workflow.internal.StepResult; + +import java.sql.SQLException; + +import org.jooq.DSLContext; +import org.jooq.SQLDialect; +import org.jooq.impl.DSL; +import org.jspecify.annotations.Nullable; + +public class JooqStepFactory extends PostgresStepFactory { + + private final DSLContext dsl; + + public JooqStepFactory(DBOS dbos, DSLContext dsl) { + this(dbos, dsl, null, null); + } + + public JooqStepFactory(DBOS dbos, DSLContext dsl, String schema) { + this(dbos, dsl, schema, null); + } + + public JooqStepFactory(DBOS dbos, DSLContext dsl, DBOSSerializer serializer) { + this(dbos, dsl, null, serializer); + } + + public JooqStepFactory(DBOS dbos, DSLContext dsl, String schema, DBOSSerializer serializer) { + super(dbos, schema, serializer); + this.dsl = dsl; + try { + dsl.connection( + conn -> { + try { + ensurePostgres(conn); + ensureSchema(conn, this.schema); + ensureTxOutputTable(conn, this.schema); + } catch (SQLException e) { + throw new RuntimeException(e.getMessage(), e); + } + }); + } catch (Exception e) { + if (e instanceof RuntimeException re) { + throw re; + } + throw new RuntimeException(e); + } + } + + @Override + protected DSLContext openTransaction() { + try { + var conn = dsl.configuration().connectionProvider().acquire(); + conn.setAutoCommit(false); + return DSL.using(conn, SQLDialect.POSTGRES); + } catch (SQLException e) { + throw new RuntimeException(e.getMessage(), e); + } + } + + @Override + protected DSLContext openConnection() { + var conn = dsl.configuration().connectionProvider().acquire(); + return DSL.using(conn, SQLDialect.POSTGRES); + } + + @Override + protected void commit(DSLContext ctx) { + try { + ctx.connection(conn -> conn.commit()); + } catch (Exception e) { + throw new RuntimeException(e.getMessage(), e); + } + } + + @Override + protected void rollback(DSLContext ctx) { + try { + ctx.connection(conn -> conn.rollback()); + } catch (Exception e) { + throw new RuntimeException(e.getMessage(), e); + } + } + + @Override + protected void close(DSLContext ctx) { + try { + ctx.connection(conn -> conn.close()); + } catch (Exception e) { + throw new RuntimeException(e.getMessage(), e); + } + } + + @Override + protected @Nullable StepResult checkExecution(String workflowId, int stepId, String stepName) { + var sql = CHECK_SQL_TEMPLATE.formatted(this.schema); + return dsl.fetchOptional(sql, workflowId, stepId) + .map( + r -> + new StepResult( + workflowId, + stepId, + stepName, + r.get("output", String.class), + r.get("error", String.class), + null, + r.get("serialization", String.class))) + .orElse(null); + } + + @Override + protected void recordResult( + DSLContext ctx, + String workflowId, + int stepId, + String output, + String error, + String serialization) { + if (output != null && error != null) { + throw new IllegalArgumentException("attempted to record non null output and error result"); + } + ctx.execute(UPSERT_SQL_TEMPLATE.formatted(schema), workflowId, stepId, output, error, serialization); + } +} diff --git a/transact-jooq-step-factory/src/test/java/dev/dbos/transact/jooq/JooqStepFactoryTest.java b/transact-jooq-step-factory/src/test/java/dev/dbos/transact/jooq/JooqStepFactoryTest.java new file mode 100644 index 000000000..37ebd239a --- /dev/null +++ b/transact-jooq-step-factory/src/test/java/dev/dbos/transact/jooq/JooqStepFactoryTest.java @@ -0,0 +1,417 @@ +package dev.dbos.transact.jooq; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import dev.dbos.transact.DBOS; +import dev.dbos.transact.config.DBOSConfig; +import dev.dbos.transact.context.WorkflowOptions; +import dev.dbos.transact.json.SerializationUtil; +import dev.dbos.transact.utils.DBUtils; +import dev.dbos.transact.utils.PgContainer; +import dev.dbos.transact.workflow.Workflow; +import dev.dbos.transact.workflow.WorkflowHandle; + +import java.sql.SQLException; +import java.util.Objects; + +import com.zaxxer.hikari.HikariDataSource; +import org.jooq.DSLContext; +import org.jooq.SQLDialect; +import org.jooq.impl.DSL; +import org.junit.jupiter.api.AutoClose; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +interface FactoryTestService { + record TestResult(String user, int greetCount) {} + + TestResult insertWorkflow(String user); + + TestResult errorWorkflow(String user); + + TestResult readWorkflow(String user); + + TestResult insertThenReadWorkflow(String user); +} + +class FactoryTestServiceImpl implements FactoryTestService { + + private final JooqStepFactory stepFactory; + + public FactoryTestServiceImpl(JooqStepFactory stepFactory) { + this.stepFactory = stepFactory; + } + + FactoryTestService.TestResult insertGreeting(DSLContext ctx, String user) { + var sql = + """ + INSERT INTO greetings(name, greet_count) + VALUES (?, 1) + ON CONFLICT(name) + DO UPDATE SET greet_count = greetings.greet_count + 1 + RETURNING greet_count + """; + var record = ctx.fetchOne(sql, Objects.requireNonNull(user)); + int greetCount = record != null ? record.get("greet_count", Integer.class) : 0; + return new FactoryTestService.TestResult(user, greetCount); + } + + FactoryTestService.TestResult errorGreeting(DSLContext ctx, String user) { + insertGreeting(ctx, user); + throw new RuntimeException("Test Exception %d".formatted(System.currentTimeMillis())); + } + + FactoryTestService.TestResult readGreeting(DSLContext ctx, String user) { + var sql = "SELECT greet_count FROM greetings WHERE name = ?"; + var record = ctx.fetchOne(sql, Objects.requireNonNull(user)); + int greetCount = record != null ? record.get("greet_count", Integer.class) : 0; + return new FactoryTestService.TestResult(user, greetCount); + } + + @Override + @Workflow + public FactoryTestService.TestResult insertWorkflow(String user) { + return stepFactory.txStep((DSLContext ctx) -> insertGreeting(ctx, user), "insertGreeting"); + } + + @Override + @Workflow + public FactoryTestService.TestResult errorWorkflow(String user) { + return stepFactory.txStep((DSLContext ctx) -> errorGreeting(ctx, user), "errorGreeting"); + } + + @Override + @Workflow + public FactoryTestService.TestResult readWorkflow(String user) { + return stepFactory.txStep((DSLContext ctx) -> readGreeting(ctx, user), "readGreeting"); + } + + @Override + @Workflow + public FactoryTestService.TestResult insertThenReadWorkflow(String user) { + stepFactory.txStep((DSLContext ctx) -> insertGreeting(ctx, user), "insertGreeting"); + return stepFactory.txStep((DSLContext ctx) -> readGreeting(ctx, user), "readGreeting"); + } +} + +public class JooqStepFactoryTest { + @AutoClose final PgContainer pgContainer = new PgContainer(); + + DBOSConfig dbosConfig; + @AutoClose DBOS dbos; + @AutoClose HikariDataSource dataSource; + JooqStepFactory stepFactory; + FactoryTestService proxy; + FactoryTestServiceImpl impl; + + @BeforeEach + void beforeEach() throws SQLException { + dbosConfig = pgContainer.dbosConfig(); + dataSource = pgContainer.dataSource(); + + try (var conn = dataSource.getConnection(); + var stmt = conn.createStatement()) { + stmt.execute( + "CREATE TABLE greetings(name text NOT NULL, greet_count integer DEFAULT 0, PRIMARY KEY(name))"); + } + + dbos = new DBOS(dbosConfig); + DSLContext dsl = DSL.using(dataSource, SQLDialect.POSTGRES); + stepFactory = new JooqStepFactory(dbos, dsl); + + impl = new FactoryTestServiceImpl(stepFactory); + proxy = dbos.registerProxy(FactoryTestService.class, impl); + + dbos.launch(); + } + + private int getGreetCount(String user) throws SQLException { + var sql = "SELECT greet_count FROM greetings WHERE name = ?"; + try (var conn = dataSource.getConnection(); + var stmt = conn.prepareStatement(sql)) { + stmt.setString(1, user); + try (var rs = stmt.executeQuery()) { + return rs.next() ? rs.getInt("greet_count") : 0; + } + } + } + + @Test + public void testInsert() throws Exception { + var wfid = "wf1"; + var user = "testUser"; + try (var _o = new WorkflowOptions(wfid).setContext()) { + var result = proxy.insertWorkflow(user); + assertEquals(1, result.greetCount()); + assertEquals(user, result.user()); + } + + var rows = DBUtils.getTxStepRows(dataSource, wfid); + assertEquals(1, rows.size()); + var row = rows.get(0); + assertEquals(wfid, row.workflowId()); + assertEquals(0, row.stepId()); + assertNotNull(row.output()); + assertNull(row.error()); + assertEquals(SerializationUtil.NATIVE, row.serialization()); + var output = SerializationUtil.deserializeValue(row.output(), row.serialization(), null); + assertEquals(new FactoryTestService.TestResult(user, 1), output); + + assertEquals(1, getGreetCount(user)); + } + + @Test + public void testError() throws Exception { + var wfid = "wf1"; + var user = "testUser"; + try (var _o = new WorkflowOptions(wfid).setContext()) { + assertThrows(RuntimeException.class, () -> proxy.errorWorkflow(user)); + } + + // Transaction rolled back — no greeting inserted + var rows = DBUtils.getTxStepRows(dataSource, wfid); + assertEquals(1, rows.size()); + var row = rows.get(0); + assertEquals(wfid, row.workflowId()); + assertEquals(0, row.stepId()); + assertNull(row.output()); + assertNotNull(row.error()); + + assertEquals(0, getGreetCount(user)); + } + + @Test + public void testRead() throws Exception { + var insertWfid = "wf1"; + var readWfid = "wf2"; + var user = "testUser"; + + try (var _o = new WorkflowOptions(insertWfid).setContext()) { + proxy.insertWorkflow(user); + } + + try (var _o = new WorkflowOptions(readWfid).setContext()) { + var result = proxy.readWorkflow(user); + assertEquals(1, result.greetCount()); + assertEquals(user, result.user()); + } + + var rows = DBUtils.getTxStepRows(dataSource, readWfid); + assertEquals(1, rows.size()); + var row = rows.get(0); + assertEquals(readWfid, row.workflowId()); + assertEquals(0, row.stepId()); + assertNotNull(row.output()); + assertNull(row.error()); + assertEquals(SerializationUtil.NATIVE, row.serialization()); + var output = SerializationUtil.deserializeValue(row.output(), row.serialization(), null); + assertEquals(new FactoryTestService.TestResult(user, 1), output); + + assertEquals(1, getGreetCount(user)); + } + + @Test + public void testIdempotency() throws Exception { + var wfid = "wf1"; + var user = "testUser"; + + try (var _o = new WorkflowOptions(wfid).setContext()) { + var result = proxy.insertWorkflow(user); + assertEquals(1, result.greetCount()); + assertEquals(user, result.user()); + } + + // Second call with same wfid — txStep output is cached, insert not re-executed + try (var _o = new WorkflowOptions(wfid).setContext()) { + var result = proxy.insertWorkflow(user); + assertEquals(1, result.greetCount()); + assertEquals(user, result.user()); + } + + assertEquals(1, getGreetCount(user)); + assertEquals(1, DBUtils.getTxStepRows(dataSource, wfid).size()); + } + + @Test + public void testRetryError() throws Exception { + var wfid = "wf1"; + var user = "testUser"; + + try (var _o = new WorkflowOptions(wfid).setContext()) { + assertThrows(RuntimeException.class, () -> proxy.errorWorkflow(user)); + } + assertEquals(0, getGreetCount(user)); + dbos.close(); + + try (var conn = dataSource.getConnection(); + var stmt = + conn.prepareStatement("DELETE FROM dbos.operation_outputs WHERE workflow_uuid = ?")) { + stmt.setString(1, wfid); + stmt.executeUpdate(); + } + DBUtils.setWorkflowState(dataSource, wfid, "PENDING"); + + assertEquals(0, DBUtils.getStepRows(dataSource, wfid).size()); + assertEquals(1, DBUtils.getTxStepRows(dataSource, wfid).size()); + + dbos.launch(); + WorkflowHandle handle = + dbos.retrieveWorkflow(wfid); + assertThrows(RuntimeException.class, handle::getResult); + + // Cached error replayed — insert still not committed + assertEquals(0, getGreetCount(user)); + var txSteps = DBUtils.getTxStepRows(dataSource, wfid); + assertEquals(1, txSteps.size()); + assertNull(txSteps.get(0).output()); + assertNotNull(txSteps.get(0).error()); + } + + @Test + public void testMultipleTxSteps() throws Exception { + var wfid = "wf1"; + var user = "testUser"; + + try (var _o = new WorkflowOptions(wfid).setContext()) { + var result = proxy.insertThenReadWorkflow(user); + assertEquals(1, result.greetCount()); + assertEquals(user, result.user()); + } + + assertEquals(1, getGreetCount(user)); + + var rows = DBUtils.getTxStepRows(dataSource, wfid); + assertEquals(2, rows.size()); + assertEquals(0, rows.get(0).stepId()); + assertNotNull(rows.get(0).output()); + assertNull(rows.get(0).error()); + assertEquals(1, rows.get(1).stepId()); + assertNotNull(rows.get(1).output()); + assertNull(rows.get(1).error()); + } + + @Test + public void testDistinctWorkflows() throws Exception { + var wfid1 = "wf1"; + var wfid2 = "wf2"; + var user = "testUser"; + + try (var _o = new WorkflowOptions(wfid1).setContext()) { + var result = proxy.insertWorkflow(user); + assertEquals(1, result.greetCount()); + } + + try (var _o = new WorkflowOptions(wfid2).setContext()) { + var result = proxy.insertWorkflow(user); + assertEquals(2, result.greetCount()); + } + + assertEquals(2, getGreetCount(user)); + assertEquals(1, DBUtils.getTxStepRows(dataSource, wfid1).size()); + assertEquals(1, DBUtils.getTxStepRows(dataSource, wfid2).size()); + } + + @Test + public void testRetryPartialMultipleSteps() throws Exception { + var wfid = "wf1"; + var user = "testUser"; + + try (var _o = new WorkflowOptions(wfid).setContext()) { + var result = proxy.insertThenReadWorkflow(user); + assertEquals(1, result.greetCount()); + assertEquals(user, result.user()); + } + assertEquals(1, getGreetCount(user)); + dbos.close(); + + // Simulate crash after step 0 wrote tx_step_outputs but before step 1 ran: + // both operation_outputs rows are gone, and step 1 has no tx_step_outputs entry + try (var conn = dataSource.getConnection(); + var stmt = + conn.prepareStatement("DELETE FROM dbos.operation_outputs WHERE workflow_uuid = ?")) { + stmt.setString(1, wfid); + stmt.executeUpdate(); + } + try (var conn = dataSource.getConnection(); + var stmt = + conn.prepareStatement( + "DELETE FROM dbos.tx_step_outputs WHERE workflow_id = ? AND step_id = 1")) { + stmt.setString(1, wfid); + stmt.executeUpdate(); + } + DBUtils.setWorkflowState(dataSource, wfid, "PENDING"); + + assertEquals(0, DBUtils.getStepRows(dataSource, wfid).size()); + assertEquals(1, DBUtils.getTxStepRows(dataSource, wfid).size()); + + var relaunchTimestamp = System.currentTimeMillis(); + dbos.launch(); + WorkflowHandle handle = + dbos.retrieveWorkflow(wfid); + var result = (FactoryTestService.TestResult) handle.getResult(); + assertEquals(1, result.greetCount()); + assertEquals(user, result.user()); + + // Step 0 cache hit — insert not re-executed + assertEquals(1, getGreetCount(user)); + + var txSteps = DBUtils.getTxStepRows(dataSource, wfid); + assertEquals(2, txSteps.size()); + assertTrue(txSteps.get(0).createdAt() < relaunchTimestamp); // step 0: original run + assertTrue(txSteps.get(1).createdAt() >= relaunchTimestamp); // step 1: re-executed on retry + } + + @Test + public void testRetryInsert() throws Exception { + var timestamp = System.currentTimeMillis(); + + var wfid = "wf1"; + var user = "testUser"; + try (var _o = new WorkflowOptions(wfid).setContext()) { + var result = proxy.insertWorkflow(user); + assertEquals(1, result.greetCount()); + assertEquals(user, result.user()); + } + dbos.close(); + + try (var conn = dataSource.getConnection(); + var stmt = + conn.prepareStatement("DELETE FROM dbos.operation_outputs WHERE workflow_uuid = ?")) { + stmt.setString(1, wfid); + stmt.executeUpdate(); + } + DBUtils.setWorkflowState(dataSource, wfid, "PENDING"); + + assertEquals(0, DBUtils.getStepRows(dataSource, wfid).size()); + assertEquals(1, DBUtils.getTxStepRows(dataSource, wfid).size()); + + var relaunchTimestamp = System.currentTimeMillis(); + dbos.launch(); + var handle = dbos.retrieveWorkflow(wfid); + var result = (FactoryTestService.TestResult) handle.getResult(); + assertEquals(1, result.greetCount()); + assertEquals(user, result.user()); + + var steps = DBUtils.getStepRows(dataSource, wfid); + var txSteps = DBUtils.getTxStepRows(dataSource, wfid); + assertEquals(1, steps.size()); + assertEquals(1, txSteps.size()); + + var step = steps.get(0); + var txStep = txSteps.get(0); + assertEquals(step.output(), txStep.output()); + assertEquals(step.error(), txStep.error()); + + assertTrue(txStep.createdAt() < step.startedAt()); + assertTrue(timestamp < txStep.createdAt()); + assertTrue(txStep.createdAt() < relaunchTimestamp); + assertTrue(relaunchTimestamp < step.startedAt()); + + // Retry reads from tx_step_outputs cache — insert not re-executed + assertEquals(1, getGreetCount(user)); + } +} diff --git a/transact-jooq-step-factory/src/test/java/dev/dbos/transact/utils/DBUtils.java b/transact-jooq-step-factory/src/test/java/dev/dbos/transact/utils/DBUtils.java new file mode 100644 index 000000000..7f16faa15 --- /dev/null +++ b/transact-jooq-step-factory/src/test/java/dev/dbos/transact/utils/DBUtils.java @@ -0,0 +1,69 @@ +package dev.dbos.transact.utils; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import dev.dbos.transact.database.SystemDatabase; + +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; + +import javax.sql.DataSource; + +public class DBUtils { + + public static List getTxStepRows(DataSource ds, String workflowId) + throws SQLException { + var sql = + "SELECT * FROM \"%s\".tx_step_outputs WHERE workflow_id = ? ORDER BY step_id" + .formatted(SystemDatabase.sanitizeSchema(null)); + try (var conn = ds.getConnection(); + var stmt = conn.prepareStatement(sql)) { + stmt.setString(1, Objects.requireNonNull(workflowId)); + try (var rs = stmt.executeQuery()) { + List rows = new ArrayList<>(); + while (rs.next()) { + rows.add(new TxStepOutputRow(rs)); + } + return rows; + } + } + } + + public static List getStepRows(DataSource ds, String workflowId) + throws SQLException { + var sql = + "SELECT * FROM \"%s\".operation_outputs WHERE workflow_uuid = ? ORDER BY function_id" + .formatted(SystemDatabase.sanitizeSchema(null)); + try (var conn = ds.getConnection(); + var stmt = conn.prepareStatement(sql)) { + stmt.setString(1, workflowId); + try (var rs = stmt.executeQuery()) { + List rows = new ArrayList<>(); + while (rs.next()) { + rows.add(new OperationOutputRow(rs)); + } + return rows; + } + } + } + + public static void setWorkflowState(DataSource ds, String workflowId, String newState) + throws SQLException { + String sql = + "UPDATE dbos.workflow_status SET status = ?, updated_at = ? WHERE workflow_uuid = ?"; + + try (var connection = ds.getConnection(); + PreparedStatement pstmt = connection.prepareStatement(sql)) { + pstmt.setString(1, newState); + pstmt.setLong(2, Instant.now().toEpochMilli()); + pstmt.setString(3, workflowId); + + int rowsAffected = pstmt.executeUpdate(); + assertEquals(1, rowsAffected); + } + } +} diff --git a/transact-jooq-step-factory/src/test/java/dev/dbos/transact/utils/OperationOutputRow.java b/transact-jooq-step-factory/src/test/java/dev/dbos/transact/utils/OperationOutputRow.java new file mode 100644 index 000000000..c1ac2938b --- /dev/null +++ b/transact-jooq-step-factory/src/test/java/dev/dbos/transact/utils/OperationOutputRow.java @@ -0,0 +1,27 @@ +package dev.dbos.transact.utils; + +import java.sql.ResultSet; +import java.sql.SQLException; + +public record OperationOutputRow( + String workflowId, + int functionId, + String output, + String error, + String functionName, + String childWorkflowId, + Long startedAt, + Long completedAt) { + + public OperationOutputRow(ResultSet rs) throws SQLException { + this( + rs.getString("workflow_uuid"), + rs.getInt("function_id"), + rs.getString("output"), + rs.getString("error"), + rs.getString("function_name"), + rs.getString("child_workflow_id"), + rs.getObject("started_at_epoch_ms", Long.class), + rs.getObject("completed_at_epoch_ms", Long.class)); + } +} diff --git a/transact-jooq-step-factory/src/test/java/dev/dbos/transact/utils/PgContainer.java b/transact-jooq-step-factory/src/test/java/dev/dbos/transact/utils/PgContainer.java new file mode 100644 index 000000000..2c1431dd5 --- /dev/null +++ b/transact-jooq-step-factory/src/test/java/dev/dbos/transact/utils/PgContainer.java @@ -0,0 +1,117 @@ +package dev.dbos.transact.utils; + +import dev.dbos.transact.DBOSClient; +import dev.dbos.transact.config.DBOSConfig; +import dev.dbos.transact.database.SystemDatabase; + +import java.sql.DriverManager; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Objects; +import java.util.UUID; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.Semaphore; + +import com.zaxxer.hikari.HikariDataSource; +import org.testcontainers.postgresql.PostgreSQLContainer; + +public class PgContainer implements AutoCloseable { + + private static final int SIZE = Runtime.getRuntime().availableProcessors(); + private static final BlockingQueue POOL = new ArrayBlockingQueue<>(SIZE); + private static final Semaphore PERMITS = new Semaphore(SIZE); + + static { + Runtime.getRuntime() + .addShutdownHook( + new Thread( + () -> { + var containers = new ArrayList(); + POOL.drainTo(containers); + containers.forEach(PostgreSQLContainer::stop); + })); + } + + static PostgreSQLContainer acquire() { + try { + PERMITS.acquire(); + var container = POOL.poll(); + if (container == null) { + container = new PostgreSQLContainer("postgres:18"); + container.start(); + } + return container; + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + + static void release(PostgreSQLContainer c) { + POOL.offer(c); + PERMITS.release(); + } + + private final PostgreSQLContainer pgContainer; + private final String jdbcUrl; + private final String dbName; + + public PgContainer() { + // take a container from the pool and create a new database for it + pgContainer = acquire(); + dbName = "test_" + UUID.randomUUID().toString().replace("-", ""); + jdbcUrl = pgContainer.getJdbcUrl().replaceFirst("/[^/]+$", "/" + dbName); + + try (var conn = + DriverManager.getConnection( + pgContainer.getJdbcUrl(), pgContainer.getUsername(), pgContainer.getPassword()); + var stmt = conn.createStatement()) { + stmt.execute("CREATE DATABASE " + dbName); + } catch (SQLException e) { + throw new RuntimeException(e); + } + } + + @Override + public void close() throws Exception { + // drop the database we created and return the container too the pool + var _jdbcUrl = pgContainer.getJdbcUrl(); + try (var conn = DriverManager.getConnection(_jdbcUrl, username(), password()); + var stmt = conn.createStatement()) { + var sql = "DROP DATABASE IF EXISTS %s WITH (FORCE)".formatted(dbName); + stmt.execute(sql); + } + release(pgContainer); + } + + public String jdbcUrl() { + return jdbcUrl; + } + + public String username() { + return pgContainer.getUsername(); + } + + public String password() { + return pgContainer.getPassword(); + } + + public DBOSConfig dbosConfig() { + return dbosConfig(null); + } + + public DBOSConfig dbosConfig(String appName) { + return DBOSConfig.defaults(Objects.requireNonNullElse(appName, "transact-java-test")) + .withDatabaseUrl(jdbcUrl()) + .withDbUser(username()) + .withDbPassword(password()); + } + + public HikariDataSource dataSource() { + return SystemDatabase.createDataSource(jdbcUrl(), username(), password()); + } + + public DBOSClient dbosClient() { + return new DBOSClient(jdbcUrl(), username(), password()); + } +} diff --git a/transact-jooq-step-factory/src/test/java/dev/dbos/transact/utils/TxStepOutputRow.java b/transact-jooq-step-factory/src/test/java/dev/dbos/transact/utils/TxStepOutputRow.java new file mode 100644 index 000000000..7472fe314 --- /dev/null +++ b/transact-jooq-step-factory/src/test/java/dev/dbos/transact/utils/TxStepOutputRow.java @@ -0,0 +1,23 @@ +package dev.dbos.transact.utils; + +import java.sql.ResultSet; +import java.sql.SQLException; + +public record TxStepOutputRow( + String workflowId, + int stepId, + String output, + String error, + String serialization, + Long createdAt) { + + public TxStepOutputRow(ResultSet rs) throws SQLException { + this( + rs.getString("workflow_id"), + rs.getInt("step_id"), + rs.getString("output"), + rs.getString("error"), + rs.getString("serialization"), + rs.getObject("created_at", Long.class)); + } +} From aa2ce51873dd78a6325348a2a842493c08c77c67 Mon Sep 17 00:00:00 2001 From: Harry Pierson Date: Mon, 4 May 2026 14:20:48 -0700 Subject: [PATCH 11/29] spotless --- settings.gradle.kts | 8 +++++++- .../main/java/dev/dbos/transact/jooq/JooqStepFactory.java | 3 ++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/settings.gradle.kts b/settings.gradle.kts index 80ef2e6f5..af6f2a875 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -1,6 +1,12 @@ rootProject.name = "dbos-transact-java" -include("transact", "transact-cli", "transact-spring-boot-starter", "transact-jdbi-step-factory", "transact-jooq-step-factory") +include( + "transact", + "transact-cli", + "transact-spring-boot-starter", + "transact-jdbi-step-factory", + "transact-jooq-step-factory", +) plugins { id("org.gradle.toolchains.foojay-resolver") version "1.0.0" } diff --git a/transact-jooq-step-factory/src/main/java/dev/dbos/transact/jooq/JooqStepFactory.java b/transact-jooq-step-factory/src/main/java/dev/dbos/transact/jooq/JooqStepFactory.java index 514f29660..b4740aa99 100644 --- a/transact-jooq-step-factory/src/main/java/dev/dbos/transact/jooq/JooqStepFactory.java +++ b/transact-jooq-step-factory/src/main/java/dev/dbos/transact/jooq/JooqStepFactory.java @@ -122,6 +122,7 @@ protected void recordResult( if (output != null && error != null) { throw new IllegalArgumentException("attempted to record non null output and error result"); } - ctx.execute(UPSERT_SQL_TEMPLATE.formatted(schema), workflowId, stepId, output, error, serialization); + ctx.execute( + UPSERT_SQL_TEMPLATE.formatted(schema), workflowId, stepId, output, error, serialization); } } From 9312e45e8d6e7915fbabc777d3be56b24d724e69 Mon Sep 17 00:00:00 2001 From: Harry Pierson Date: Mon, 4 May 2026 17:15:34 -0700 Subject: [PATCH 12/29] java docs --- gradle/libs.versions.toml | 18 +-- .../dbos/transact/jdbi/JdbiStepFactory.java | 21 ++++ .../dbos/transact/jooq/JooqStepFactory.java | 23 ++++ .../dbos/transact/txstep/JdbcStepFactory.java | 26 ++++ .../transact/txstep/PostgresStepFactory.java | 119 +++++++++++++++--- 5 files changed, 184 insertions(+), 23 deletions(-) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 4d3757127..fdfda10ef 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -4,16 +4,16 @@ aspectj = "1.9.22.1" assertj = "3.27.3" cron-utils = "9.2.1" hikaricp = "7.0.2" -jdbi = "3.47.0" -jaxb-api = "4.0.2" -jooq = "3.19.15" -kryo = "5.6.2" jackson = "2.21.2" java-websocket = "1.6.0" +jaxb-api = "4.0.2" +jdbi = "3.47.0" +jooq = "3.19.15" jspecify = "1.0.0" junit = "6.0.3" junit-pioneer = "2.3.0" kotlin = "2.3.10" +kryo = "5.6.2" logback = "1.5.32" maven-artifact = "3.9.13" maven-publish = "0.36.0" @@ -23,10 +23,10 @@ postgresql = "42.7.10" rest-assured = "6.0.0" shadow = "9.4.1" slf4j = "2.0.17" -sqlite-jdbc = "3.49.1.0" spotless = "8.4.0" spring-boot = "3.4.4" spring-framework = "6.2.5" +sqlite-jdbc = "3.49.1.0" system-stubs = "2.1.8" testcontainers = "2.0.4" versions = "0.53.0" @@ -37,12 +37,12 @@ aspectjweaver = { module = "org.aspectj:aspectjweaver", version.ref = "aspectj" assertj-core = { module = "org.assertj:assertj-core", version.ref = "assertj" } cron-utils = { module = "com.cronutils:cron-utils", version.ref = "cron-utils" } hikaricp = { module = "com.zaxxer:HikariCP", version.ref = "hikaricp" } -jdbi-core = { module = "org.jdbi:jdbi3-core", version.ref = "jdbi" } -jaxb-api = { module = "jakarta.xml.bind:jakarta.xml.bind-api", version.ref = "jaxb-api" } -jooq = { module = "org.jooq:jooq", version.ref = "jooq" } jackson-databind = { module = "com.fasterxml.jackson.core:jackson-databind", version.ref = "jackson" } jackson-jsr310 = { module = "com.fasterxml.jackson.datatype:jackson-datatype-jsr310", version.ref = "jackson" } java-websocket = { module = "org.java-websocket:Java-WebSocket", version.ref = "java-websocket" } +jaxb-api = { module = "jakarta.xml.bind:jakarta.xml.bind-api", version.ref = "jaxb-api" } +jdbi-core = { module = "org.jdbi:jdbi3-core", version.ref = "jdbi" } +jooq = { module = "org.jooq:jooq", version.ref = "jooq" } jspecify = { module = "org.jspecify:jspecify", version.ref = "jspecify" } junit-bom = { module = "org.junit:junit-bom", version.ref = "junit" } junit-jupiter = { module = "org.junit.jupiter:junit-jupiter" } @@ -57,11 +57,11 @@ postgresql = { module = "org.postgresql:postgresql", version.ref = "postgresql" rest-assured = { module = "io.rest-assured:rest-assured", version.ref = "rest-assured" } slf4j-api = { module = "org.slf4j:slf4j-api", version.ref = "slf4j" } slf4j-simple = { module = "org.slf4j:slf4j-simple", version.ref = "slf4j" } -sqlite-jdbc = { module = "org.xerial:sqlite-jdbc", version.ref = "sqlite-jdbc" } spring-aop = { module = "org.springframework:spring-aop", version.ref = "spring-framework" } spring-boot-autoconfigure = { module = "org.springframework.boot:spring-boot-autoconfigure", version.ref = "spring-boot" } spring-boot-configuration-processor = { module = "org.springframework.boot:spring-boot-configuration-processor", version.ref = "spring-boot" } spring-boot-test = { module = "org.springframework.boot:spring-boot-test", version.ref = "spring-boot" } +sqlite-jdbc = { module = "org.xerial:sqlite-jdbc", version.ref = "sqlite-jdbc" } system-stubs-jupiter = { module = "uk.org.webcompere:system-stubs-jupiter", version.ref = "system-stubs" } testcontainers-postgresql = { module = "org.testcontainers:testcontainers-postgresql", version.ref = "testcontainers" } diff --git a/transact-jdbi-step-factory/src/main/java/dev/dbos/transact/jdbi/JdbiStepFactory.java b/transact-jdbi-step-factory/src/main/java/dev/dbos/transact/jdbi/JdbiStepFactory.java index 99790abec..ff1782677 100644 --- a/transact-jdbi-step-factory/src/main/java/dev/dbos/transact/jdbi/JdbiStepFactory.java +++ b/transact-jdbi-step-factory/src/main/java/dev/dbos/transact/jdbi/JdbiStepFactory.java @@ -11,22 +11,43 @@ import org.jdbi.v3.core.Jdbi; import org.jspecify.annotations.Nullable; +/** + * A {@link PostgresStepFactory} implementation backed by Jdbi3 {@link Handle} objects. + * + *

Construct one with a {@link Jdbi} instance pointing at a PostgreSQL database. The constructor + * verifies the datasource is PostgreSQL and creates the {@code tx_step_outputs} table if needed. + * User lambdas passed to {@code txStep} receive a {@link Handle} with a transaction already + * started; they should not call {@code commit} or {@code close} themselves. + * + * 

{@code
+ * JdbiStepFactory factory = new JdbiStepFactory(dbos, Jdbi.create(dataSource));
+ *
+ * // inside a @Workflow method:
+ * int count = factory.txStep(handle -> {
+ *     return handle.createUpdate("INSERT INTO ...").execute();
+ * }, "myStep");
+ * }
+ */ public class JdbiStepFactory extends PostgresStepFactory { private final Jdbi jdbi; + /** Creates a factory using the schema from the DBOS config. */ public JdbiStepFactory(DBOS dbos, Jdbi jdbi) { this(dbos, jdbi, null, null); } + /** Creates a factory using a custom schema for {@code tx_step_outputs}. */ public JdbiStepFactory(DBOS dbos, Jdbi jdbi, String schema) { this(dbos, jdbi, schema, null); } + /** Creates a factory using a custom serializer. */ public JdbiStepFactory(DBOS dbos, Jdbi jdbi, DBOSSerializer serializer) { this(dbos, jdbi, null, serializer); } + /** Creates a factory with a custom schema and serializer. */ public JdbiStepFactory(DBOS dbos, Jdbi jdbi, String schema, DBOSSerializer serializer) { super(dbos, schema, serializer); this.jdbi = jdbi; diff --git a/transact-jooq-step-factory/src/main/java/dev/dbos/transact/jooq/JooqStepFactory.java b/transact-jooq-step-factory/src/main/java/dev/dbos/transact/jooq/JooqStepFactory.java index b4740aa99..afb7da358 100644 --- a/transact-jooq-step-factory/src/main/java/dev/dbos/transact/jooq/JooqStepFactory.java +++ b/transact-jooq-step-factory/src/main/java/dev/dbos/transact/jooq/JooqStepFactory.java @@ -12,22 +12,45 @@ import org.jooq.impl.DSL; import org.jspecify.annotations.Nullable; +/** + * A {@link PostgresStepFactory} implementation backed by jOOQ {@link DSLContext} objects. + * + *

Construct one with a pool-backed {@link DSLContext} pointing at a PostgreSQL database. The + * constructor verifies the datasource is PostgreSQL and creates the {@code tx_step_outputs} table + * if needed. User lambdas passed to {@code txStep} receive a per-transaction {@link DSLContext} + * backed by a single connection with a transaction already started; they should not call {@code + * commit} or {@code close} themselves. + * + * 

{@code
+ * DSLContext dsl = DSL.using(dataSource, SQLDialect.POSTGRES);
+ * JooqStepFactory factory = new JooqStepFactory(dbos, dsl);
+ *
+ * // inside a @Workflow method:
+ * int count = factory.txStep(ctx -> {
+ *     return ctx.execute("INSERT INTO ...");
+ * }, "myStep");
+ * }
+ */ public class JooqStepFactory extends PostgresStepFactory { private final DSLContext dsl; + /** Creates a factory using the schema from the DBOS config. */ public JooqStepFactory(DBOS dbos, DSLContext dsl) { this(dbos, dsl, null, null); } + /** Creates a factory using a custom schema for {@code tx_step_outputs}. */ public JooqStepFactory(DBOS dbos, DSLContext dsl, String schema) { this(dbos, dsl, schema, null); } + /** Creates a factory using a custom serializer. */ public JooqStepFactory(DBOS dbos, DSLContext dsl, DBOSSerializer serializer) { this(dbos, dsl, null, serializer); } + /** Creates a factory with a custom schema and serializer. */ public JooqStepFactory(DBOS dbos, DSLContext dsl, String schema, DBOSSerializer serializer) { super(dbos, schema, serializer); this.dsl = dsl; diff --git a/transact/src/main/java/dev/dbos/transact/txstep/JdbcStepFactory.java b/transact/src/main/java/dev/dbos/transact/txstep/JdbcStepFactory.java index 1fd334a1c..bba5fe6af 100644 --- a/transact/src/main/java/dev/dbos/transact/txstep/JdbcStepFactory.java +++ b/transact/src/main/java/dev/dbos/transact/txstep/JdbcStepFactory.java @@ -12,22 +12,44 @@ import org.jspecify.annotations.Nullable; +/** + * A {@link PostgresStepFactory} implementation backed by plain JDBC {@link Connection} objects. + * + *

Construct one with a {@link DataSource} pointing at a PostgreSQL database. The constructor + * verifies the datasource is PostgreSQL and creates the {@code tx_step_outputs} table if needed. + * User lambdas passed to {@code txStep} receive a {@link Connection} with a transaction already + * started; they should not call {@code commit} or {@code close} themselves. + * + * 

{@code
+ * JdbcStepFactory factory = new JdbcStepFactory(dbos, dataSource);
+ *
+ * // inside a @Workflow method:
+ * int count = factory.txStep(conn -> {
+ *     try (var stmt = conn.prepareStatement("INSERT INTO ...")) { ... }
+ *     return rowCount;
+ * }, "myStep");
+ * }
+ */ public class JdbcStepFactory extends PostgresStepFactory { private final DataSource dataSource; + /** Creates a factory using the schema from the DBOS config. */ public JdbcStepFactory(DBOS dbos, DataSource dataSource) { this(dbos, dataSource, null, null); } + /** Creates a factory using a custom schema for {@code tx_step_outputs}. */ public JdbcStepFactory(DBOS dbos, DataSource dataSource, String schema) { this(dbos, dataSource, schema, null); } + /** Creates a factory using a custom serializer. */ public JdbcStepFactory(DBOS dbos, DataSource dataSource, DBOSSerializer serializer) { this(dbos, dataSource, null, serializer); } + /** Creates a factory with a custom schema and serializer. */ public JdbcStepFactory( DBOS dbos, DataSource dataSource, String schema, DBOSSerializer serializer) { super(dbos, schema, serializer); @@ -35,6 +57,10 @@ public JdbcStepFactory( createTxOutputTable(dataSource, this.schema); } + /** + * Verifies the datasource is PostgreSQL and creates the {@code tx_step_outputs} table if it does + * not exist. Useful when the {@code DataSource} is managed separately from the factory lifecycle. + */ public static void createTxOutputTable(DataSource dataSource, String schema) { try (var conn = dataSource.getConnection()) { ensurePostgres(conn); diff --git a/transact/src/main/java/dev/dbos/transact/txstep/PostgresStepFactory.java b/transact/src/main/java/dev/dbos/transact/txstep/PostgresStepFactory.java index 184e03317..de81380e7 100644 --- a/transact/src/main/java/dev/dbos/transact/txstep/PostgresStepFactory.java +++ b/transact/src/main/java/dev/dbos/transact/txstep/PostgresStepFactory.java @@ -16,6 +16,27 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +/** + * Abstract base class for PostgreSQL-backed transactional step factories. + * + *

A step factory wraps user-provided database operations as durable DBOS workflow steps. Before + * executing a step, the factory checks {@code tx_step_outputs} for a prior result. If one exists, + * it is returned directly (idempotency / crash recovery). If not, the operation runs inside a + * database transaction: the result is written to {@code tx_step_outputs} atomically with the user's + * data, and the transaction is committed. On failure the transaction is rolled back and the error + * is recorded so it can be replayed on retry. + * + *

Subclasses provide the connection-management primitives ({@link #openTransaction()}, {@link + * #commit}, {@link #rollback}, {@link #close}) and SQL execution ({@link #checkExecution}, {@link + * #recordResult}) for a specific database access layer. The type parameter {@code C} represents the + * connection/session object that the user's lambda receives. + * + *

All implementations require a PostgreSQL datasource. Use {@link #ensurePostgres} during + * construction to fail fast if a non-PostgreSQL datasource is supplied. + * + * @param the connection type exposed to user lambdas (e.g. {@code Connection}, {@code Handle}, + * {@code DSLContext}) + */ public abstract class PostgresStepFactory { private static final Logger DDL_LOGGER = LoggerFactory.getLogger(PostgresStepFactory.class); @@ -40,6 +61,12 @@ public abstract class PostgresStepFactory { ON CONFLICT DO NOTHING """; + /** + * Verifies that the given connection is to a PostgreSQL database. + * + * @throws IllegalArgumentException if the database is not PostgreSQL + * @throws SQLException if database metadata cannot be read + */ public static void ensurePostgres(Connection conn) throws SQLException { var productName = conn.getMetaData().getDatabaseProductName(); if (!productName.equalsIgnoreCase("PostgreSQL")) { @@ -48,6 +75,11 @@ public static void ensurePostgres(Connection conn) throws SQLException { } } + /** + * Returns {@code true} if the named schema exists in the database. + * + * @throws SQLException if the query fails + */ public static boolean schemaExists(Connection conn, String schema) throws SQLException { var sql = "SELECT schema_name FROM information_schema.schemata WHERE schema_name = ?"; try (var stmt = conn.prepareStatement(sql)) { @@ -58,6 +90,11 @@ public static boolean schemaExists(Connection conn, String schema) throws SQLExc } } + /** + * Creates the named schema if it does not already exist. + * + * @throws SQLException if the DDL fails + */ public static void ensureSchema(Connection conn, String schema) throws SQLException { Objects.requireNonNull(schema, "schema must not be null"); if (!schemaExists(conn, schema)) { @@ -67,6 +104,11 @@ public static void ensureSchema(Connection conn, String schema) throws SQLExcept } } + /** + * Returns {@code true} if the {@code tx_step_outputs} table exists in the named schema. + * + * @throws SQLException if the query fails + */ public static boolean tableExists(Connection conn, String schema) throws SQLException { var sql = "SELECT 1 FROM information_schema.tables WHERE table_schema = ? AND table_name = ?"; try (var stmt = conn.prepareStatement(sql)) { @@ -78,6 +120,11 @@ public static boolean tableExists(Connection conn, String schema) throws SQLExce } } + /** + * Creates the {@code tx_step_outputs} table in the named schema if it does not already exist. + * + * @throws SQLException if the DDL fails + */ public static void ensureTxOutputTable(Connection conn, String schema) throws SQLException { Objects.requireNonNull(schema, "schema must not be null"); if (tableExists(conn, schema)) { @@ -101,6 +148,15 @@ PRIMARY KEY (workflow_id, step_id) } } + /** + * Constructs a factory for the given DBOS instance. + * + * @param dbos the DBOS runtime + * @param rawSchema the schema for {@code tx_step_outputs}, or {@code null} to use the schema from + * the DBOS config + * @param rawSerializer the serializer for step outputs, or {@code null} to use the serializer + * from the DBOS config + */ protected PostgresStepFactory( DBOS dbos, @Nullable String rawSchema, @Nullable DBOSSerializer rawSerializer) { this.dbos = Objects.requireNonNull(dbos); @@ -110,22 +166,71 @@ protected PostgresStepFactory( this.serializer = rawSerializer == null ? config.serializer() : rawSerializer; } + /** Opens a connection with a transaction already started (autoCommit off). */ protected abstract C openTransaction(); + /** Opens a connection without starting a transaction (used for recording errors). */ protected abstract C openConnection(); + /** Commits the transaction on the given connection. */ protected abstract void commit(C conn); + /** Rolls back the transaction on the given connection. */ protected abstract void rollback(C conn); + /** Closes and releases the given connection. */ protected abstract void close(C conn); + /** + * Checks {@code tx_step_outputs} for a prior result for the given workflow step. + * + * @return the prior result, or {@code null} if the step has not been executed + */ protected abstract @Nullable StepResult checkExecution( String workflowId, int stepId, String stepName); + /** + * Writes a step result to {@code tx_step_outputs} using the given connection. Exactly one of + * {@code output} and {@code error} must be non-null. + */ protected abstract void recordResult( C conn, String workflowId, int stepId, String output, String error, String serialization); + /** + * Executes a transactional step that returns a value. + * + *

On the first call for a given {@code (workflowId, stepId)}, the step function is invoked + * inside a database transaction. The return value is serialized and written to {@code + * tx_step_outputs} atomically with the user's data before the transaction commits. On subsequent + * calls with the same IDs (idempotency or crash recovery), the cached output is deserialized and + * returned without re-executing the function. + * + * @param func the database operation to run; receives the transactional connection object + * @param stepName a human-readable name for logging and DBOS step tracking + * @return the value returned by {@code func} + * @throws E if {@code func} throws, after rolling back the transaction and recording the error + */ + public R txStep(ThrowingFunction func, String stepName) + throws E { + return dbos.runStep(() -> txStepInternal(func, stepName), stepName); + } + + /** + * Executes a transactional step that returns no value. + * + * @param func the database operation to run; receives the transactional connection object + * @param stepName a human-readable name for logging and DBOS step tracking + * @throws E if {@code func} throws, after rolling back the transaction and recording the error + */ + public void txStep(ThrowingConsumer func, String stepName) throws E { + txStep( + c -> { + func.execute(c); + return null; + }, + stepName); + } + protected final void recordOutput(C conn, String workflowId, int stepId, R retVal) { var value = SerializationUtil.serializeValue(retVal, null, serializer); recordResult(conn, workflowId, stepId, value.serializedValue(), null, value.serialization()); @@ -180,18 +285,4 @@ protected final R txStepInternal( close(conn); } } - - public R txStep(ThrowingFunction func, String stepName) - throws E { - return dbos.runStep(() -> txStepInternal(func, stepName), stepName); - } - - public void txStep(ThrowingConsumer func, String stepName) throws E { - txStep( - c -> { - func.execute(c); - return null; - }, - stepName); - } } From 24ac5a05fd38854a66bb7cee10a26a7523b1dbc3 Mon Sep 17 00:00:00 2001 From: Harry Pierson Date: Tue, 5 May 2026 15:32:52 -0700 Subject: [PATCH 13/29] update JdbiStepFactory --- .../dbos/transact/jdbi/JdbiStepFactory.java | 188 +++++++++++++----- .../transact/jdbi/JdbiStepFactoryTest.java | 10 +- .../transact/txstep/PostgresStepFactory.java | 4 +- 3 files changed, 149 insertions(+), 53 deletions(-) diff --git a/transact-jdbi-step-factory/src/main/java/dev/dbos/transact/jdbi/JdbiStepFactory.java b/transact-jdbi-step-factory/src/main/java/dev/dbos/transact/jdbi/JdbiStepFactory.java index ff1782677..68fc3449f 100644 --- a/transact-jdbi-step-factory/src/main/java/dev/dbos/transact/jdbi/JdbiStepFactory.java +++ b/transact-jdbi-step-factory/src/main/java/dev/dbos/transact/jdbi/JdbiStepFactory.java @@ -1,56 +1,101 @@ package dev.dbos.transact.jdbi; import dev.dbos.transact.DBOS; +import dev.dbos.transact.database.SystemDatabase; import dev.dbos.transact.json.DBOSSerializer; +import dev.dbos.transact.json.SerializationUtil; import dev.dbos.transact.txstep.PostgresStepFactory; import dev.dbos.transact.workflow.internal.StepResult; import java.sql.SQLException; +import java.util.Objects; +import java.util.Optional; import org.jdbi.v3.core.Handle; +import org.jdbi.v3.core.HandleCallback; +import org.jdbi.v3.core.HandleConsumer; import org.jdbi.v3.core.Jdbi; -import org.jspecify.annotations.Nullable; /** - * A {@link PostgresStepFactory} implementation backed by Jdbi3 {@link Handle} objects. + * Runs idempotent transactional steps inside DBOS workflows using Jdbi3 {@link Handle} objects. * *

Construct one with a {@link Jdbi} instance pointing at a PostgreSQL database. The constructor * verifies the datasource is PostgreSQL and creates the {@code tx_step_outputs} table if needed. - * User lambdas passed to {@code txStep} receive a {@link Handle} with a transaction already - * started; they should not call {@code commit} or {@code close} themselves. + * Lambdas passed to {@link #inStep} or {@link #useStep} receive a {@link Handle} with a + * transaction already open; they must not call {@code commit} or {@code close} themselves. * * 

{@code
  * JdbiStepFactory factory = new JdbiStepFactory(dbos, Jdbi.create(dataSource));
  *
  * // inside a @Workflow method:
- * int count = factory.txStep(handle -> {
+ * int count = factory.inStep(handle -> {
  *     return handle.createUpdate("INSERT INTO ...").execute();
  * }, "myStep");
  * }
*/ -public class JdbiStepFactory extends PostgresStepFactory { +public class JdbiStepFactory { + private final DBOS dbos; private final Jdbi jdbi; - - /** Creates a factory using the schema from the DBOS config. */ + private final String schema; + private final DBOSSerializer serializer; + + /** + * Creates a factory using the schema and serializer from {@code dbos} configuration. + * + * @param dbos the DBOS runtime instance + * @param jdbi a Jdbi instance connected to a PostgreSQL database + */ public JdbiStepFactory(DBOS dbos, Jdbi jdbi) { this(dbos, jdbi, null, null); } - /** Creates a factory using a custom schema for {@code tx_step_outputs}. */ + /** + * Creates a factory using the given schema and the serializer from {@code dbos} configuration. + * + * @param dbos the DBOS runtime instance + * @param jdbi a Jdbi instance connected to a PostgreSQL database + * @param schema the PostgreSQL schema to use for {@code tx_step_outputs}; {@code null} uses the + * schema from {@code dbos} configuration + */ public JdbiStepFactory(DBOS dbos, Jdbi jdbi, String schema) { this(dbos, jdbi, schema, null); } - /** Creates a factory using a custom serializer. */ + /** + * Creates a factory using the given serializer and the schema from {@code dbos} configuration. + * + * @param dbos the DBOS runtime instance + * @param jdbi a Jdbi instance connected to a PostgreSQL database + * @param serializer the serializer to use for step outputs; {@code null} uses the serializer from + * {@code dbos} configuration + */ public JdbiStepFactory(DBOS dbos, Jdbi jdbi, DBOSSerializer serializer) { this(dbos, jdbi, null, serializer); } - /** Creates a factory with a custom schema and serializer. */ + /** + * Creates a factory with explicit schema and serializer overrides. + * + *

Connects to the database immediately to verify it is PostgreSQL and to create the {@code + * tx_step_outputs} table in the given schema if it does not already exist. + * + * @param dbos the DBOS runtime instance + * @param jdbi a Jdbi instance connected to a PostgreSQL database + * @param schema the PostgreSQL schema to use for {@code tx_step_outputs}; {@code null} uses the + * schema from {@code dbos} configuration + * @param serializer the serializer to use for step outputs; {@code null} uses the serializer from + * {@code dbos} configuration + * @throws RuntimeException if the datasource is not PostgreSQL or the schema setup fails + */ public JdbiStepFactory(DBOS dbos, Jdbi jdbi, String schema, DBOSSerializer serializer) { - super(dbos, schema, serializer); + this.dbos = dbos; this.jdbi = jdbi; + var config = dbos.integration().config(); + + this.schema = SystemDatabase.sanitizeSchema(schema == null ? config.databaseSchema() : schema); + this.serializer = serializer == null ? config.serializer() : serializer; + try { jdbi.useHandle( handle -> { @@ -70,40 +115,79 @@ public JdbiStepFactory(DBOS dbos, Jdbi jdbi, String schema, DBOSSerializer seria } } - @Override - protected Handle openTransaction() { - var handle = jdbi.open(); - handle.begin(); - return handle; - } - - @Override - protected Handle openConnection() { - return jdbi.open(); - } - - @Override - protected void commit(Handle handle) { - handle.commit(); + /** + * Executes {@code callback} as an idempotent DBOS step inside a Jdbi transaction. + * + *

If a result for this step is already recorded (e.g. on workflow retry), the callback is + * skipped and the cached result is returned. Otherwise the callback runs inside an open + * transaction; the output is recorded atomically with the database work so the step is + * exactly-once on success. + * + * @param the return type of the callback + * @param the checked exception type the callback may throw + * @param callback the database work to perform; receives an open {@link Handle} and must not + * commit or close it + * @param stepName a stable name that identifies this step within the workflow + * @return the value returned by {@code callback} + * @throws X if the callback throws + */ + @SuppressWarnings("unchecked") + public R inStep(final HandleCallback callback, String stepName) + throws X { + + return dbos.runStep( + () -> { + var workflowId = Objects.requireNonNull(DBOS.workflowId()); + int stepId = Objects.requireNonNull(DBOS.stepId()); + + var prevResult = checkExecution(workflowId, stepId, stepName); + if (prevResult.isPresent()) { + return prevResult.get().toResult(serializer); + } + + try { + return jdbi.inTransaction( + h -> { + var result = callback.withHandle(h); + recordOutput(h, workflowId, stepId, result); + return result; + }); + } catch (Exception e) { + recordError(workflowId, stepId, e); + throw (X) e; + } + }, + stepName); } - @Override - protected void rollback(Handle handle) { - handle.rollback(); + /** + * Executes {@code callback} as an idempotent DBOS step inside a Jdbi transaction, with no return + * value. + * + *

Behaves identically to {@link #inStep} but accepts a {@link HandleConsumer} for callers + * that do not need to return a result. + * + * @param the checked exception type the callback may throw + * @param callback the database work to perform; receives an open {@link Handle} and must not + * commit or close it + * @param stepName a stable name that identifies this step within the workflow + * @throws X if the callback throws + */ + public void useStep(final HandleConsumer callback, String stepName) + throws X { + inStep( + handle -> { + callback.useHandle(handle); + return null; + }, + stepName); } - @Override - protected void close(Handle handle) { - handle.close(); - } - - @Override - protected @Nullable StepResult checkExecution(String workflowId, int stepId, String stepName) { - var sql = CHECK_SQL_TEMPLATE.formatted(this.schema); + private Optional checkExecution(String workflowId, int stepId, String stepName) { + var sql = PostgresStepFactory.CHECK_SQL_TEMPLATE.formatted(schema); return jdbi.withHandle( - handle -> - handle - .createQuery(sql) + h -> + h.createQuery(sql) .bind(0, workflowId) .bind(1, stepId) .map( @@ -116,12 +200,23 @@ protected void close(Handle handle) { rs.getString("error"), null, rs.getString("serialization"))) - .findFirst() - .orElse(null)); + .findOne()); + } + + private void recordOutput(Handle handle, String workflowId, int stepId, R result) { + var value = SerializationUtil.serializeValue(result, null, serializer); + recordResult(handle, workflowId, stepId, value.serializedValue(), null, value.serialization()); + } + + private void recordError(String workflowId, int stepId, X exception) { + var value = SerializationUtil.serializeError(exception, null, serializer); + jdbi.useTransaction( + h -> { + recordResult(h, workflowId, stepId, null, value.serializedValue(), value.serialization()); + }); } - @Override - protected void recordResult( + private void recordResult( Handle handle, String workflowId, int stepId, @@ -131,8 +226,9 @@ protected void recordResult( if (output != null && error != null) { throw new IllegalArgumentException("attempted to record non null output and error result"); } + var sql = PostgresStepFactory.UPSERT_SQL_TEMPLATE.formatted(schema); handle - .createUpdate(UPSERT_SQL_TEMPLATE.formatted(schema)) + .createUpdate(sql) .bind(0, workflowId) .bind(1, stepId) .bind(2, output) diff --git a/transact-jdbi-step-factory/src/test/java/dev/dbos/transact/jdbi/JdbiStepFactoryTest.java b/transact-jdbi-step-factory/src/test/java/dev/dbos/transact/jdbi/JdbiStepFactoryTest.java index 7f9699e37..2e101468b 100644 --- a/transact-jdbi-step-factory/src/test/java/dev/dbos/transact/jdbi/JdbiStepFactoryTest.java +++ b/transact-jdbi-step-factory/src/test/java/dev/dbos/transact/jdbi/JdbiStepFactoryTest.java @@ -85,26 +85,26 @@ FactoryTestService.TestResult readGreeting(Handle handle, String user) { @Override @Workflow public FactoryTestService.TestResult insertWorkflow(String user) { - return stepFactory.txStep((Handle h) -> insertGreeting(h, user), "insertGreeting"); + return stepFactory.inStep((Handle h) -> insertGreeting(h, user), "insertGreeting"); } @Override @Workflow public FactoryTestService.TestResult errorWorkflow(String user) { - return stepFactory.txStep((Handle h) -> errorGreeting(h, user), "errorGreeting"); + return stepFactory.inStep((Handle h) -> errorGreeting(h, user), "errorGreeting"); } @Override @Workflow public FactoryTestService.TestResult readWorkflow(String user) { - return stepFactory.txStep((Handle h) -> readGreeting(h, user), "readGreeting"); + return stepFactory.inStep((Handle h) -> readGreeting(h, user), "readGreeting"); } @Override @Workflow public FactoryTestService.TestResult insertThenReadWorkflow(String user) { - stepFactory.txStep((Handle h) -> insertGreeting(h, user), "insertGreeting"); - return stepFactory.txStep((Handle h) -> readGreeting(h, user), "readGreeting"); + stepFactory.useStep((Handle h) -> insertGreeting(h, user), "insertGreeting"); + return stepFactory.inStep((Handle h) -> readGreeting(h, user), "readGreeting"); } } diff --git a/transact/src/main/java/dev/dbos/transact/txstep/PostgresStepFactory.java b/transact/src/main/java/dev/dbos/transact/txstep/PostgresStepFactory.java index de81380e7..c6522a810 100644 --- a/transact/src/main/java/dev/dbos/transact/txstep/PostgresStepFactory.java +++ b/transact/src/main/java/dev/dbos/transact/txstep/PostgresStepFactory.java @@ -46,14 +46,14 @@ public abstract class PostgresStepFactory { protected final String schema; protected final DBOSSerializer serializer; - protected static final String CHECK_SQL_TEMPLATE = + public static final String CHECK_SQL_TEMPLATE = """ SELECT output, error, serialization FROM "%s".tx_step_outputs WHERE workflow_id = ? AND step_id = ? """; - protected static final String UPSERT_SQL_TEMPLATE = + public static final String UPSERT_SQL_TEMPLATE = """ INSERT INTO "%s".tx_step_outputs (workflow_id, step_id, output, error, serialization) From d516cafdb3cfea2c76d90b3b95401c3b7e928b27 Mon Sep 17 00:00:00 2001 From: Harry Pierson Date: Tue, 5 May 2026 15:33:15 -0700 Subject: [PATCH 14/29] spotless --- .../main/java/dev/dbos/transact/jdbi/JdbiStepFactory.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/transact-jdbi-step-factory/src/main/java/dev/dbos/transact/jdbi/JdbiStepFactory.java b/transact-jdbi-step-factory/src/main/java/dev/dbos/transact/jdbi/JdbiStepFactory.java index 68fc3449f..ea3ae0eb7 100644 --- a/transact-jdbi-step-factory/src/main/java/dev/dbos/transact/jdbi/JdbiStepFactory.java +++ b/transact-jdbi-step-factory/src/main/java/dev/dbos/transact/jdbi/JdbiStepFactory.java @@ -21,8 +21,8 @@ * *

Construct one with a {@link Jdbi} instance pointing at a PostgreSQL database. The constructor * verifies the datasource is PostgreSQL and creates the {@code tx_step_outputs} table if needed. - * Lambdas passed to {@link #inStep} or {@link #useStep} receive a {@link Handle} with a - * transaction already open; they must not call {@code commit} or {@code close} themselves. + * Lambdas passed to {@link #inStep} or {@link #useStep} receive a {@link Handle} with a transaction + * already open; they must not call {@code commit} or {@code close} themselves. * * 

{@code
  * JdbiStepFactory factory = new JdbiStepFactory(dbos, Jdbi.create(dataSource));
@@ -164,8 +164,8 @@ public  R inStep(final HandleCallback callback, St
    * Executes {@code callback} as an idempotent DBOS step inside a Jdbi transaction, with no return
    * value.
    *
-   * 
Behaves identically to {@link #inStep} but accepts a {@link HandleConsumer} for callers
-   * that do not need to return a result.
+   * 
Behaves identically to {@link #inStep} but accepts a {@link HandleConsumer} for callers that
+   * do not need to return a result.
    *
    * @param  the checked exception type the callback may throw
    * @param callback the database work to perform; receives an open {@link Handle} and must not

From b7ad64a825a922793809426b0e13a1639b72dc90 Mon Sep 17 00:00:00 2001
From: Harry Pierson 
Date: Tue, 5 May 2026 16:07:09 -0700
Subject: [PATCH 15/29] JooqStepFactory

---
 .../dbos/transact/jdbi/JdbiStepFactory.java   |  14 +-
 .../dbos/transact/jooq/JooqStepFactory.java   | 153 ++++++++----------
 .../transact/jooq/JooqStepFactoryTest.java    |  10 +-
 3 files changed, 79 insertions(+), 98 deletions(-)

diff --git a/transact-jdbi-step-factory/src/main/java/dev/dbos/transact/jdbi/JdbiStepFactory.java b/transact-jdbi-step-factory/src/main/java/dev/dbos/transact/jdbi/JdbiStepFactory.java
index ea3ae0eb7..b4a51167c 100644
--- a/transact-jdbi-step-factory/src/main/java/dev/dbos/transact/jdbi/JdbiStepFactory.java
+++ b/transact-jdbi-step-factory/src/main/java/dev/dbos/transact/jdbi/JdbiStepFactory.java
@@ -7,7 +7,6 @@
 import dev.dbos.transact.txstep.PostgresStepFactory;
 import dev.dbos.transact.workflow.internal.StepResult;
 
-import java.sql.SQLException;
 import java.util.Objects;
 import java.util.Optional;
 
@@ -92,26 +91,21 @@ public JdbiStepFactory(DBOS dbos, Jdbi jdbi, String schema, DBOSSerializer seria
     this.dbos = dbos;
     this.jdbi = jdbi;
     var config = dbos.integration().config();
-
     this.schema = SystemDatabase.sanitizeSchema(schema == null ? config.databaseSchema() : schema);
     this.serializer = serializer == null ? config.serializer() : serializer;
 
     try {
       jdbi.useHandle(
           handle -> {
-            try {
-              PostgresStepFactory.ensurePostgres(handle.getConnection());
-              PostgresStepFactory.ensureSchema(handle.getConnection(), this.schema);
-              PostgresStepFactory.ensureTxOutputTable(handle.getConnection(), this.schema);
-            } catch (SQLException e) {
-              throw new RuntimeException(e.getMessage(), e);
-            }
+            PostgresStepFactory.ensurePostgres(handle.getConnection());
+            PostgresStepFactory.ensureSchema(handle.getConnection(), this.schema);
+            PostgresStepFactory.ensureTxOutputTable(handle.getConnection(), this.schema);
           });
     } catch (Exception e) {
       if (e instanceof RuntimeException re) {
         throw re;
       }
-      throw new RuntimeException(e);
+      throw new RuntimeException(e.getMessage(), e);
     }
   }
 
diff --git a/transact-jooq-step-factory/src/main/java/dev/dbos/transact/jooq/JooqStepFactory.java b/transact-jooq-step-factory/src/main/java/dev/dbos/transact/jooq/JooqStepFactory.java
index afb7da358..44f0e763c 100644
--- a/transact-jooq-step-factory/src/main/java/dev/dbos/transact/jooq/JooqStepFactory.java
+++ b/transact-jooq-step-factory/src/main/java/dev/dbos/transact/jooq/JooqStepFactory.java
@@ -1,39 +1,26 @@
 package dev.dbos.transact.jooq;
 
 import dev.dbos.transact.DBOS;
+import dev.dbos.transact.database.SystemDatabase;
 import dev.dbos.transact.json.DBOSSerializer;
+import dev.dbos.transact.json.SerializationUtil;
 import dev.dbos.transact.txstep.PostgresStepFactory;
 import dev.dbos.transact.workflow.internal.StepResult;
 
-import java.sql.SQLException;
+import java.util.Objects;
+import java.util.Optional;
 
+import org.jooq.Configuration;
 import org.jooq.DSLContext;
-import org.jooq.SQLDialect;
-import org.jooq.impl.DSL;
-import org.jspecify.annotations.Nullable;
-
-/**
- * A {@link PostgresStepFactory} implementation backed by jOOQ {@link DSLContext} objects.
- *
- * 
Construct one with a pool-backed {@link DSLContext} pointing at a PostgreSQL database. The
- * constructor verifies the datasource is PostgreSQL and creates the {@code tx_step_outputs} table
- * if needed. User lambdas passed to {@code txStep} receive a per-transaction {@link DSLContext}
- * backed by a single connection with a transaction already started; they should not call {@code
- * commit} or {@code close} themselves.
- *
- * 
{@code
- * DSLContext dsl = DSL.using(dataSource, SQLDialect.POSTGRES);
- * JooqStepFactory factory = new JooqStepFactory(dbos, dsl);
- *
- * // inside a @Workflow method:
- * int count = factory.txStep(ctx -> {
- *     return ctx.execute("INSERT INTO ...");
- * }, "myStep");
- * }

- */
-public class JooqStepFactory extends PostgresStepFactory {
+import org.jooq.TransactionalCallable;
+import org.jooq.TransactionalRunnable;
 
+public class JooqStepFactory {
+
+  private final DBOS dbos;
   private final DSLContext dsl;
+  private final String schema;
+  private final DBOSSerializer serializer;
 
   /** Creates a factory using the schema from the DBOS config. */
   public JooqStepFactory(DBOS dbos, DSLContext dsl) {
@@ -52,18 +39,17 @@ public JooqStepFactory(DBOS dbos, DSLContext dsl, DBOSSerializer serializer) {
 
   /** Creates a factory with a custom schema and serializer. */
   public JooqStepFactory(DBOS dbos, DSLContext dsl, String schema, DBOSSerializer serializer) {
-    super(dbos, schema, serializer);
+    this.dbos = dbos;
     this.dsl = dsl;
+    var config = dbos.integration().config();
+    this.schema = SystemDatabase.sanitizeSchema(schema == null ? config.databaseSchema() : schema);
+    this.serializer = serializer == null ? config.serializer() : serializer;
     try {
       dsl.connection(
           conn -> {
-            try {
-              ensurePostgres(conn);
-              ensureSchema(conn, this.schema);
-              ensureTxOutputTable(conn, this.schema);
-            } catch (SQLException e) {
-              throw new RuntimeException(e.getMessage(), e);
-            }
+            PostgresStepFactory.ensurePostgres(conn);
+            PostgresStepFactory.ensureSchema(conn, this.schema);
+            PostgresStepFactory.ensureTxOutputTable(conn, this.schema);
           });
     } catch (Exception e) {
       if (e instanceof RuntimeException re) {
@@ -73,53 +59,43 @@ public JooqStepFactory(DBOS dbos, DSLContext dsl, String schema, DBOSSerializer
     }
   }
 
-  @Override
-  protected DSLContext openTransaction() {
-    try {
-      var conn = dsl.configuration().connectionProvider().acquire();
-      conn.setAutoCommit(false);
-      return DSL.using(conn, SQLDialect.POSTGRES);
-    } catch (SQLException e) {
-      throw new RuntimeException(e.getMessage(), e);
-    }
-  }
-
-  @Override
-  protected DSLContext openConnection() {
-    var conn = dsl.configuration().connectionProvider().acquire();
-    return DSL.using(conn, SQLDialect.POSTGRES);
-  }
-
-  @Override
-  protected void commit(DSLContext ctx) {
-    try {
-      ctx.connection(conn -> conn.commit());
-    } catch (Exception e) {
-      throw new RuntimeException(e.getMessage(), e);
-    }
-  }
-
-  @Override
-  protected void rollback(DSLContext ctx) {
-    try {
-      ctx.connection(conn -> conn.rollback());
-    } catch (Exception e) {
-      throw new RuntimeException(e.getMessage(), e);
-    }
+  public  T txStepResult(TransactionalCallable callback, String stepName) {
+    return dbos.runStep(
+        () -> {
+          var workflowId = Objects.requireNonNull(DBOS.workflowId());
+          int stepId = Objects.requireNonNull(DBOS.stepId());
+
+          var prevResult = checkExecution(workflowId, stepId, stepName);
+          if (prevResult.isPresent()) {
+            return prevResult.get().toResult(serializer);
+          }
+
+          try {
+            return dsl.transactionResult(
+                trx -> {
+                  var result = callback.run(trx);
+                  recordOutput(trx, workflowId, stepId, result);
+                  return result;
+                });
+          } catch (Exception e) {
+            recordError(workflowId, stepId, e);
+            throw e;
+          }
+        },
+        stepName);
   }
 
-  @Override
-  protected void close(DSLContext ctx) {
-    try {
-      ctx.connection(conn -> conn.close());
-    } catch (Exception e) {
-      throw new RuntimeException(e.getMessage(), e);
-    }
+  public void txStep(TransactionalRunnable transactional, String stepName) {
+    txStepResult(
+        c -> {
+          transactional.run(c);
+          return null;
+        },
+        stepName);
   }
 
-  @Override
-  protected @Nullable StepResult checkExecution(String workflowId, int stepId, String stepName) {
-    var sql = CHECK_SQL_TEMPLATE.formatted(this.schema);
+  private Optional checkExecution(String workflowId, int stepId, String stepName) {
+    var sql = PostgresStepFactory.CHECK_SQL_TEMPLATE.formatted(this.schema);
     return dsl.fetchOptional(sql, workflowId, stepId)
         .map(
             r ->
@@ -130,13 +106,24 @@ protected void close(DSLContext ctx) {
                     r.get("output", String.class),
                     r.get("error", String.class),
                     null,
-                    r.get("serialization", String.class)))
-        .orElse(null);
+                    r.get("serialization", String.class)));
+  }
+
+  private  void recordOutput(Configuration trx, String workflowId, int stepId, R result) {
+    var value = SerializationUtil.serializeValue(result, null, serializer);
+    recordResult(trx, workflowId, stepId, value.serializedValue(), null, value.serialization());
+  }
+
+  private  void recordError(String workflowId, int stepId, X exception) {
+    var value = SerializationUtil.serializeError(exception, null, serializer);
+    dsl.transaction(
+        trx ->
+            recordResult(
+                trx, workflowId, stepId, null, value.serializedValue(), value.serialization()));
   }
 
-  @Override
-  protected void recordResult(
-      DSLContext ctx,
+  private void recordResult(
+      Configuration trx,
       String workflowId,
       int stepId,
       String output,
@@ -145,7 +132,7 @@ protected void recordResult(
     if (output != null && error != null) {
       throw new IllegalArgumentException("attempted to record non null output and error result");
     }
-    ctx.execute(
-        UPSERT_SQL_TEMPLATE.formatted(schema), workflowId, stepId, output, error, serialization);
+    var sql = PostgresStepFactory.UPSERT_SQL_TEMPLATE.formatted(schema);
+    trx.dsl().execute(sql, workflowId, stepId, output, error, serialization);
   }
 }
diff --git a/transact-jooq-step-factory/src/test/java/dev/dbos/transact/jooq/JooqStepFactoryTest.java b/transact-jooq-step-factory/src/test/java/dev/dbos/transact/jooq/JooqStepFactoryTest.java
index 37ebd239a..2f5846ff1 100644
--- a/transact-jooq-step-factory/src/test/java/dev/dbos/transact/jooq/JooqStepFactoryTest.java
+++ b/transact-jooq-step-factory/src/test/java/dev/dbos/transact/jooq/JooqStepFactoryTest.java
@@ -75,26 +75,26 @@ var record = ctx.fetchOne(sql, Objects.requireNonNull(user));
   @Override
   @Workflow
   public FactoryTestService.TestResult insertWorkflow(String user) {
-    return stepFactory.txStep((DSLContext ctx) -> insertGreeting(ctx, user), "insertGreeting");
+    return stepFactory.txStepResult(ctx -> insertGreeting(ctx.dsl(), user), "insertGreeting");
   }
 
   @Override
   @Workflow
   public FactoryTestService.TestResult errorWorkflow(String user) {
-    return stepFactory.txStep((DSLContext ctx) -> errorGreeting(ctx, user), "errorGreeting");
+    return stepFactory.txStepResult(ctx -> errorGreeting(ctx.dsl(), user), "errorGreeting");
   }
 
   @Override
   @Workflow
   public FactoryTestService.TestResult readWorkflow(String user) {
-    return stepFactory.txStep((DSLContext ctx) -> readGreeting(ctx, user), "readGreeting");
+    return stepFactory.txStepResult(ctx -> readGreeting(ctx.dsl(), user), "readGreeting");
   }
 
   @Override
   @Workflow
   public FactoryTestService.TestResult insertThenReadWorkflow(String user) {
-    stepFactory.txStep((DSLContext ctx) -> insertGreeting(ctx, user), "insertGreeting");
-    return stepFactory.txStep((DSLContext ctx) -> readGreeting(ctx, user), "readGreeting");
+    stepFactory.txStep(ctx -> insertGreeting(ctx.dsl(), user), "insertGreeting");
+    return stepFactory.txStepResult(ctx -> readGreeting(ctx.dsl(), user), "readGreeting");
   }
 }
 

From 11f737904a94cc4057c9ab2b12ef910a151bf4c8 Mon Sep 17 00:00:00 2001
From: Harry Pierson 
Date: Tue, 5 May 2026 17:18:52 -0700
Subject: [PATCH 16/29] rework JdbcStepFactory

---
 .../transact/execution/ThrowingConsumer.java  |   6 -
 .../transact/execution/ThrowingFunction.java  |   6 -
 .../dbos/transact/txstep/JdbcStepFactory.java | 196 +++++++++----
 .../transact/txstep/PostgresStepFactory.java  | 272 +++++++++---------
 4 files changed, 265 insertions(+), 215 deletions(-)
 delete mode 100644 transact/src/main/java/dev/dbos/transact/execution/ThrowingConsumer.java
 delete mode 100644 transact/src/main/java/dev/dbos/transact/execution/ThrowingFunction.java

diff --git a/transact/src/main/java/dev/dbos/transact/execution/ThrowingConsumer.java b/transact/src/main/java/dev/dbos/transact/execution/ThrowingConsumer.java
deleted file mode 100644
index 29b4b0692..000000000
--- a/transact/src/main/java/dev/dbos/transact/execution/ThrowingConsumer.java
+++ /dev/null
@@ -1,6 +0,0 @@
-package dev.dbos.transact.execution;
-
-@FunctionalInterface
-public interface ThrowingConsumer {
-  void execute(P p) throws E;
-}
diff --git a/transact/src/main/java/dev/dbos/transact/execution/ThrowingFunction.java b/transact/src/main/java/dev/dbos/transact/execution/ThrowingFunction.java
deleted file mode 100644
index 2c2c1cee7..000000000
--- a/transact/src/main/java/dev/dbos/transact/execution/ThrowingFunction.java
+++ /dev/null
@@ -1,6 +0,0 @@
-package dev.dbos.transact.execution;
-
-@FunctionalInterface
-public interface ThrowingFunction {
-  T execute(P p) throws E;
-}
diff --git a/transact/src/main/java/dev/dbos/transact/txstep/JdbcStepFactory.java b/transact/src/main/java/dev/dbos/transact/txstep/JdbcStepFactory.java
index bba5fe6af..8d6ae47f4 100644
--- a/transact/src/main/java/dev/dbos/transact/txstep/JdbcStepFactory.java
+++ b/transact/src/main/java/dev/dbos/transact/txstep/JdbcStepFactory.java
@@ -1,17 +1,18 @@
 package dev.dbos.transact.txstep;
 
 import dev.dbos.transact.DBOS;
+import dev.dbos.transact.database.SystemDatabase;
 import dev.dbos.transact.json.DBOSSerializer;
+import dev.dbos.transact.json.SerializationUtil;
 import dev.dbos.transact.workflow.internal.StepResult;
 
 import java.sql.Connection;
 import java.sql.SQLException;
 import java.util.Objects;
+import java.util.Optional;
 
 import javax.sql.DataSource;
 
-import org.jspecify.annotations.Nullable;
-
 /**
  * A {@link PostgresStepFactory} implementation backed by plain JDBC {@link Connection} objects.
  *
@@ -30,127 +31,198 @@
  * }, "myStep");
  * }
*/ -public class JdbcStepFactory extends PostgresStepFactory { +public class JdbcStepFactory { + private final DBOS dbos; private final DataSource dataSource; + private final String schema; + private final DBOSSerializer serializer; /** Creates a factory using the schema from the DBOS config. */ - public JdbcStepFactory(DBOS dbos, DataSource dataSource) { + public JdbcStepFactory(DBOS dbos, DataSource dataSource) throws SQLException { this(dbos, dataSource, null, null); } /** Creates a factory using a custom schema for {@code tx_step_outputs}. */ - public JdbcStepFactory(DBOS dbos, DataSource dataSource, String schema) { + public JdbcStepFactory(DBOS dbos, DataSource dataSource, String schema) throws SQLException { this(dbos, dataSource, schema, null); } /** Creates a factory using a custom serializer. */ - public JdbcStepFactory(DBOS dbos, DataSource dataSource, DBOSSerializer serializer) { + public JdbcStepFactory(DBOS dbos, DataSource dataSource, DBOSSerializer serializer) + throws SQLException { this(dbos, dataSource, null, serializer); } /** Creates a factory with a custom schema and serializer. */ - public JdbcStepFactory( - DBOS dbos, DataSource dataSource, String schema, DBOSSerializer serializer) { - super(dbos, schema, serializer); + public JdbcStepFactory(DBOS dbos, DataSource dataSource, String schema, DBOSSerializer serializer) + throws SQLException { + this.dbos = dbos; this.dataSource = Objects.requireNonNull(dataSource); - createTxOutputTable(dataSource, this.schema); - } + var config = dbos.integration().config(); + this.schema = SystemDatabase.sanitizeSchema(schema == null ? config.databaseSchema() : schema); + this.serializer = serializer == null ? config.serializer() : serializer; - /** - * Verifies the datasource is PostgreSQL and creates the {@code tx_step_outputs} table if it does - * not exist. Useful when the {@code DataSource} is managed separately from the factory lifecycle. - */ - public static void createTxOutputTable(DataSource dataSource, String schema) { try (var conn = dataSource.getConnection()) { - ensurePostgres(conn); - ensureSchema(conn, schema); - ensureTxOutputTable(conn, schema); - } catch (SQLException e) { - throw new DBOSSqlException(e); + PostgresStepFactory.ensurePostgres(conn); + PostgresStepFactory.ensureSchema(conn, this.schema); + PostgresStepFactory.ensureTxOutputTable(conn, this.schema); } } - private static class DBOSSqlException extends RuntimeException { - public DBOSSqlException(SQLException wrappedException) { - super(wrappedException.getMessage(), wrappedException); - } + @FunctionalInterface + public interface TransactionalFunction { + R execute(Connection conn) throws X; + } + + @SuppressWarnings("unchecked") + public R txStep( + final TransactionalFunction callback, String stepName) throws X { + return dbos.runStep( + () -> { + var workflowId = Objects.requireNonNull(DBOS.workflowId()); + int stepId = Objects.requireNonNull(DBOS.stepId()); + + var prevResult = checkExecution(workflowId, stepId, stepName); + if (prevResult.isPresent()) { + return prevResult.get().toResult(serializer); + } + + try { + return executeTransaction( + dataSource, + c -> { + var result = callback.execute(c); + recordOutput(c, workflowId, stepId, result); + return result; + }); + } catch (Exception e) { + recordError(workflowId, stepId, e); + throw (X) e; + } + }, + stepName); } - @Override - protected Connection openTransaction() { + @FunctionalInterface + public interface TransactionalRunnable { + void execute(Connection conn) throws X; + } + + public void txStep(final TransactionalRunnable callback, String stepName) + throws X { + txStep( + c -> { + callback.execute(c); + return null; + }, + stepName); + } + + private static R executeTransaction( + final DataSource ds, TransactionalFunction func) throws X { + var conn = openTransaction(ds); try { - var conn = dataSource.getConnection(); - conn.setAutoCommit(false); - return conn; - } catch (SQLException e) { - throw new DBOSSqlException(e); + var result = func.execute(conn); + commit(conn); + return result; + } catch (Exception e) { + rollback(conn); + throw e; + } finally { + close(conn); } } - @Override - protected Connection openConnection() { + static class WrappedSqlException extends RuntimeException { + private final SQLException wrappedException; + + public WrappedSqlException(SQLException wrappedException) { + super(wrappedException.getMessage(), wrappedException); + this.wrappedException = wrappedException; + } + + public SQLException wrappedException() { + return this.wrappedException; + } + } + + private static Connection openTransaction(DataSource ds) { try { - return dataSource.getConnection(); + var conn = ds.getConnection(); + conn.setAutoCommit(false); + return conn; } catch (SQLException e) { - throw new DBOSSqlException(e); + throw new WrappedSqlException(e); } } - @Override - protected void commit(Connection conn) { + private static void commit(Connection conn) { try { conn.commit(); } catch (SQLException e) { - throw new DBOSSqlException(e); + throw new WrappedSqlException(e); } } - @Override - protected void rollback(Connection conn) { + private static void rollback(Connection conn) { try { conn.rollback(); } catch (SQLException e) { - throw new DBOSSqlException(e); + throw new WrappedSqlException(e); } } - @Override - protected void close(Connection conn) { + private static void close(Connection conn) { try { conn.close(); } catch (SQLException e) { - throw new DBOSSqlException(e); + throw new WrappedSqlException(e); } } - @Override - protected @Nullable StepResult checkExecution(String workflowId, int stepId, String stepName) { - var sql = CHECK_SQL_TEMPLATE.formatted(this.schema); + private Optional checkExecution(String workflowId, int stepId, String stepName) { + var sql = PostgresStepFactory.CHECK_SQL_TEMPLATE.formatted(this.schema); try (var conn = dataSource.getConnection(); var stmt = conn.prepareStatement(sql)) { stmt.setString(1, workflowId); stmt.setInt(2, stepId); try (var rs = stmt.executeQuery()) { if (rs.next()) { - return new StepResult( - workflowId, - stepId, - stepName, - rs.getString("output"), - rs.getString("error"), - null, - rs.getString("serialization")); + return Optional.of( + new StepResult( + workflowId, + stepId, + stepName, + rs.getString("output"), + rs.getString("error"), + null, + rs.getString("serialization"))); } - return null; + return Optional.empty(); } } catch (SQLException e) { - throw new DBOSSqlException(e); + throw new WrappedSqlException(e); } } - @Override - protected void recordResult( + private void recordOutput(Connection conn, String workflowId, int stepId, R result) { + var value = SerializationUtil.serializeValue(result, null, serializer); + recordResult(conn, workflowId, stepId, value.serializedValue(), null, value.serialization()); + } + + private void recordError(String workflowId, int stepId, X exception) { + final var value = SerializationUtil.serializeError(exception, null, serializer); + executeTransaction( + dataSource, + (Connection conn) -> { + recordResult( + conn, workflowId, stepId, null, value.serializedValue(), value.serialization()); + return null; + }); + } + + private void recordResult( Connection conn, String workflowId, int stepId, @@ -160,7 +232,7 @@ protected void recordResult( if (output != null && error != null) { throw new IllegalArgumentException("attempted to record non null output and error result"); } - var sql = UPSERT_SQL_TEMPLATE.formatted(schema); + var sql = PostgresStepFactory.UPSERT_SQL_TEMPLATE.formatted(schema); try (var stmt = conn.prepareStatement(sql)) { stmt.setString(1, workflowId); stmt.setInt(2, stepId); @@ -169,7 +241,7 @@ protected void recordResult( stmt.setString(5, serialization); stmt.executeUpdate(); } catch (SQLException e) { - throw new DBOSSqlException(e); + throw new WrappedSqlException(e); } } } diff --git a/transact/src/main/java/dev/dbos/transact/txstep/PostgresStepFactory.java b/transact/src/main/java/dev/dbos/transact/txstep/PostgresStepFactory.java index c6522a810..0a40e76ce 100644 --- a/transact/src/main/java/dev/dbos/transact/txstep/PostgresStepFactory.java +++ b/transact/src/main/java/dev/dbos/transact/txstep/PostgresStepFactory.java @@ -1,18 +1,9 @@ package dev.dbos.transact.txstep; -import dev.dbos.transact.DBOS; -import dev.dbos.transact.database.SystemDatabase; -import dev.dbos.transact.execution.ThrowingConsumer; -import dev.dbos.transact.execution.ThrowingFunction; -import dev.dbos.transact.json.DBOSSerializer; -import dev.dbos.transact.json.SerializationUtil; -import dev.dbos.transact.workflow.internal.StepResult; - import java.sql.Connection; import java.sql.SQLException; import java.util.Objects; -import org.jspecify.annotations.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -40,11 +31,11 @@ public abstract class PostgresStepFactory { private static final Logger DDL_LOGGER = LoggerFactory.getLogger(PostgresStepFactory.class); - private final Logger logger = LoggerFactory.getLogger(getClass()); + // private final Logger logger = LoggerFactory.getLogger(getClass()); - protected final DBOS dbos; - protected final String schema; - protected final DBOSSerializer serializer; + // protected final DBOS dbos; + // protected final String schema; + // protected final DBOSSerializer serializer; public static final String CHECK_SQL_TEMPLATE = """ @@ -148,141 +139,140 @@ PRIMARY KEY (workflow_id, step_id) } } - /** - * Constructs a factory for the given DBOS instance. - * - * @param dbos the DBOS runtime - * @param rawSchema the schema for {@code tx_step_outputs}, or {@code null} to use the schema from - * the DBOS config - * @param rawSerializer the serializer for step outputs, or {@code null} to use the serializer - * from the DBOS config - */ - protected PostgresStepFactory( - DBOS dbos, @Nullable String rawSchema, @Nullable DBOSSerializer rawSerializer) { - this.dbos = Objects.requireNonNull(dbos); - var config = dbos.integration().config(); - this.schema = - SystemDatabase.sanitizeSchema(rawSchema == null ? config.databaseSchema() : rawSchema); - this.serializer = rawSerializer == null ? config.serializer() : rawSerializer; - } - - /** Opens a connection with a transaction already started (autoCommit off). */ - protected abstract C openTransaction(); - - /** Opens a connection without starting a transaction (used for recording errors). */ - protected abstract C openConnection(); - - /** Commits the transaction on the given connection. */ - protected abstract void commit(C conn); + // /** + // * Constructs a factory for the given DBOS instance. + // * + // * @param dbos the DBOS runtime + // * @param rawSchema the schema for {@code tx_step_outputs}, or {@code null} to use the schema + // from + // * the DBOS config + // * @param rawSerializer the serializer for step outputs, or {@code null} to use the serializer + // * from the DBOS config + // */ + // protected PostgresStepFactory( + // DBOS dbos, @Nullable String rawSchema, @Nullable DBOSSerializer rawSerializer) { + // this.dbos = Objects.requireNonNull(dbos); + // var config = dbos.integration().config(); + // this.schema = + // SystemDatabase.sanitizeSchema(rawSchema == null ? config.databaseSchema() : rawSchema); + // this.serializer = rawSerializer == null ? config.serializer() : rawSerializer; + // } - /** Rolls back the transaction on the given connection. */ - protected abstract void rollback(C conn); + // /** + // * Checks {@code tx_step_outputs} for a prior result for the given workflow step. + // * + // * @return the prior result, or {@code null} if the step has not been executed + // */ + // protected abstract @Nullable StepResult checkExecution( + // String workflowId, int stepId, String stepName); - /** Closes and releases the given connection. */ - protected abstract void close(C conn); - - /** - * Checks {@code tx_step_outputs} for a prior result for the given workflow step. - * - * @return the prior result, or {@code null} if the step has not been executed - */ - protected abstract @Nullable StepResult checkExecution( - String workflowId, int stepId, String stepName); - - /** - * Writes a step result to {@code tx_step_outputs} using the given connection. Exactly one of - * {@code output} and {@code error} must be non-null. - */ - protected abstract void recordResult( - C conn, String workflowId, int stepId, String output, String error, String serialization); + // /** + // * Writes a step result to {@code tx_step_outputs} using the given connection. Exactly one of + // * {@code output} and {@code error} must be non-null. + // */ + // protected abstract void recordResult( + // C conn, String workflowId, int stepId, String output, String error, String serialization); - /** - * Executes a transactional step that returns a value. - * - *

On the first call for a given {@code (workflowId, stepId)}, the step function is invoked - * inside a database transaction. The return value is serialized and written to {@code - * tx_step_outputs} atomically with the user's data before the transaction commits. On subsequent - * calls with the same IDs (idempotency or crash recovery), the cached output is deserialized and - * returned without re-executing the function. - * - * @param func the database operation to run; receives the transactional connection object - * @param stepName a human-readable name for logging and DBOS step tracking - * @return the value returned by {@code func} - * @throws E if {@code func} throws, after rolling back the transaction and recording the error - */ - public R txStep(ThrowingFunction func, String stepName) - throws E { - return dbos.runStep(() -> txStepInternal(func, stepName), stepName); - } + // // /** + // // * Executes a transactional step that returns a value. + // // * + // // *

On the first call for a given {@code (workflowId, stepId)}, the step function is + // invoked + // // * inside a database transaction. The return value is serialized and written to {@code + // // * tx_step_outputs} atomically with the user's data before the transaction commits. On + // // subsequent + // // * calls with the same IDs (idempotency or crash recovery), the cached output is + // deserialized + // // and + // // * returned without re-executing the function. + // // * + // // * @param func the database operation to run; receives the transactional connection object + // // * @param stepName a human-readable name for logging and DBOS step tracking + // // * @return the value returned by {@code func} + // // * @throws E if {@code func} throws, after rolling back the transaction and recording the + // error + // // */ + // // public R txStep(ThrowingFunction func, String stepName) + // // throws E { + // // return dbos.runStep(() -> txStepInternal(func, stepName), stepName); + // // } - /** - * Executes a transactional step that returns no value. - * - * @param func the database operation to run; receives the transactional connection object - * @param stepName a human-readable name for logging and DBOS step tracking - * @throws E if {@code func} throws, after rolling back the transaction and recording the error - */ - public void txStep(ThrowingConsumer func, String stepName) throws E { - txStep( - c -> { - func.execute(c); - return null; - }, - stepName); - } + // // /** + // // * Executes a transactional step that returns no value. + // // * + // // * @param func the database operation to run; receives the transactional connection object + // // * @param stepName a human-readable name for logging and DBOS step tracking + // // * @throws E if {@code func} throws, after rolling back the transaction and recording the + // error + // // */ + // // public void txStep(ThrowingConsumer func, String stepName) + // throws E + // // { + // // txStep( + // // c -> { + // // func.execute(c); + // // return null; + // // }, + // // stepName); + // // } - protected final void recordOutput(C conn, String workflowId, int stepId, R retVal) { - var value = SerializationUtil.serializeValue(retVal, null, serializer); - recordResult(conn, workflowId, stepId, value.serializedValue(), null, value.serialization()); - } + // // protected final void recordOutput(C conn, String workflowId, int stepId, R retVal) { + // // var value = SerializationUtil.serializeValue(retVal, null, serializer); + // // recordResult(conn, workflowId, stepId, value.serializedValue(), null, + // value.serialization()); + // // } - protected final void recordError( - String workflowId, int stepId, E exception) { - var value = SerializationUtil.serializeError(exception, null, serializer); - var conn = openConnection(); - try { - recordResult(conn, workflowId, stepId, null, value.serializedValue(), value.serialization()); - } finally { - close(conn); - } - } + // // protected final void recordError( + // // String workflowId, int stepId, E exception) { + // // var value = SerializationUtil.serializeError(exception, null, serializer); + // // var conn = openConnection(); + // // try { + // // recordResult(conn, workflowId, stepId, null, value.serializedValue(), + // // value.serialization()); + // // } finally { + // // close(conn); + // // } + // // } - protected final R txStepInternal( - ThrowingFunction func, String stepName) throws E { - var workflowId = Objects.requireNonNull(DBOS.workflowId()); - int stepId = Objects.requireNonNull(DBOS.stepId()); + // // protected final R txStepInternal( + // // ThrowingFunction func, String stepName) throws E { + // // var workflowId = Objects.requireNonNull(DBOS.workflowId()); + // // int stepId = Objects.requireNonNull(DBOS.stepId()); - logger.debug( - "txStep starting: workflowId={} stepId={} stepName={}", workflowId, stepId, stepName); - var prevResult = this.checkExecution(workflowId, stepId, stepName); - if (prevResult != null) { - logger.debug( - "txStep cache hit: workflowId={} stepId={} stepName={}", workflowId, stepId, stepName); - return prevResult.toResult(serializer); - } + // // logger.debug( + // // "txStep starting: workflowId={} stepId={} stepName={}", workflowId, stepId, stepName); + // // var prevResult = this.checkExecution(workflowId, stepId, stepName); + // // if (prevResult != null) { + // // logger.debug( + // // "txStep cache hit: workflowId={} stepId={} stepName={}", workflowId, stepId, + // stepName); + // // return prevResult.toResult(serializer); + // // } - logger.debug( - "txStep executing: workflowId={} stepId={} stepName={}", workflowId, stepId, stepName); - var conn = openTransaction(); - try { - var retVal = func.execute(conn); - recordOutput(conn, workflowId, stepId, retVal); - commit(conn); - logger.debug( - "txStep succeeded: workflowId={} stepId={} stepName={}", workflowId, stepId, stepName); - return retVal; - } catch (Exception e) { - rollback(conn); - recordError(workflowId, stepId, e); - logger.debug( - "txStep failed: workflowId={} stepId={} stepName={} error={}", - workflowId, - stepId, - stepName, - e.getMessage()); - throw e; - } finally { - close(conn); - } - } + // // logger.debug( + // // "txStep executing: workflowId={} stepId={} stepName={}", workflowId, stepId, + // stepName); + // // var conn = openTransaction(); + // // try { + // // var retVal = func.execute(conn); + // // recordOutput(conn, workflowId, stepId, retVal); + // // commit(conn); + // // logger.debug( + // // "txStep succeeded: workflowId={} stepId={} stepName={}", workflowId, stepId, + // stepName); + // // return retVal; + // // } catch (Exception e) { + // // rollback(conn); + // // recordError(workflowId, stepId, e); + // // logger.debug( + // // "txStep failed: workflowId={} stepId={} stepName={} error={}", + // // workflowId, + // // stepId, + // // stepName, + // // e.getMessage()); + // // throw e; + // // } finally { + // // close(conn); + // // } + // // } } From b1d4f8278818b0689fd8565c9ced8f9212ce4fe9 Mon Sep 17 00:00:00 2001 From: Harry Pierson Date: Tue, 5 May 2026 17:22:35 -0700 Subject: [PATCH 17/29] WIP --- .../dbos/transact/jdbi/JdbiStepFactory.java | 12 +- .../dbos/transact/jooq/JooqStepFactory.java | 12 +- .../dbos/transact/txstep/JdbcStepFactory.java | 12 +- .../transact/txstep/PostgresStepFactory.java | 278 ------------------ .../txstep/PostgresStepFactoryHelpers.java | 115 ++++++++ 5 files changed, 133 insertions(+), 296 deletions(-) delete mode 100644 transact/src/main/java/dev/dbos/transact/txstep/PostgresStepFactory.java create mode 100644 transact/src/main/java/dev/dbos/transact/txstep/PostgresStepFactoryHelpers.java diff --git a/transact-jdbi-step-factory/src/main/java/dev/dbos/transact/jdbi/JdbiStepFactory.java b/transact-jdbi-step-factory/src/main/java/dev/dbos/transact/jdbi/JdbiStepFactory.java index b4a51167c..c9b302876 100644 --- a/transact-jdbi-step-factory/src/main/java/dev/dbos/transact/jdbi/JdbiStepFactory.java +++ b/transact-jdbi-step-factory/src/main/java/dev/dbos/transact/jdbi/JdbiStepFactory.java @@ -4,7 +4,7 @@ import dev.dbos.transact.database.SystemDatabase; import dev.dbos.transact.json.DBOSSerializer; import dev.dbos.transact.json.SerializationUtil; -import dev.dbos.transact.txstep.PostgresStepFactory; +import dev.dbos.transact.txstep.PostgresStepFactoryHelpers; import dev.dbos.transact.workflow.internal.StepResult; import java.util.Objects; @@ -97,9 +97,9 @@ public JdbiStepFactory(DBOS dbos, Jdbi jdbi, String schema, DBOSSerializer seria try { jdbi.useHandle( handle -> { - PostgresStepFactory.ensurePostgres(handle.getConnection()); - PostgresStepFactory.ensureSchema(handle.getConnection(), this.schema); - PostgresStepFactory.ensureTxOutputTable(handle.getConnection(), this.schema); + PostgresStepFactoryHelpers.ensurePostgres(handle.getConnection()); + PostgresStepFactoryHelpers.ensureSchema(handle.getConnection(), this.schema); + PostgresStepFactoryHelpers.ensureTxOutputTable(handle.getConnection(), this.schema); }); } catch (Exception e) { if (e instanceof RuntimeException re) { @@ -178,7 +178,7 @@ public void useStep(final HandleConsumer callback, Stri } private Optional checkExecution(String workflowId, int stepId, String stepName) { - var sql = PostgresStepFactory.CHECK_SQL_TEMPLATE.formatted(schema); + var sql = PostgresStepFactoryHelpers.CHECK_SQL_TEMPLATE.formatted(schema); return jdbi.withHandle( h -> h.createQuery(sql) @@ -220,7 +220,7 @@ private void recordResult( if (output != null && error != null) { throw new IllegalArgumentException("attempted to record non null output and error result"); } - var sql = PostgresStepFactory.UPSERT_SQL_TEMPLATE.formatted(schema); + var sql = PostgresStepFactoryHelpers.UPSERT_SQL_TEMPLATE.formatted(schema); handle .createUpdate(sql) .bind(0, workflowId) diff --git a/transact-jooq-step-factory/src/main/java/dev/dbos/transact/jooq/JooqStepFactory.java b/transact-jooq-step-factory/src/main/java/dev/dbos/transact/jooq/JooqStepFactory.java index 44f0e763c..8202197c0 100644 --- a/transact-jooq-step-factory/src/main/java/dev/dbos/transact/jooq/JooqStepFactory.java +++ b/transact-jooq-step-factory/src/main/java/dev/dbos/transact/jooq/JooqStepFactory.java @@ -4,7 +4,7 @@ import dev.dbos.transact.database.SystemDatabase; import dev.dbos.transact.json.DBOSSerializer; import dev.dbos.transact.json.SerializationUtil; -import dev.dbos.transact.txstep.PostgresStepFactory; +import dev.dbos.transact.txstep.PostgresStepFactoryHelpers; import dev.dbos.transact.workflow.internal.StepResult; import java.util.Objects; @@ -47,9 +47,9 @@ public JooqStepFactory(DBOS dbos, DSLContext dsl, String schema, DBOSSerializer try { dsl.connection( conn -> { - PostgresStepFactory.ensurePostgres(conn); - PostgresStepFactory.ensureSchema(conn, this.schema); - PostgresStepFactory.ensureTxOutputTable(conn, this.schema); + PostgresStepFactoryHelpers.ensurePostgres(conn); + PostgresStepFactoryHelpers.ensureSchema(conn, this.schema); + PostgresStepFactoryHelpers.ensureTxOutputTable(conn, this.schema); }); } catch (Exception e) { if (e instanceof RuntimeException re) { @@ -95,7 +95,7 @@ public void txStep(TransactionalRunnable transactional, String stepName) { } private Optional checkExecution(String workflowId, int stepId, String stepName) { - var sql = PostgresStepFactory.CHECK_SQL_TEMPLATE.formatted(this.schema); + var sql = PostgresStepFactoryHelpers.CHECK_SQL_TEMPLATE.formatted(this.schema); return dsl.fetchOptional(sql, workflowId, stepId) .map( r -> @@ -132,7 +132,7 @@ private void recordResult( if (output != null && error != null) { throw new IllegalArgumentException("attempted to record non null output and error result"); } - var sql = PostgresStepFactory.UPSERT_SQL_TEMPLATE.formatted(schema); + var sql = PostgresStepFactoryHelpers.UPSERT_SQL_TEMPLATE.formatted(schema); trx.dsl().execute(sql, workflowId, stepId, output, error, serialization); } } diff --git a/transact/src/main/java/dev/dbos/transact/txstep/JdbcStepFactory.java b/transact/src/main/java/dev/dbos/transact/txstep/JdbcStepFactory.java index 8d6ae47f4..2f3c0d736 100644 --- a/transact/src/main/java/dev/dbos/transact/txstep/JdbcStepFactory.java +++ b/transact/src/main/java/dev/dbos/transact/txstep/JdbcStepFactory.java @@ -14,7 +14,7 @@ import javax.sql.DataSource; /** - * A {@link PostgresStepFactory} implementation backed by plain JDBC {@link Connection} objects. + * A {@link PostgresStepFactoryHelpers} implementation backed by plain JDBC {@link Connection} objects. * *

Construct one with a {@link DataSource} pointing at a PostgreSQL database. The constructor * verifies the datasource is PostgreSQL and creates the {@code tx_step_outputs} table if needed. @@ -64,9 +64,9 @@ public JdbcStepFactory(DBOS dbos, DataSource dataSource, String schema, DBOSSeri this.serializer = serializer == null ? config.serializer() : serializer; try (var conn = dataSource.getConnection()) { - PostgresStepFactory.ensurePostgres(conn); - PostgresStepFactory.ensureSchema(conn, this.schema); - PostgresStepFactory.ensureTxOutputTable(conn, this.schema); + PostgresStepFactoryHelpers.ensurePostgres(conn); + PostgresStepFactoryHelpers.ensureSchema(conn, this.schema); + PostgresStepFactoryHelpers.ensureTxOutputTable(conn, this.schema); } } @@ -182,7 +182,7 @@ private static void close(Connection conn) { } private Optional checkExecution(String workflowId, int stepId, String stepName) { - var sql = PostgresStepFactory.CHECK_SQL_TEMPLATE.formatted(this.schema); + var sql = PostgresStepFactoryHelpers.CHECK_SQL_TEMPLATE.formatted(this.schema); try (var conn = dataSource.getConnection(); var stmt = conn.prepareStatement(sql)) { stmt.setString(1, workflowId); @@ -232,7 +232,7 @@ private void recordResult( if (output != null && error != null) { throw new IllegalArgumentException("attempted to record non null output and error result"); } - var sql = PostgresStepFactory.UPSERT_SQL_TEMPLATE.formatted(schema); + var sql = PostgresStepFactoryHelpers.UPSERT_SQL_TEMPLATE.formatted(schema); try (var stmt = conn.prepareStatement(sql)) { stmt.setString(1, workflowId); stmt.setInt(2, stepId); diff --git a/transact/src/main/java/dev/dbos/transact/txstep/PostgresStepFactory.java b/transact/src/main/java/dev/dbos/transact/txstep/PostgresStepFactory.java deleted file mode 100644 index 0a40e76ce..000000000 --- a/transact/src/main/java/dev/dbos/transact/txstep/PostgresStepFactory.java +++ /dev/null @@ -1,278 +0,0 @@ -package dev.dbos.transact.txstep; - -import java.sql.Connection; -import java.sql.SQLException; -import java.util.Objects; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Abstract base class for PostgreSQL-backed transactional step factories. - * - *

A step factory wraps user-provided database operations as durable DBOS workflow steps. Before - * executing a step, the factory checks {@code tx_step_outputs} for a prior result. If one exists, - * it is returned directly (idempotency / crash recovery). If not, the operation runs inside a - * database transaction: the result is written to {@code tx_step_outputs} atomically with the user's - * data, and the transaction is committed. On failure the transaction is rolled back and the error - * is recorded so it can be replayed on retry. - * - *

Subclasses provide the connection-management primitives ({@link #openTransaction()}, {@link - * #commit}, {@link #rollback}, {@link #close}) and SQL execution ({@link #checkExecution}, {@link - * #recordResult}) for a specific database access layer. The type parameter {@code C} represents the - * connection/session object that the user's lambda receives. - * - *

All implementations require a PostgreSQL datasource. Use {@link #ensurePostgres} during - * construction to fail fast if a non-PostgreSQL datasource is supplied. - * - * @param the connection type exposed to user lambdas (e.g. {@code Connection}, {@code Handle}, - * {@code DSLContext}) - */ -public abstract class PostgresStepFactory { - - private static final Logger DDL_LOGGER = LoggerFactory.getLogger(PostgresStepFactory.class); - // private final Logger logger = LoggerFactory.getLogger(getClass()); - - // protected final DBOS dbos; - // protected final String schema; - // protected final DBOSSerializer serializer; - - public static final String CHECK_SQL_TEMPLATE = - """ - SELECT output, error, serialization - FROM "%s".tx_step_outputs - WHERE workflow_id = ? AND step_id = ? - """; - - public static final String UPSERT_SQL_TEMPLATE = - """ - INSERT INTO "%s".tx_step_outputs - (workflow_id, step_id, output, error, serialization) - VALUES (?, ?, ?, ?, ?) - ON CONFLICT DO NOTHING - """; - - /** - * Verifies that the given connection is to a PostgreSQL database. - * - * @throws IllegalArgumentException if the database is not PostgreSQL - * @throws SQLException if database metadata cannot be read - */ - public static void ensurePostgres(Connection conn) throws SQLException { - var productName = conn.getMetaData().getDatabaseProductName(); - if (!productName.equalsIgnoreCase("PostgreSQL")) { - throw new IllegalArgumentException( - "PostgresStepFactory requires a PostgreSQL datasource, got: " + productName); - } - } - - /** - * Returns {@code true} if the named schema exists in the database. - * - * @throws SQLException if the query fails - */ - public static boolean schemaExists(Connection conn, String schema) throws SQLException { - var sql = "SELECT schema_name FROM information_schema.schemata WHERE schema_name = ?"; - try (var stmt = conn.prepareStatement(sql)) { - stmt.setString(1, Objects.requireNonNull(schema, "schema must not be null")); - try (var rs = stmt.executeQuery()) { - return rs.next(); - } - } - } - - /** - * Creates the named schema if it does not already exist. - * - * @throws SQLException if the DDL fails - */ - public static void ensureSchema(Connection conn, String schema) throws SQLException { - Objects.requireNonNull(schema, "schema must not be null"); - if (!schemaExists(conn, schema)) { - try (var stmt = conn.createStatement()) { - stmt.execute("CREATE SCHEMA IF NOT EXISTS \"%s\"".formatted(schema)); - } - } - } - - /** - * Returns {@code true} if the {@code tx_step_outputs} table exists in the named schema. - * - * @throws SQLException if the query fails - */ - public static boolean tableExists(Connection conn, String schema) throws SQLException { - var sql = "SELECT 1 FROM information_schema.tables WHERE table_schema = ? AND table_name = ?"; - try (var stmt = conn.prepareStatement(sql)) { - stmt.setString(1, Objects.requireNonNull(schema, "schema must not be null")); - stmt.setString(2, Objects.requireNonNull("tx_step_outputs", "tableName must not be null")); - try (var rs = stmt.executeQuery()) { - return rs.next(); - } - } - } - - /** - * Creates the {@code tx_step_outputs} table in the named schema if it does not already exist. - * - * @throws SQLException if the DDL fails - */ - public static void ensureTxOutputTable(Connection conn, String schema) throws SQLException { - Objects.requireNonNull(schema, "schema must not be null"); - if (tableExists(conn, schema)) { - return; - } - DDL_LOGGER.debug("Creating tx_step_outputs table in schema={}", schema); - try (var stmt = conn.createStatement()) { - var ddlSql = - """ - CREATE TABLE IF NOT EXISTS "%1$s".tx_step_outputs ( - workflow_id TEXT NOT NULL, - step_id INT NOT NULL, - output TEXT, - error TEXT, - serialization TEXT, - created_at BIGINT NOT NULL DEFAULT (EXTRACT(EPOCH FROM now())*1000)::bigint, - PRIMARY KEY (workflow_id, step_id) - )""" - .formatted(schema); - stmt.execute(ddlSql); - } - } - - // /** - // * Constructs a factory for the given DBOS instance. - // * - // * @param dbos the DBOS runtime - // * @param rawSchema the schema for {@code tx_step_outputs}, or {@code null} to use the schema - // from - // * the DBOS config - // * @param rawSerializer the serializer for step outputs, or {@code null} to use the serializer - // * from the DBOS config - // */ - // protected PostgresStepFactory( - // DBOS dbos, @Nullable String rawSchema, @Nullable DBOSSerializer rawSerializer) { - // this.dbos = Objects.requireNonNull(dbos); - // var config = dbos.integration().config(); - // this.schema = - // SystemDatabase.sanitizeSchema(rawSchema == null ? config.databaseSchema() : rawSchema); - // this.serializer = rawSerializer == null ? config.serializer() : rawSerializer; - // } - - // /** - // * Checks {@code tx_step_outputs} for a prior result for the given workflow step. - // * - // * @return the prior result, or {@code null} if the step has not been executed - // */ - // protected abstract @Nullable StepResult checkExecution( - // String workflowId, int stepId, String stepName); - - // /** - // * Writes a step result to {@code tx_step_outputs} using the given connection. Exactly one of - // * {@code output} and {@code error} must be non-null. - // */ - // protected abstract void recordResult( - // C conn, String workflowId, int stepId, String output, String error, String serialization); - - // // /** - // // * Executes a transactional step that returns a value. - // // * - // // *

On the first call for a given {@code (workflowId, stepId)}, the step function is - // invoked - // // * inside a database transaction. The return value is serialized and written to {@code - // // * tx_step_outputs} atomically with the user's data before the transaction commits. On - // // subsequent - // // * calls with the same IDs (idempotency or crash recovery), the cached output is - // deserialized - // // and - // // * returned without re-executing the function. - // // * - // // * @param func the database operation to run; receives the transactional connection object - // // * @param stepName a human-readable name for logging and DBOS step tracking - // // * @return the value returned by {@code func} - // // * @throws E if {@code func} throws, after rolling back the transaction and recording the - // error - // // */ - // // public R txStep(ThrowingFunction func, String stepName) - // // throws E { - // // return dbos.runStep(() -> txStepInternal(func, stepName), stepName); - // // } - - // // /** - // // * Executes a transactional step that returns no value. - // // * - // // * @param func the database operation to run; receives the transactional connection object - // // * @param stepName a human-readable name for logging and DBOS step tracking - // // * @throws E if {@code func} throws, after rolling back the transaction and recording the - // error - // // */ - // // public void txStep(ThrowingConsumer func, String stepName) - // throws E - // // { - // // txStep( - // // c -> { - // // func.execute(c); - // // return null; - // // }, - // // stepName); - // // } - - // // protected final void recordOutput(C conn, String workflowId, int stepId, R retVal) { - // // var value = SerializationUtil.serializeValue(retVal, null, serializer); - // // recordResult(conn, workflowId, stepId, value.serializedValue(), null, - // value.serialization()); - // // } - - // // protected final void recordError( - // // String workflowId, int stepId, E exception) { - // // var value = SerializationUtil.serializeError(exception, null, serializer); - // // var conn = openConnection(); - // // try { - // // recordResult(conn, workflowId, stepId, null, value.serializedValue(), - // // value.serialization()); - // // } finally { - // // close(conn); - // // } - // // } - - // // protected final R txStepInternal( - // // ThrowingFunction func, String stepName) throws E { - // // var workflowId = Objects.requireNonNull(DBOS.workflowId()); - // // int stepId = Objects.requireNonNull(DBOS.stepId()); - - // // logger.debug( - // // "txStep starting: workflowId={} stepId={} stepName={}", workflowId, stepId, stepName); - // // var prevResult = this.checkExecution(workflowId, stepId, stepName); - // // if (prevResult != null) { - // // logger.debug( - // // "txStep cache hit: workflowId={} stepId={} stepName={}", workflowId, stepId, - // stepName); - // // return prevResult.toResult(serializer); - // // } - - // // logger.debug( - // // "txStep executing: workflowId={} stepId={} stepName={}", workflowId, stepId, - // stepName); - // // var conn = openTransaction(); - // // try { - // // var retVal = func.execute(conn); - // // recordOutput(conn, workflowId, stepId, retVal); - // // commit(conn); - // // logger.debug( - // // "txStep succeeded: workflowId={} stepId={} stepName={}", workflowId, stepId, - // stepName); - // // return retVal; - // // } catch (Exception e) { - // // rollback(conn); - // // recordError(workflowId, stepId, e); - // // logger.debug( - // // "txStep failed: workflowId={} stepId={} stepName={} error={}", - // // workflowId, - // // stepId, - // // stepName, - // // e.getMessage()); - // // throw e; - // // } finally { - // // close(conn); - // // } - // // } -} diff --git a/transact/src/main/java/dev/dbos/transact/txstep/PostgresStepFactoryHelpers.java b/transact/src/main/java/dev/dbos/transact/txstep/PostgresStepFactoryHelpers.java new file mode 100644 index 000000000..c4dca2992 --- /dev/null +++ b/transact/src/main/java/dev/dbos/transact/txstep/PostgresStepFactoryHelpers.java @@ -0,0 +1,115 @@ +package dev.dbos.transact.txstep; + +import java.sql.Connection; +import java.sql.SQLException; +import java.util.Objects; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class PostgresStepFactoryHelpers { + + private static final Logger logger = LoggerFactory.getLogger(PostgresStepFactoryHelpers.class); + + public static final String CHECK_SQL_TEMPLATE = + """ + SELECT output, error, serialization + FROM "%s".tx_step_outputs + WHERE workflow_id = ? AND step_id = ? + """; + + public static final String UPSERT_SQL_TEMPLATE = + """ + INSERT INTO "%s".tx_step_outputs + (workflow_id, step_id, output, error, serialization) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT DO NOTHING + """; + + /** + * Verifies that the given connection is to a PostgreSQL database. + * + * @throws IllegalArgumentException if the database is not PostgreSQL + * @throws SQLException if database metadata cannot be read + */ + public static void ensurePostgres(Connection conn) throws SQLException { + var productName = conn.getMetaData().getDatabaseProductName(); + if (!productName.equalsIgnoreCase("PostgreSQL")) { + throw new IllegalArgumentException( + "PostgresStepFactory requires a PostgreSQL datasource, got: " + productName); + } + } + + /** + * Returns {@code true} if the named schema exists in the database. + * + * @throws SQLException if the query fails + */ + public static boolean schemaExists(Connection conn, String schema) throws SQLException { + var sql = "SELECT schema_name FROM information_schema.schemata WHERE schema_name = ?"; + try (var stmt = conn.prepareStatement(sql)) { + stmt.setString(1, Objects.requireNonNull(schema, "schema must not be null")); + try (var rs = stmt.executeQuery()) { + return rs.next(); + } + } + } + + /** + * Creates the named schema if it does not already exist. + * + * @throws SQLException if the DDL fails + */ + public static void ensureSchema(Connection conn, String schema) throws SQLException { + Objects.requireNonNull(schema, "schema must not be null"); + if (!schemaExists(conn, schema)) { + try (var stmt = conn.createStatement()) { + stmt.execute("CREATE SCHEMA IF NOT EXISTS \"%s\"".formatted(schema)); + } + } + } + + /** + * Returns {@code true} if the {@code tx_step_outputs} table exists in the named schema. + * + * @throws SQLException if the query fails + */ + public static boolean tableExists(Connection conn, String schema) throws SQLException { + var sql = "SELECT 1 FROM information_schema.tables WHERE table_schema = ? AND table_name = ?"; + try (var stmt = conn.prepareStatement(sql)) { + stmt.setString(1, Objects.requireNonNull(schema, "schema must not be null")); + stmt.setString(2, Objects.requireNonNull("tx_step_outputs", "tableName must not be null")); + try (var rs = stmt.executeQuery()) { + return rs.next(); + } + } + } + + /** + * Creates the {@code tx_step_outputs} table in the named schema if it does not already exist. + * + * @throws SQLException if the DDL fails + */ + public static void ensureTxOutputTable(Connection conn, String schema) throws SQLException { + Objects.requireNonNull(schema, "schema must not be null"); + if (tableExists(conn, schema)) { + return; + } + logger.debug("Creating tx_step_outputs table in schema={}", schema); + try (var stmt = conn.createStatement()) { + var ddlSql = + """ + CREATE TABLE IF NOT EXISTS "%1$s".tx_step_outputs ( + workflow_id TEXT NOT NULL, + step_id INT NOT NULL, + output TEXT, + error TEXT, + serialization TEXT, + created_at BIGINT NOT NULL DEFAULT (EXTRACT(EPOCH FROM now())*1000)::bigint, + PRIMARY KEY (workflow_id, step_id) + )""" + .formatted(schema); + stmt.execute(ddlSql); + } + } +} From de16d45cc070551b9dc6d1b5c52fbccb0232b764 Mon Sep 17 00:00:00 2001 From: Harry Pierson Date: Tue, 5 May 2026 17:22:47 -0700 Subject: [PATCH 18/29] spotless --- .../main/java/dev/dbos/transact/txstep/JdbcStepFactory.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/transact/src/main/java/dev/dbos/transact/txstep/JdbcStepFactory.java b/transact/src/main/java/dev/dbos/transact/txstep/JdbcStepFactory.java index 2f3c0d736..3e428d4ff 100644 --- a/transact/src/main/java/dev/dbos/transact/txstep/JdbcStepFactory.java +++ b/transact/src/main/java/dev/dbos/transact/txstep/JdbcStepFactory.java @@ -14,7 +14,8 @@ import javax.sql.DataSource; /** - * A {@link PostgresStepFactoryHelpers} implementation backed by plain JDBC {@link Connection} objects. + * A {@link PostgresStepFactoryHelpers} implementation backed by plain JDBC {@link Connection} + * objects. * *

Construct one with a {@link DataSource} pointing at a PostgreSQL database. The constructor * verifies the datasource is PostgreSQL and creates the {@code tx_step_outputs} table if needed. From 73767c65c0d51c009aaa041ecd1bf2106e3415e1 Mon Sep 17 00:00:00 2001 From: Harry Pierson Date: Wed, 6 May 2026 14:02:16 -0700 Subject: [PATCH 19/29] WIP --- .../dbos/transact/jdbi/JdbiStepFactory.java | 12 +- .../dbos/transact/jooq/JooqStepFactory.java | 12 +- .../dbos/transact/txstep/JdbcStepFactory.java | 132 ++++++++++++++++-- .../txstep/PostgresStepFactoryHelpers.java | 115 --------------- 4 files changed, 136 insertions(+), 135 deletions(-) delete mode 100644 transact/src/main/java/dev/dbos/transact/txstep/PostgresStepFactoryHelpers.java diff --git a/transact-jdbi-step-factory/src/main/java/dev/dbos/transact/jdbi/JdbiStepFactory.java b/transact-jdbi-step-factory/src/main/java/dev/dbos/transact/jdbi/JdbiStepFactory.java index c9b302876..8768b68a1 100644 --- a/transact-jdbi-step-factory/src/main/java/dev/dbos/transact/jdbi/JdbiStepFactory.java +++ b/transact-jdbi-step-factory/src/main/java/dev/dbos/transact/jdbi/JdbiStepFactory.java @@ -4,7 +4,7 @@ import dev.dbos.transact.database.SystemDatabase; import dev.dbos.transact.json.DBOSSerializer; import dev.dbos.transact.json.SerializationUtil; -import dev.dbos.transact.txstep.PostgresStepFactoryHelpers; +import dev.dbos.transact.txstep.JdbcStepFactory; import dev.dbos.transact.workflow.internal.StepResult; import java.util.Objects; @@ -97,9 +97,9 @@ public JdbiStepFactory(DBOS dbos, Jdbi jdbi, String schema, DBOSSerializer seria try { jdbi.useHandle( handle -> { - PostgresStepFactoryHelpers.ensurePostgres(handle.getConnection()); - PostgresStepFactoryHelpers.ensureSchema(handle.getConnection(), this.schema); - PostgresStepFactoryHelpers.ensureTxOutputTable(handle.getConnection(), this.schema); + JdbcStepFactory.ensurePostgres(handle.getConnection()); + JdbcStepFactory.ensureSchema(handle.getConnection(), this.schema); + JdbcStepFactory.ensureTxOutputTable(handle.getConnection(), this.schema); }); } catch (Exception e) { if (e instanceof RuntimeException re) { @@ -178,7 +178,7 @@ public void useStep(final HandleConsumer callback, Stri } private Optional checkExecution(String workflowId, int stepId, String stepName) { - var sql = PostgresStepFactoryHelpers.CHECK_SQL_TEMPLATE.formatted(schema); + var sql = JdbcStepFactory.CHECK_SQL_TEMPLATE.formatted(schema); return jdbi.withHandle( h -> h.createQuery(sql) @@ -220,7 +220,7 @@ private void recordResult( if (output != null && error != null) { throw new IllegalArgumentException("attempted to record non null output and error result"); } - var sql = PostgresStepFactoryHelpers.UPSERT_SQL_TEMPLATE.formatted(schema); + var sql = JdbcStepFactory.UPSERT_SQL_TEMPLATE.formatted(schema); handle .createUpdate(sql) .bind(0, workflowId) diff --git a/transact-jooq-step-factory/src/main/java/dev/dbos/transact/jooq/JooqStepFactory.java b/transact-jooq-step-factory/src/main/java/dev/dbos/transact/jooq/JooqStepFactory.java index 8202197c0..a7942a467 100644 --- a/transact-jooq-step-factory/src/main/java/dev/dbos/transact/jooq/JooqStepFactory.java +++ b/transact-jooq-step-factory/src/main/java/dev/dbos/transact/jooq/JooqStepFactory.java @@ -4,7 +4,7 @@ import dev.dbos.transact.database.SystemDatabase; import dev.dbos.transact.json.DBOSSerializer; import dev.dbos.transact.json.SerializationUtil; -import dev.dbos.transact.txstep.PostgresStepFactoryHelpers; +import dev.dbos.transact.txstep.JdbcStepFactory; import dev.dbos.transact.workflow.internal.StepResult; import java.util.Objects; @@ -47,9 +47,9 @@ public JooqStepFactory(DBOS dbos, DSLContext dsl, String schema, DBOSSerializer try { dsl.connection( conn -> { - PostgresStepFactoryHelpers.ensurePostgres(conn); - PostgresStepFactoryHelpers.ensureSchema(conn, this.schema); - PostgresStepFactoryHelpers.ensureTxOutputTable(conn, this.schema); + JdbcStepFactory.ensurePostgres(conn); + JdbcStepFactory.ensureSchema(conn, this.schema); + JdbcStepFactory.ensureTxOutputTable(conn, this.schema); }); } catch (Exception e) { if (e instanceof RuntimeException re) { @@ -95,7 +95,7 @@ public void txStep(TransactionalRunnable transactional, String stepName) { } private Optional checkExecution(String workflowId, int stepId, String stepName) { - var sql = PostgresStepFactoryHelpers.CHECK_SQL_TEMPLATE.formatted(this.schema); + var sql = JdbcStepFactory.CHECK_SQL_TEMPLATE.formatted(this.schema); return dsl.fetchOptional(sql, workflowId, stepId) .map( r -> @@ -132,7 +132,7 @@ private void recordResult( if (output != null && error != null) { throw new IllegalArgumentException("attempted to record non null output and error result"); } - var sql = PostgresStepFactoryHelpers.UPSERT_SQL_TEMPLATE.formatted(schema); + var sql = JdbcStepFactory.UPSERT_SQL_TEMPLATE.formatted(schema); trx.dsl().execute(sql, workflowId, stepId, output, error, serialization); } } diff --git a/transact/src/main/java/dev/dbos/transact/txstep/JdbcStepFactory.java b/transact/src/main/java/dev/dbos/transact/txstep/JdbcStepFactory.java index 3e428d4ff..5dcaa8bcc 100644 --- a/transact/src/main/java/dev/dbos/transact/txstep/JdbcStepFactory.java +++ b/transact/src/main/java/dev/dbos/transact/txstep/JdbcStepFactory.java @@ -4,6 +4,7 @@ import dev.dbos.transact.database.SystemDatabase; import dev.dbos.transact.json.DBOSSerializer; import dev.dbos.transact.json.SerializationUtil; +import dev.dbos.transact.txstep.TransactionalRunnable.WrappedSqlException; import dev.dbos.transact.workflow.internal.StepResult; import java.sql.Connection; @@ -13,6 +14,9 @@ import javax.sql.DataSource; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + /** * A {@link PostgresStepFactoryHelpers} implementation backed by plain JDBC {@link Connection} * objects. @@ -34,6 +38,110 @@ */ public class JdbcStepFactory { + private static final Logger logger = LoggerFactory.getLogger(JdbcStepFactory.class); + + public static final String CHECK_SQL_TEMPLATE = + """ + SELECT output, error, serialization + FROM "%s".tx_step_outputs + WHERE workflow_id = ? AND step_id = ? + """; + + public static final String UPSERT_SQL_TEMPLATE = + """ + INSERT INTO "%s".tx_step_outputs + (workflow_id, step_id, output, error, serialization) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT DO NOTHING + """; + + /** + * Verifies that the given connection is to a PostgreSQL database. + * + * @throws IllegalArgumentException if the database is not PostgreSQL + * @throws SQLException if database metadata cannot be read + */ + public static void ensurePostgres(Connection conn) throws SQLException { + var productName = conn.getMetaData().getDatabaseProductName(); + if (!productName.equalsIgnoreCase("PostgreSQL")) { + throw new IllegalArgumentException( + "PostgresStepFactory requires a PostgreSQL datasource, got: " + productName); + } + } + + /** + * Returns {@code true} if the named schema exists in the database. + * + * @throws SQLException if the query fails + */ + public static boolean schemaExists(Connection conn, String schema) throws SQLException { + var sql = "SELECT schema_name FROM information_schema.schemata WHERE schema_name = ?"; + try (var stmt = conn.prepareStatement(sql)) { + stmt.setString(1, Objects.requireNonNull(schema, "schema must not be null")); + try (var rs = stmt.executeQuery()) { + return rs.next(); + } + } + } + + /** + * Creates the named schema if it does not already exist. + * + * @throws SQLException if the DDL fails + */ + public static void ensureSchema(Connection conn, String schema) throws SQLException { + Objects.requireNonNull(schema, "schema must not be null"); + if (!schemaExists(conn, schema)) { + try (var stmt = conn.createStatement()) { + stmt.execute("CREATE SCHEMA IF NOT EXISTS \"%s\"".formatted(schema)); + } + } + } + + /** + * Returns {@code true} if the {@code tx_step_outputs} table exists in the named schema. + * + * @throws SQLException if the query fails + */ + public static boolean tableExists(Connection conn, String schema) throws SQLException { + var sql = "SELECT 1 FROM information_schema.tables WHERE table_schema = ? AND table_name = ?"; + try (var stmt = conn.prepareStatement(sql)) { + stmt.setString(1, Objects.requireNonNull(schema, "schema must not be null")); + stmt.setString(2, Objects.requireNonNull("tx_step_outputs", "tableName must not be null")); + try (var rs = stmt.executeQuery()) { + return rs.next(); + } + } + } + + /** + * Creates the {@code tx_step_outputs} table in the named schema if it does not already exist. + * + * @throws SQLException if the DDL fails + */ + public static void ensureTxOutputTable(Connection conn, String schema) throws SQLException { + Objects.requireNonNull(schema, "schema must not be null"); + if (tableExists(conn, schema)) { + return; + } + logger.debug("Creating tx_step_outputs table in schema={}", schema); + try (var stmt = conn.createStatement()) { + var ddlSql = + """ + CREATE TABLE IF NOT EXISTS "%1$s".tx_step_outputs ( + workflow_id TEXT NOT NULL, + step_id INT NOT NULL, + output TEXT, + error TEXT, + serialization TEXT, + created_at BIGINT NOT NULL DEFAULT (EXTRACT(EPOCH FROM now())*1000)::bigint, + PRIMARY KEY (workflow_id, step_id) + )""" + .formatted(schema); + stmt.execute(ddlSql); + } + } + private final DBOS dbos; private final DataSource dataSource; private final String schema; @@ -65,9 +173,9 @@ public JdbcStepFactory(DBOS dbos, DataSource dataSource, String schema, DBOSSeri this.serializer = serializer == null ? config.serializer() : serializer; try (var conn = dataSource.getConnection()) { - PostgresStepFactoryHelpers.ensurePostgres(conn); - PostgresStepFactoryHelpers.ensureSchema(conn, this.schema); - PostgresStepFactoryHelpers.ensureTxOutputTable(conn, this.schema); + ensurePostgres(conn); + ensureSchema(conn, this.schema); + ensureTxOutputTable(conn, this.schema); } } @@ -183,7 +291,7 @@ private static void close(Connection conn) { } private Optional checkExecution(String workflowId, int stepId, String stepName) { - var sql = PostgresStepFactoryHelpers.CHECK_SQL_TEMPLATE.formatted(this.schema); + var sql = CHECK_SQL_TEMPLATE.formatted(this.schema); try (var conn = dataSource.getConnection(); var stmt = conn.prepareStatement(sql)) { stmt.setString(1, workflowId); @@ -209,7 +317,8 @@ private Optional checkExecution(String workflowId, int stepId, Strin private void recordOutput(Connection conn, String workflowId, int stepId, R result) { var value = SerializationUtil.serializeValue(result, null, serializer); - recordResult(conn, workflowId, stepId, value.serializedValue(), null, value.serialization()); + recordResult( + conn, schema, workflowId, stepId, value.serializedValue(), null, value.serialization()); } private void recordError(String workflowId, int stepId, X exception) { @@ -218,13 +327,20 @@ private void recordError(String workflowId, int stepId, X dataSource, (Connection conn) -> { recordResult( - conn, workflowId, stepId, null, value.serializedValue(), value.serialization()); + conn, + schema, + workflowId, + stepId, + null, + value.serializedValue(), + value.serialization()); return null; }); } - private void recordResult( + public static void recordResult( Connection conn, + String schema, String workflowId, int stepId, String output, @@ -233,7 +349,7 @@ private void recordResult( if (output != null && error != null) { throw new IllegalArgumentException("attempted to record non null output and error result"); } - var sql = PostgresStepFactoryHelpers.UPSERT_SQL_TEMPLATE.formatted(schema); + var sql = UPSERT_SQL_TEMPLATE.formatted(schema); try (var stmt = conn.prepareStatement(sql)) { stmt.setString(1, workflowId); stmt.setInt(2, stepId); diff --git a/transact/src/main/java/dev/dbos/transact/txstep/PostgresStepFactoryHelpers.java b/transact/src/main/java/dev/dbos/transact/txstep/PostgresStepFactoryHelpers.java deleted file mode 100644 index c4dca2992..000000000 --- a/transact/src/main/java/dev/dbos/transact/txstep/PostgresStepFactoryHelpers.java +++ /dev/null @@ -1,115 +0,0 @@ -package dev.dbos.transact.txstep; - -import java.sql.Connection; -import java.sql.SQLException; -import java.util.Objects; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class PostgresStepFactoryHelpers { - - private static final Logger logger = LoggerFactory.getLogger(PostgresStepFactoryHelpers.class); - - public static final String CHECK_SQL_TEMPLATE = - """ - SELECT output, error, serialization - FROM "%s".tx_step_outputs - WHERE workflow_id = ? AND step_id = ? - """; - - public static final String UPSERT_SQL_TEMPLATE = - """ - INSERT INTO "%s".tx_step_outputs - (workflow_id, step_id, output, error, serialization) - VALUES (?, ?, ?, ?, ?) - ON CONFLICT DO NOTHING - """; - - /** - * Verifies that the given connection is to a PostgreSQL database. - * - * @throws IllegalArgumentException if the database is not PostgreSQL - * @throws SQLException if database metadata cannot be read - */ - public static void ensurePostgres(Connection conn) throws SQLException { - var productName = conn.getMetaData().getDatabaseProductName(); - if (!productName.equalsIgnoreCase("PostgreSQL")) { - throw new IllegalArgumentException( - "PostgresStepFactory requires a PostgreSQL datasource, got: " + productName); - } - } - - /** - * Returns {@code true} if the named schema exists in the database. - * - * @throws SQLException if the query fails - */ - public static boolean schemaExists(Connection conn, String schema) throws SQLException { - var sql = "SELECT schema_name FROM information_schema.schemata WHERE schema_name = ?"; - try (var stmt = conn.prepareStatement(sql)) { - stmt.setString(1, Objects.requireNonNull(schema, "schema must not be null")); - try (var rs = stmt.executeQuery()) { - return rs.next(); - } - } - } - - /** - * Creates the named schema if it does not already exist. - * - * @throws SQLException if the DDL fails - */ - public static void ensureSchema(Connection conn, String schema) throws SQLException { - Objects.requireNonNull(schema, "schema must not be null"); - if (!schemaExists(conn, schema)) { - try (var stmt = conn.createStatement()) { - stmt.execute("CREATE SCHEMA IF NOT EXISTS \"%s\"".formatted(schema)); - } - } - } - - /** - * Returns {@code true} if the {@code tx_step_outputs} table exists in the named schema. - * - * @throws SQLException if the query fails - */ - public static boolean tableExists(Connection conn, String schema) throws SQLException { - var sql = "SELECT 1 FROM information_schema.tables WHERE table_schema = ? AND table_name = ?"; - try (var stmt = conn.prepareStatement(sql)) { - stmt.setString(1, Objects.requireNonNull(schema, "schema must not be null")); - stmt.setString(2, Objects.requireNonNull("tx_step_outputs", "tableName must not be null")); - try (var rs = stmt.executeQuery()) { - return rs.next(); - } - } - } - - /** - * Creates the {@code tx_step_outputs} table in the named schema if it does not already exist. - * - * @throws SQLException if the DDL fails - */ - public static void ensureTxOutputTable(Connection conn, String schema) throws SQLException { - Objects.requireNonNull(schema, "schema must not be null"); - if (tableExists(conn, schema)) { - return; - } - logger.debug("Creating tx_step_outputs table in schema={}", schema); - try (var stmt = conn.createStatement()) { - var ddlSql = - """ - CREATE TABLE IF NOT EXISTS "%1$s".tx_step_outputs ( - workflow_id TEXT NOT NULL, - step_id INT NOT NULL, - output TEXT, - error TEXT, - serialization TEXT, - created_at BIGINT NOT NULL DEFAULT (EXTRACT(EPOCH FROM now())*1000)::bigint, - PRIMARY KEY (workflow_id, step_id) - )""" - .formatted(schema); - stmt.execute(ddlSql); - } - } -} From 2fd3ffecacc527b26e78ada2e1861dd6396a092b Mon Sep 17 00:00:00 2001 From: Harry Pierson Date: Wed, 6 May 2026 14:03:55 -0700 Subject: [PATCH 20/29] cleanup --- .../main/java/dev/dbos/transact/txstep/JdbcStepFactory.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/transact/src/main/java/dev/dbos/transact/txstep/JdbcStepFactory.java b/transact/src/main/java/dev/dbos/transact/txstep/JdbcStepFactory.java index 5dcaa8bcc..b3646e165 100644 --- a/transact/src/main/java/dev/dbos/transact/txstep/JdbcStepFactory.java +++ b/transact/src/main/java/dev/dbos/transact/txstep/JdbcStepFactory.java @@ -4,7 +4,6 @@ import dev.dbos.transact.database.SystemDatabase; import dev.dbos.transact.json.DBOSSerializer; import dev.dbos.transact.json.SerializationUtil; -import dev.dbos.transact.txstep.TransactionalRunnable.WrappedSqlException; import dev.dbos.transact.workflow.internal.StepResult; import java.sql.Connection; @@ -18,8 +17,7 @@ import org.slf4j.LoggerFactory; /** - * A {@link PostgresStepFactoryHelpers} implementation backed by plain JDBC {@link Connection} - * objects. + * A StepFactory implementation backed by plain JDBC {@link Connection} objects. * *

Construct one with a {@link DataSource} pointing at a PostgreSQL database. The constructor * verifies the datasource is PostgreSQL and creates the {@code tx_step_outputs} table if needed. From d982bccbea8cb6353d49e78a1599f9e5a6c0ec34 Mon Sep 17 00:00:00 2001 From: Harry Pierson Date: Wed, 6 May 2026 15:27:09 -0700 Subject: [PATCH 21/29] WIP --- .../dbos/transact/jdbi/JdbiStepFactory.java | 130 +++-------- .../dbos/transact/jooq/JooqStepFactory.java | 112 +++------ .../txstep/AbstractTxStepFactory.java | 215 ++++++++++++++++++ .../dbos/transact/txstep/JdbcStepFactory.java | 210 ++--------------- 4 files changed, 304 insertions(+), 363 deletions(-) create mode 100644 transact/src/main/java/dev/dbos/transact/txstep/AbstractTxStepFactory.java diff --git a/transact-jdbi-step-factory/src/main/java/dev/dbos/transact/jdbi/JdbiStepFactory.java b/transact-jdbi-step-factory/src/main/java/dev/dbos/transact/jdbi/JdbiStepFactory.java index 8768b68a1..87f63c066 100644 --- a/transact-jdbi-step-factory/src/main/java/dev/dbos/transact/jdbi/JdbiStepFactory.java +++ b/transact-jdbi-step-factory/src/main/java/dev/dbos/transact/jdbi/JdbiStepFactory.java @@ -1,14 +1,12 @@ package dev.dbos.transact.jdbi; import dev.dbos.transact.DBOS; -import dev.dbos.transact.database.SystemDatabase; import dev.dbos.transact.json.DBOSSerializer; import dev.dbos.transact.json.SerializationUtil; -import dev.dbos.transact.txstep.JdbcStepFactory; -import dev.dbos.transact.workflow.internal.StepResult; +import dev.dbos.transact.txstep.AbstractTxStepFactory; +import java.sql.SQLException; import java.util.Objects; -import java.util.Optional; import org.jdbi.v3.core.Handle; import org.jdbi.v3.core.HandleCallback; @@ -32,12 +30,18 @@ * }, "myStep"); * } */ -public class JdbiStepFactory { +public class JdbiStepFactory extends AbstractTxStepFactory { - private final DBOS dbos; private final Jdbi jdbi; - private final String schema; - private final DBOSSerializer serializer; + + @Override + protected T withConnection(ConnectionFn fn) { + try { + return jdbi.withHandle(h -> fn.apply(h.getConnection())); + } catch (SQLException e) { + throw new WrappedSqlException(e); + } + } /** * Creates a factory using the schema and serializer from {@code dbos} configuration. @@ -88,25 +92,8 @@ public JdbiStepFactory(DBOS dbos, Jdbi jdbi, DBOSSerializer serializer) { * @throws RuntimeException if the datasource is not PostgreSQL or the schema setup fails */ public JdbiStepFactory(DBOS dbos, Jdbi jdbi, String schema, DBOSSerializer serializer) { - this.dbos = dbos; - this.jdbi = jdbi; - var config = dbos.integration().config(); - this.schema = SystemDatabase.sanitizeSchema(schema == null ? config.databaseSchema() : schema); - this.serializer = serializer == null ? config.serializer() : serializer; - - try { - jdbi.useHandle( - handle -> { - JdbcStepFactory.ensurePostgres(handle.getConnection()); - JdbcStepFactory.ensureSchema(handle.getConnection(), this.schema); - JdbcStepFactory.ensureTxOutputTable(handle.getConnection(), this.schema); - }); - } catch (Exception e) { - if (e instanceof RuntimeException re) { - throw re; - } - throw new RuntimeException(e.getMessage(), e); - } + super(dbos, schema, serializer, () -> jdbi.open().getConnection()); + this.jdbi = Objects.requireNonNull(jdbi); } /** @@ -125,32 +112,17 @@ public JdbiStepFactory(DBOS dbos, Jdbi jdbi, String schema, DBOSSerializer seria * @return the value returned by {@code callback} * @throws X if the callback throws */ - @SuppressWarnings("unchecked") public R inStep(final HandleCallback callback, String stepName) throws X { - - return dbos.runStep( - () -> { - var workflowId = Objects.requireNonNull(DBOS.workflowId()); - int stepId = Objects.requireNonNull(DBOS.stepId()); - - var prevResult = checkExecution(workflowId, stepId, stepName); - if (prevResult.isPresent()) { - return prevResult.get().toResult(serializer); - } - - try { - return jdbi.inTransaction( + return runTxStep( + (wfId, stepId) -> + jdbi.inTransaction( h -> { var result = callback.withHandle(h); - recordOutput(h, workflowId, stepId, result); + recordOutput(h, wfId, stepId, result); return result; - }); - } catch (Exception e) { - recordError(workflowId, stepId, e); - throw (X) e; - } - }, + }), + this::recordError, stepName); } @@ -177,57 +149,29 @@ public void useStep(final HandleConsumer callback, Stri stepName); } - private Optional checkExecution(String workflowId, int stepId, String stepName) { - var sql = JdbcStepFactory.CHECK_SQL_TEMPLATE.formatted(schema); - return jdbi.withHandle( - h -> - h.createQuery(sql) - .bind(0, workflowId) - .bind(1, stepId) - .map( - (rs, ctx) -> - new StepResult( - workflowId, - stepId, - stepName, - rs.getString("output"), - rs.getString("error"), - null, - rs.getString("serialization"))) - .findOne()); - } - private void recordOutput(Handle handle, String workflowId, int stepId, R result) { var value = SerializationUtil.serializeValue(result, null, serializer); - recordResult(handle, workflowId, stepId, value.serializedValue(), null, value.serialization()); + upsertResult( + handle.getConnection(), + schema, + workflowId, + stepId, + value.serializedValue(), + null, + value.serialization()); } private void recordError(String workflowId, int stepId, X exception) { var value = SerializationUtil.serializeError(exception, null, serializer); jdbi.useTransaction( - h -> { - recordResult(h, workflowId, stepId, null, value.serializedValue(), value.serialization()); - }); - } - - private void recordResult( - Handle handle, - String workflowId, - int stepId, - String output, - String error, - String serialization) { - if (output != null && error != null) { - throw new IllegalArgumentException("attempted to record non null output and error result"); - } - var sql = JdbcStepFactory.UPSERT_SQL_TEMPLATE.formatted(schema); - handle - .createUpdate(sql) - .bind(0, workflowId) - .bind(1, stepId) - .bind(2, output) - .bind(3, error) - .bind(4, serialization) - .execute(); + h -> + upsertResult( + h.getConnection(), + schema, + workflowId, + stepId, + null, + value.serializedValue(), + value.serialization())); } } diff --git a/transact-jooq-step-factory/src/main/java/dev/dbos/transact/jooq/JooqStepFactory.java b/transact-jooq-step-factory/src/main/java/dev/dbos/transact/jooq/JooqStepFactory.java index a7942a467..f2a72304d 100644 --- a/transact-jooq-step-factory/src/main/java/dev/dbos/transact/jooq/JooqStepFactory.java +++ b/transact-jooq-step-factory/src/main/java/dev/dbos/transact/jooq/JooqStepFactory.java @@ -1,26 +1,25 @@ package dev.dbos.transact.jooq; import dev.dbos.transact.DBOS; -import dev.dbos.transact.database.SystemDatabase; import dev.dbos.transact.json.DBOSSerializer; import dev.dbos.transact.json.SerializationUtil; -import dev.dbos.transact.txstep.JdbcStepFactory; -import dev.dbos.transact.workflow.internal.StepResult; +import dev.dbos.transact.txstep.AbstractTxStepFactory; import java.util.Objects; -import java.util.Optional; import org.jooq.Configuration; import org.jooq.DSLContext; import org.jooq.TransactionalCallable; import org.jooq.TransactionalRunnable; -public class JooqStepFactory { +public class JooqStepFactory extends AbstractTxStepFactory { - private final DBOS dbos; private final DSLContext dsl; - private final String schema; - private final DBOSSerializer serializer; + + @Override + protected T withConnection(ConnectionFn fn) { + return dsl.connectionResult(conn -> fn.apply(conn)); + } /** Creates a factory using the schema from the DBOS config. */ public JooqStepFactory(DBOS dbos, DSLContext dsl) { @@ -39,49 +38,20 @@ public JooqStepFactory(DBOS dbos, DSLContext dsl, DBOSSerializer serializer) { /** Creates a factory with a custom schema and serializer. */ public JooqStepFactory(DBOS dbos, DSLContext dsl, String schema, DBOSSerializer serializer) { - this.dbos = dbos; - this.dsl = dsl; - var config = dbos.integration().config(); - this.schema = SystemDatabase.sanitizeSchema(schema == null ? config.databaseSchema() : schema); - this.serializer = serializer == null ? config.serializer() : serializer; - try { - dsl.connection( - conn -> { - JdbcStepFactory.ensurePostgres(conn); - JdbcStepFactory.ensureSchema(conn, this.schema); - JdbcStepFactory.ensureTxOutputTable(conn, this.schema); - }); - } catch (Exception e) { - if (e instanceof RuntimeException re) { - throw re; - } - throw new RuntimeException(e); - } + super(dbos, schema, serializer, () -> dsl.configuration().connectionProvider().acquire()); + this.dsl = Objects.requireNonNull(dsl); } public T txStepResult(TransactionalCallable callback, String stepName) { - return dbos.runStep( - () -> { - var workflowId = Objects.requireNonNull(DBOS.workflowId()); - int stepId = Objects.requireNonNull(DBOS.stepId()); - - var prevResult = checkExecution(workflowId, stepId, stepName); - if (prevResult.isPresent()) { - return prevResult.get().toResult(serializer); - } - - try { - return dsl.transactionResult( + return runTxStep( + (wfId, stepId) -> + dsl.transactionResult( trx -> { var result = callback.run(trx); - recordOutput(trx, workflowId, stepId, result); + recordOutput(trx, wfId, stepId, result); return result; - }); - } catch (Exception e) { - recordError(workflowId, stepId, e); - throw e; - } - }, + }), + this::recordError, stepName); } @@ -94,45 +64,35 @@ public void txStep(TransactionalRunnable transactional, String stepName) { stepName); } - private Optional checkExecution(String workflowId, int stepId, String stepName) { - var sql = JdbcStepFactory.CHECK_SQL_TEMPLATE.formatted(this.schema); - return dsl.fetchOptional(sql, workflowId, stepId) - .map( - r -> - new StepResult( + private void recordOutput(Configuration trx, String workflowId, int stepId, R result) { + var value = SerializationUtil.serializeValue(result, null, serializer); + trx.dsl() + .connection( + conn -> + upsertResult( + conn, + schema, workflowId, stepId, - stepName, - r.get("output", String.class), - r.get("error", String.class), + value.serializedValue(), null, - r.get("serialization", String.class))); - } - - private void recordOutput(Configuration trx, String workflowId, int stepId, R result) { - var value = SerializationUtil.serializeValue(result, null, serializer); - recordResult(trx, workflowId, stepId, value.serializedValue(), null, value.serialization()); + value.serialization())); } private void recordError(String workflowId, int stepId, X exception) { var value = SerializationUtil.serializeError(exception, null, serializer); dsl.transaction( trx -> - recordResult( - trx, workflowId, stepId, null, value.serializedValue(), value.serialization())); - } - - private void recordResult( - Configuration trx, - String workflowId, - int stepId, - String output, - String error, - String serialization) { - if (output != null && error != null) { - throw new IllegalArgumentException("attempted to record non null output and error result"); - } - var sql = JdbcStepFactory.UPSERT_SQL_TEMPLATE.formatted(schema); - trx.dsl().execute(sql, workflowId, stepId, output, error, serialization); + trx.dsl() + .connection( + conn -> + upsertResult( + conn, + schema, + workflowId, + stepId, + null, + value.serializedValue(), + value.serialization()))); } } diff --git a/transact/src/main/java/dev/dbos/transact/txstep/AbstractTxStepFactory.java b/transact/src/main/java/dev/dbos/transact/txstep/AbstractTxStepFactory.java new file mode 100644 index 000000000..2f34992eb --- /dev/null +++ b/transact/src/main/java/dev/dbos/transact/txstep/AbstractTxStepFactory.java @@ -0,0 +1,215 @@ +package dev.dbos.transact.txstep; + +import dev.dbos.transact.DBOS; +import dev.dbos.transact.database.SystemDatabase; +import dev.dbos.transact.json.DBOSSerializer; +import dev.dbos.transact.workflow.internal.StepResult; + +import java.sql.Connection; +import java.sql.SQLException; +import java.util.Objects; +import java.util.Optional; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public abstract class AbstractTxStepFactory { + + private static final Logger logger = LoggerFactory.getLogger(AbstractTxStepFactory.class); + + protected static class WrappedSqlException extends RuntimeException { + private final SQLException wrappedException; + + public WrappedSqlException(SQLException wrappedException) { + super(wrappedException.getMessage(), wrappedException); + this.wrappedException = wrappedException; + } + + public SQLException wrappedException() { + return this.wrappedException; + } + } + + protected static void ensurePostgres(Connection conn) throws SQLException { + var productName = conn.getMetaData().getDatabaseProductName(); + if (!productName.equalsIgnoreCase("PostgreSQL")) { + throw new IllegalArgumentException( + "TxStepFactory requires a PostgreSQL datasource, got: " + productName); + } + } + + protected static void ensureSchema(Connection conn, String schema) throws SQLException { + Objects.requireNonNull(schema, "schema must not be null"); + var sql = "SELECT schema_name FROM information_schema.schemata WHERE schema_name = ?"; + try (var stmt = conn.prepareStatement(sql)) { + stmt.setString(1, Objects.requireNonNull(schema, "schema must not be null")); + try (var rs = stmt.executeQuery()) { + if (rs.next()) { + return; + } + } + } + + try (var stmt = conn.createStatement()) { + stmt.execute("CREATE SCHEMA \"%s\"".formatted(schema)); + } + } + + protected static void ensureTxOutputTable(Connection conn, String schema) throws SQLException { + Objects.requireNonNull(schema, "schema must not be null"); + var sql = "SELECT 1 FROM information_schema.tables WHERE table_schema = ? AND table_name = ?"; + try (var stmt = conn.prepareStatement(sql)) { + stmt.setString(1, Objects.requireNonNull(schema, "schema must not be null")); + stmt.setString(2, "tx_step_outputs"); + try (var rs = stmt.executeQuery()) { + if (rs.next()) { + return; + } + } + } + + logger.debug("Creating tx_step_outputs table in schema={}", schema); + try (var stmt = conn.createStatement()) { + stmt.execute( + """ + CREATE TABLE "%1$s".tx_step_outputs ( + workflow_id TEXT NOT NULL, + step_id INT NOT NULL, + output TEXT, + error TEXT, + serialization TEXT, + created_at BIGINT NOT NULL DEFAULT (EXTRACT(EPOCH FROM now())*1000)::bigint, + PRIMARY KEY (workflow_id, step_id) + )""" + .formatted(schema)); + } + } + + protected final DBOS dbos; + protected final String schema; + protected final DBOSSerializer serializer; + + @FunctionalInterface + protected interface ConnectionOpener { + Connection open() throws Exception; + } + + protected AbstractTxStepFactory( + DBOS dbos, String schema, DBOSSerializer serializer, ConnectionOpener opener) { + this.dbos = Objects.requireNonNull(dbos); + var config = dbos.integration().config(); + this.schema = SystemDatabase.sanitizeSchema(schema == null ? config.databaseSchema() : schema); + this.serializer = serializer == null ? config.serializer() : serializer; + try (var conn = opener.open()) { + ensurePostgres(conn); + ensureSchema(conn, this.schema); + ensureTxOutputTable(conn, this.schema); + } catch (Exception e) { + if (e instanceof RuntimeException re) { + throw re; + } + throw new RuntimeException(e); + } + } + + @FunctionalInterface + protected interface ConnectionFn { + T apply(Connection conn) throws SQLException; + } + + protected abstract T withConnection(ConnectionFn fn); + + protected Optional checkExecution(String workflowId, int stepId, String stepName) { + var sql = + """ + SELECT output, error, serialization + FROM "%s".tx_step_outputs + WHERE workflow_id = ? AND step_id = ? + """ + .formatted(schema); + return withConnection( + conn -> { + try (var stmt = conn.prepareStatement(sql)) { + stmt.setString(1, workflowId); + stmt.setInt(2, stepId); + try (var rs = stmt.executeQuery()) { + if (!rs.next()) return Optional.empty(); + return Optional.of( + new StepResult( + workflowId, + stepId, + stepName, + rs.getString("output"), + rs.getString("error"), + null, + rs.getString("serialization"))); + } + } + }); + } + + protected static void upsertResult( + Connection conn, + String schema, + String workflowId, + int stepId, + String output, + String error, + String serialization) { + if (output != null && error != null) { + throw new IllegalArgumentException("attempted to record non null output and error result"); + } + + var sql = + """ + INSERT INTO "%s".tx_step_outputs + (workflow_id, step_id, output, error, serialization) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT DO NOTHING + """ + .formatted(schema); + try (var stmt = conn.prepareStatement(sql)) { + stmt.setString(1, workflowId); + stmt.setInt(2, stepId); + stmt.setString(3, output); + stmt.setString(4, error); + stmt.setString(5, serialization); + stmt.executeUpdate(); + } catch (SQLException e) { + throw new WrappedSqlException(e); + } + } + + @FunctionalInterface + protected interface TxStepFunction { + R execute(String workflowId, int stepId) throws X; + } + + @FunctionalInterface + protected interface ErrorFn { + void record(String workflowId, int stepId, Exception e); + } + + @SuppressWarnings("unchecked") + protected R runTxStep( + TxStepFunction execute, ErrorFn recordError, String stepName) throws X { + return dbos.runStep( + () -> { + var workflowId = Objects.requireNonNull(DBOS.workflowId()); + int stepId = Objects.requireNonNull(DBOS.stepId()); + + var prev = checkExecution(workflowId, stepId, stepName); + if (prev.isPresent()) { + return prev.get().toResult(serializer); + } + + try { + return execute.execute(workflowId, stepId); + } catch (Exception e) { + recordError.record(workflowId, stepId, e); + throw (X) e; + } + }, + stepName); + } +} diff --git a/transact/src/main/java/dev/dbos/transact/txstep/JdbcStepFactory.java b/transact/src/main/java/dev/dbos/transact/txstep/JdbcStepFactory.java index b3646e165..779392fe5 100644 --- a/transact/src/main/java/dev/dbos/transact/txstep/JdbcStepFactory.java +++ b/transact/src/main/java/dev/dbos/transact/txstep/JdbcStepFactory.java @@ -1,21 +1,15 @@ package dev.dbos.transact.txstep; import dev.dbos.transact.DBOS; -import dev.dbos.transact.database.SystemDatabase; import dev.dbos.transact.json.DBOSSerializer; import dev.dbos.transact.json.SerializationUtil; -import dev.dbos.transact.workflow.internal.StepResult; import java.sql.Connection; import java.sql.SQLException; import java.util.Objects; -import java.util.Optional; import javax.sql.DataSource; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - /** * A StepFactory implementation backed by plain JDBC {@link Connection} objects. * @@ -34,117 +28,19 @@ * }, "myStep"); * } */ -public class JdbcStepFactory { - - private static final Logger logger = LoggerFactory.getLogger(JdbcStepFactory.class); - - public static final String CHECK_SQL_TEMPLATE = - """ - SELECT output, error, serialization - FROM "%s".tx_step_outputs - WHERE workflow_id = ? AND step_id = ? - """; - - public static final String UPSERT_SQL_TEMPLATE = - """ - INSERT INTO "%s".tx_step_outputs - (workflow_id, step_id, output, error, serialization) - VALUES (?, ?, ?, ?, ?) - ON CONFLICT DO NOTHING - """; - - /** - * Verifies that the given connection is to a PostgreSQL database. - * - * @throws IllegalArgumentException if the database is not PostgreSQL - * @throws SQLException if database metadata cannot be read - */ - public static void ensurePostgres(Connection conn) throws SQLException { - var productName = conn.getMetaData().getDatabaseProductName(); - if (!productName.equalsIgnoreCase("PostgreSQL")) { - throw new IllegalArgumentException( - "PostgresStepFactory requires a PostgreSQL datasource, got: " + productName); - } - } - - /** - * Returns {@code true} if the named schema exists in the database. - * - * @throws SQLException if the query fails - */ - public static boolean schemaExists(Connection conn, String schema) throws SQLException { - var sql = "SELECT schema_name FROM information_schema.schemata WHERE schema_name = ?"; - try (var stmt = conn.prepareStatement(sql)) { - stmt.setString(1, Objects.requireNonNull(schema, "schema must not be null")); - try (var rs = stmt.executeQuery()) { - return rs.next(); - } - } - } - - /** - * Creates the named schema if it does not already exist. - * - * @throws SQLException if the DDL fails - */ - public static void ensureSchema(Connection conn, String schema) throws SQLException { - Objects.requireNonNull(schema, "schema must not be null"); - if (!schemaExists(conn, schema)) { - try (var stmt = conn.createStatement()) { - stmt.execute("CREATE SCHEMA IF NOT EXISTS \"%s\"".formatted(schema)); - } - } - } +public class JdbcStepFactory extends AbstractTxStepFactory { - /** - * Returns {@code true} if the {@code tx_step_outputs} table exists in the named schema. - * - * @throws SQLException if the query fails - */ - public static boolean tableExists(Connection conn, String schema) throws SQLException { - var sql = "SELECT 1 FROM information_schema.tables WHERE table_schema = ? AND table_name = ?"; - try (var stmt = conn.prepareStatement(sql)) { - stmt.setString(1, Objects.requireNonNull(schema, "schema must not be null")); - stmt.setString(2, Objects.requireNonNull("tx_step_outputs", "tableName must not be null")); - try (var rs = stmt.executeQuery()) { - return rs.next(); - } - } - } + private final DataSource dataSource; - /** - * Creates the {@code tx_step_outputs} table in the named schema if it does not already exist. - * - * @throws SQLException if the DDL fails - */ - public static void ensureTxOutputTable(Connection conn, String schema) throws SQLException { - Objects.requireNonNull(schema, "schema must not be null"); - if (tableExists(conn, schema)) { - return; - } - logger.debug("Creating tx_step_outputs table in schema={}", schema); - try (var stmt = conn.createStatement()) { - var ddlSql = - """ - CREATE TABLE IF NOT EXISTS "%1$s".tx_step_outputs ( - workflow_id TEXT NOT NULL, - step_id INT NOT NULL, - output TEXT, - error TEXT, - serialization TEXT, - created_at BIGINT NOT NULL DEFAULT (EXTRACT(EPOCH FROM now())*1000)::bigint, - PRIMARY KEY (workflow_id, step_id) - )""" - .formatted(schema); - stmt.execute(ddlSql); + @Override + protected T withConnection(ConnectionFn fn) { + try (var conn = dataSource.getConnection()) { + return fn.apply(conn); + } catch (SQLException e) { + throw new WrappedSqlException(e); } } - private final DBOS dbos; - private final DataSource dataSource; - private final String schema; - private final DBOSSerializer serializer; - /** Creates a factory using the schema from the DBOS config. */ public JdbcStepFactory(DBOS dbos, DataSource dataSource) throws SQLException { this(dbos, dataSource, null, null); @@ -164,17 +60,8 @@ public JdbcStepFactory(DBOS dbos, DataSource dataSource, DBOSSerializer serializ /** Creates a factory with a custom schema and serializer. */ public JdbcStepFactory(DBOS dbos, DataSource dataSource, String schema, DBOSSerializer serializer) throws SQLException { - this.dbos = dbos; + super(dbos, schema, serializer, dataSource::getConnection); this.dataSource = Objects.requireNonNull(dataSource); - var config = dbos.integration().config(); - this.schema = SystemDatabase.sanitizeSchema(schema == null ? config.databaseSchema() : schema); - this.serializer = serializer == null ? config.serializer() : serializer; - - try (var conn = dataSource.getConnection()) { - ensurePostgres(conn); - ensureSchema(conn, this.schema); - ensureTxOutputTable(conn, this.schema); - } } @FunctionalInterface @@ -182,32 +69,18 @@ public interface TransactionalFunction { R execute(Connection conn) throws X; } - @SuppressWarnings("unchecked") public R txStep( final TransactionalFunction callback, String stepName) throws X { - return dbos.runStep( - () -> { - var workflowId = Objects.requireNonNull(DBOS.workflowId()); - int stepId = Objects.requireNonNull(DBOS.stepId()); - - var prevResult = checkExecution(workflowId, stepId, stepName); - if (prevResult.isPresent()) { - return prevResult.get().toResult(serializer); - } - - try { - return executeTransaction( + return runTxStep( + (wfId, stepId) -> + executeTransaction( dataSource, c -> { var result = callback.execute(c); - recordOutput(c, workflowId, stepId, result); + recordOutput(c, wfId, stepId, result); return result; - }); - } catch (Exception e) { - recordError(workflowId, stepId, e); - throw (X) e; - } - }, + }), + this::recordError, stepName); } @@ -241,19 +114,6 @@ private static R executeTransaction( } } - static class WrappedSqlException extends RuntimeException { - private final SQLException wrappedException; - - public WrappedSqlException(SQLException wrappedException) { - super(wrappedException.getMessage(), wrappedException); - this.wrappedException = wrappedException; - } - - public SQLException wrappedException() { - return this.wrappedException; - } - } - private static Connection openTransaction(DataSource ds) { try { var conn = ds.getConnection(); @@ -288,31 +148,6 @@ private static void close(Connection conn) { } } - private Optional checkExecution(String workflowId, int stepId, String stepName) { - var sql = CHECK_SQL_TEMPLATE.formatted(this.schema); - try (var conn = dataSource.getConnection(); - var stmt = conn.prepareStatement(sql)) { - stmt.setString(1, workflowId); - stmt.setInt(2, stepId); - try (var rs = stmt.executeQuery()) { - if (rs.next()) { - return Optional.of( - new StepResult( - workflowId, - stepId, - stepName, - rs.getString("output"), - rs.getString("error"), - null, - rs.getString("serialization"))); - } - return Optional.empty(); - } - } catch (SQLException e) { - throw new WrappedSqlException(e); - } - } - private void recordOutput(Connection conn, String workflowId, int stepId, R result) { var value = SerializationUtil.serializeValue(result, null, serializer); recordResult( @@ -344,19 +179,6 @@ public static void recordResult( String output, String error, String serialization) { - if (output != null && error != null) { - throw new IllegalArgumentException("attempted to record non null output and error result"); - } - var sql = UPSERT_SQL_TEMPLATE.formatted(schema); - try (var stmt = conn.prepareStatement(sql)) { - stmt.setString(1, workflowId); - stmt.setInt(2, stepId); - stmt.setString(3, output); - stmt.setString(4, error); - stmt.setString(5, serialization); - stmt.executeUpdate(); - } catch (SQLException e) { - throw new WrappedSqlException(e); - } + upsertResult(conn, schema, workflowId, stepId, output, error, serialization); } } From e967443a4213da4985dbbd1c57ab09f80e3b8dbf Mon Sep 17 00:00:00 2001 From: Harry Pierson Date: Wed, 6 May 2026 15:35:26 -0700 Subject: [PATCH 22/29] WIP --- .../dbos/transact/jdbi/JdbiStepFactory.java | 6 +++--- .../dbos/transact/jooq/JooqStepFactory.java | 4 ++-- .../dbos/transact/txstep/JdbcStepFactory.java | 12 +++++------ ...pFactory.java => PostgresStepFactory.java} | 21 ++++--------------- 4 files changed, 15 insertions(+), 28 deletions(-) rename transact/src/main/java/dev/dbos/transact/txstep/{AbstractTxStepFactory.java => PostgresStepFactory.java} (91%) diff --git a/transact-jdbi-step-factory/src/main/java/dev/dbos/transact/jdbi/JdbiStepFactory.java b/transact-jdbi-step-factory/src/main/java/dev/dbos/transact/jdbi/JdbiStepFactory.java index 87f63c066..2777e7457 100644 --- a/transact-jdbi-step-factory/src/main/java/dev/dbos/transact/jdbi/JdbiStepFactory.java +++ b/transact-jdbi-step-factory/src/main/java/dev/dbos/transact/jdbi/JdbiStepFactory.java @@ -3,7 +3,7 @@ import dev.dbos.transact.DBOS; import dev.dbos.transact.json.DBOSSerializer; import dev.dbos.transact.json.SerializationUtil; -import dev.dbos.transact.txstep.AbstractTxStepFactory; +import dev.dbos.transact.txstep.PostgresStepFactory; import java.sql.SQLException; import java.util.Objects; @@ -30,7 +30,7 @@ * }, "myStep"); * } */ -public class JdbiStepFactory extends AbstractTxStepFactory { +public class JdbiStepFactory extends PostgresStepFactory { private final Jdbi jdbi; @@ -39,7 +39,7 @@ protected T withConnection(ConnectionFn fn) { try { return jdbi.withHandle(h -> fn.apply(h.getConnection())); } catch (SQLException e) { - throw new WrappedSqlException(e); + throw new RuntimeException(e); } } diff --git a/transact-jooq-step-factory/src/main/java/dev/dbos/transact/jooq/JooqStepFactory.java b/transact-jooq-step-factory/src/main/java/dev/dbos/transact/jooq/JooqStepFactory.java index f2a72304d..8149f807d 100644 --- a/transact-jooq-step-factory/src/main/java/dev/dbos/transact/jooq/JooqStepFactory.java +++ b/transact-jooq-step-factory/src/main/java/dev/dbos/transact/jooq/JooqStepFactory.java @@ -3,7 +3,7 @@ import dev.dbos.transact.DBOS; import dev.dbos.transact.json.DBOSSerializer; import dev.dbos.transact.json.SerializationUtil; -import dev.dbos.transact.txstep.AbstractTxStepFactory; +import dev.dbos.transact.txstep.PostgresStepFactory; import java.util.Objects; @@ -12,7 +12,7 @@ import org.jooq.TransactionalCallable; import org.jooq.TransactionalRunnable; -public class JooqStepFactory extends AbstractTxStepFactory { +public class JooqStepFactory extends PostgresStepFactory { private final DSLContext dsl; diff --git a/transact/src/main/java/dev/dbos/transact/txstep/JdbcStepFactory.java b/transact/src/main/java/dev/dbos/transact/txstep/JdbcStepFactory.java index 779392fe5..dcb71d89c 100644 --- a/transact/src/main/java/dev/dbos/transact/txstep/JdbcStepFactory.java +++ b/transact/src/main/java/dev/dbos/transact/txstep/JdbcStepFactory.java @@ -28,7 +28,7 @@ * }, "myStep"); * } */ -public class JdbcStepFactory extends AbstractTxStepFactory { +public class JdbcStepFactory extends PostgresStepFactory { private final DataSource dataSource; @@ -37,7 +37,7 @@ protected T withConnection(ConnectionFn fn) { try (var conn = dataSource.getConnection()) { return fn.apply(conn); } catch (SQLException e) { - throw new WrappedSqlException(e); + throw new RuntimeException(e); } } @@ -120,7 +120,7 @@ private static Connection openTransaction(DataSource ds) { conn.setAutoCommit(false); return conn; } catch (SQLException e) { - throw new WrappedSqlException(e); + throw new RuntimeException(e); } } @@ -128,7 +128,7 @@ private static void commit(Connection conn) { try { conn.commit(); } catch (SQLException e) { - throw new WrappedSqlException(e); + throw new RuntimeException(e); } } @@ -136,7 +136,7 @@ private static void rollback(Connection conn) { try { conn.rollback(); } catch (SQLException e) { - throw new WrappedSqlException(e); + throw new RuntimeException(e); } } @@ -144,7 +144,7 @@ private static void close(Connection conn) { try { conn.close(); } catch (SQLException e) { - throw new WrappedSqlException(e); + throw new RuntimeException(e); } } diff --git a/transact/src/main/java/dev/dbos/transact/txstep/AbstractTxStepFactory.java b/transact/src/main/java/dev/dbos/transact/txstep/PostgresStepFactory.java similarity index 91% rename from transact/src/main/java/dev/dbos/transact/txstep/AbstractTxStepFactory.java rename to transact/src/main/java/dev/dbos/transact/txstep/PostgresStepFactory.java index 2f34992eb..b300a1bb3 100644 --- a/transact/src/main/java/dev/dbos/transact/txstep/AbstractTxStepFactory.java +++ b/transact/src/main/java/dev/dbos/transact/txstep/PostgresStepFactory.java @@ -13,22 +13,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public abstract class AbstractTxStepFactory { +public abstract class PostgresStepFactory { - private static final Logger logger = LoggerFactory.getLogger(AbstractTxStepFactory.class); - - protected static class WrappedSqlException extends RuntimeException { - private final SQLException wrappedException; - - public WrappedSqlException(SQLException wrappedException) { - super(wrappedException.getMessage(), wrappedException); - this.wrappedException = wrappedException; - } - - public SQLException wrappedException() { - return this.wrappedException; - } - } + private static final Logger logger = LoggerFactory.getLogger(PostgresStepFactory.class); protected static void ensurePostgres(Connection conn) throws SQLException { var productName = conn.getMetaData().getDatabaseProductName(); @@ -94,7 +81,7 @@ protected interface ConnectionOpener { Connection open() throws Exception; } - protected AbstractTxStepFactory( + protected PostgresStepFactory( DBOS dbos, String schema, DBOSSerializer serializer, ConnectionOpener opener) { this.dbos = Objects.requireNonNull(dbos); var config = dbos.integration().config(); @@ -176,7 +163,7 @@ protected static void upsertResult( stmt.setString(5, serialization); stmt.executeUpdate(); } catch (SQLException e) { - throw new WrappedSqlException(e); + throw new RuntimeException(e); } } From 44741ee204646a5ba530427f600786beacc5e7b3 Mon Sep 17 00:00:00 2001 From: Harry Pierson Date: Wed, 6 May 2026 15:42:33 -0700 Subject: [PATCH 23/29] abstract recordError --- .../java/dev/dbos/transact/jdbi/JdbiStepFactory.java | 4 ++-- .../java/dev/dbos/transact/jooq/JooqStepFactory.java | 4 ++-- .../dev/dbos/transact/txstep/JdbcStepFactory.java | 6 +++--- .../dev/dbos/transact/txstep/PostgresStepFactory.java | 11 ++++------- 4 files changed, 11 insertions(+), 14 deletions(-) diff --git a/transact-jdbi-step-factory/src/main/java/dev/dbos/transact/jdbi/JdbiStepFactory.java b/transact-jdbi-step-factory/src/main/java/dev/dbos/transact/jdbi/JdbiStepFactory.java index 2777e7457..a3f953942 100644 --- a/transact-jdbi-step-factory/src/main/java/dev/dbos/transact/jdbi/JdbiStepFactory.java +++ b/transact-jdbi-step-factory/src/main/java/dev/dbos/transact/jdbi/JdbiStepFactory.java @@ -122,7 +122,6 @@ public R inStep(final HandleCallback callback, St recordOutput(h, wfId, stepId, result); return result; }), - this::recordError, stepName); } @@ -161,7 +160,8 @@ private void recordOutput(Handle handle, String workflowId, int stepId, R re value.serialization()); } - private void recordError(String workflowId, int stepId, X exception) { + @Override + protected void recordError(String workflowId, int stepId, Exception exception) { var value = SerializationUtil.serializeError(exception, null, serializer); jdbi.useTransaction( h -> diff --git a/transact-jooq-step-factory/src/main/java/dev/dbos/transact/jooq/JooqStepFactory.java b/transact-jooq-step-factory/src/main/java/dev/dbos/transact/jooq/JooqStepFactory.java index 8149f807d..ca350ad8a 100644 --- a/transact-jooq-step-factory/src/main/java/dev/dbos/transact/jooq/JooqStepFactory.java +++ b/transact-jooq-step-factory/src/main/java/dev/dbos/transact/jooq/JooqStepFactory.java @@ -51,7 +51,6 @@ public T txStepResult(TransactionalCallable callback, String stepName) { recordOutput(trx, wfId, stepId, result); return result; }), - this::recordError, stepName); } @@ -79,7 +78,8 @@ private void recordOutput(Configuration trx, String workflowId, int stepId, value.serialization())); } - private void recordError(String workflowId, int stepId, X exception) { + @Override + protected void recordError(String workflowId, int stepId, Exception exception) { var value = SerializationUtil.serializeError(exception, null, serializer); dsl.transaction( trx -> diff --git a/transact/src/main/java/dev/dbos/transact/txstep/JdbcStepFactory.java b/transact/src/main/java/dev/dbos/transact/txstep/JdbcStepFactory.java index dcb71d89c..e9279e663 100644 --- a/transact/src/main/java/dev/dbos/transact/txstep/JdbcStepFactory.java +++ b/transact/src/main/java/dev/dbos/transact/txstep/JdbcStepFactory.java @@ -80,7 +80,6 @@ public R txStep( recordOutput(c, wfId, stepId, result); return result; }), - this::recordError, stepName); } @@ -154,8 +153,9 @@ private void recordOutput(Connection conn, String workflowId, int stepId, R conn, schema, workflowId, stepId, value.serializedValue(), null, value.serialization()); } - private void recordError(String workflowId, int stepId, X exception) { - final var value = SerializationUtil.serializeError(exception, null, serializer); + @Override + protected void recordError(String workflowId, int stepId, Exception exception) { + var value = SerializationUtil.serializeError(exception, null, serializer); executeTransaction( dataSource, (Connection conn) -> { diff --git a/transact/src/main/java/dev/dbos/transact/txstep/PostgresStepFactory.java b/transact/src/main/java/dev/dbos/transact/txstep/PostgresStepFactory.java index b300a1bb3..fc3c2ff53 100644 --- a/transact/src/main/java/dev/dbos/transact/txstep/PostgresStepFactory.java +++ b/transact/src/main/java/dev/dbos/transact/txstep/PostgresStepFactory.java @@ -167,19 +167,16 @@ protected static void upsertResult( } } + protected abstract void recordError(String workflowId, int stepId, Exception exception); + @FunctionalInterface protected interface TxStepFunction { R execute(String workflowId, int stepId) throws X; } - @FunctionalInterface - protected interface ErrorFn { - void record(String workflowId, int stepId, Exception e); - } - @SuppressWarnings("unchecked") protected R runTxStep( - TxStepFunction execute, ErrorFn recordError, String stepName) throws X { + TxStepFunction execute, String stepName) throws X { return dbos.runStep( () -> { var workflowId = Objects.requireNonNull(DBOS.workflowId()); @@ -193,7 +190,7 @@ protected R runTxStep( try { return execute.execute(workflowId, stepId); } catch (Exception e) { - recordError.record(workflowId, stepId, e); + recordError(workflowId, stepId, e); throw (X) e; } }, From 0a7fa1b7dd37feb5fbc639f369ea68f3986577fb Mon Sep 17 00:00:00 2001 From: Harry Pierson Date: Wed, 6 May 2026 15:42:47 -0700 Subject: [PATCH 24/29] spotless --- .../java/dev/dbos/transact/txstep/PostgresStepFactory.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/transact/src/main/java/dev/dbos/transact/txstep/PostgresStepFactory.java b/transact/src/main/java/dev/dbos/transact/txstep/PostgresStepFactory.java index fc3c2ff53..0eded1ba2 100644 --- a/transact/src/main/java/dev/dbos/transact/txstep/PostgresStepFactory.java +++ b/transact/src/main/java/dev/dbos/transact/txstep/PostgresStepFactory.java @@ -175,8 +175,8 @@ protected interface TxStepFunction { } @SuppressWarnings("unchecked") - protected R runTxStep( - TxStepFunction execute, String stepName) throws X { + protected R runTxStep(TxStepFunction execute, String stepName) + throws X { return dbos.runStep( () -> { var workflowId = Objects.requireNonNull(DBOS.workflowId()); From 97c6abbe9cc816c91b7233e672305d688bf30d9b Mon Sep 17 00:00:00 2001 From: Harry Pierson Date: Wed, 6 May 2026 16:04:48 -0700 Subject: [PATCH 25/29] abstract checkExecution --- .../dbos/transact/jdbi/JdbiStepFactory.java | 32 +++++++++---- .../dbos/transact/jooq/JooqStepFactory.java | 16 ++++++- .../dbos/transact/txstep/JdbcStepFactory.java | 34 ++++++++++---- .../transact/txstep/PostgresStepFactory.java | 47 ++++--------------- 4 files changed, 71 insertions(+), 58 deletions(-) diff --git a/transact-jdbi-step-factory/src/main/java/dev/dbos/transact/jdbi/JdbiStepFactory.java b/transact-jdbi-step-factory/src/main/java/dev/dbos/transact/jdbi/JdbiStepFactory.java index a3f953942..467e4a84c 100644 --- a/transact-jdbi-step-factory/src/main/java/dev/dbos/transact/jdbi/JdbiStepFactory.java +++ b/transact-jdbi-step-factory/src/main/java/dev/dbos/transact/jdbi/JdbiStepFactory.java @@ -4,9 +4,10 @@ import dev.dbos.transact.json.DBOSSerializer; import dev.dbos.transact.json.SerializationUtil; import dev.dbos.transact.txstep.PostgresStepFactory; +import dev.dbos.transact.workflow.internal.StepResult; -import java.sql.SQLException; import java.util.Objects; +import java.util.Optional; import org.jdbi.v3.core.Handle; import org.jdbi.v3.core.HandleCallback; @@ -34,15 +35,6 @@ public class JdbiStepFactory extends PostgresStepFactory { private final Jdbi jdbi; - @Override - protected T withConnection(ConnectionFn fn) { - try { - return jdbi.withHandle(h -> fn.apply(h.getConnection())); - } catch (SQLException e) { - throw new RuntimeException(e); - } - } - /** * Creates a factory using the schema and serializer from {@code dbos} configuration. * @@ -148,6 +140,26 @@ public void useStep(final HandleConsumer callback, Stri stepName); } + @Override + protected Optional checkExecution(String workflowId, int stepId, String stepName) { + return jdbi.withHandle( + h -> + h.createQuery(checkSql()) + .bind(0, workflowId) + .bind(1, stepId) + .map( + (rs, ctx) -> + new StepResult( + workflowId, + stepId, + stepName, + rs.getString("output"), + rs.getString("error"), + null, + rs.getString("serialization"))) + .findFirst()); + } + private void recordOutput(Handle handle, String workflowId, int stepId, R result) { var value = SerializationUtil.serializeValue(result, null, serializer); upsertResult( diff --git a/transact-jooq-step-factory/src/main/java/dev/dbos/transact/jooq/JooqStepFactory.java b/transact-jooq-step-factory/src/main/java/dev/dbos/transact/jooq/JooqStepFactory.java index ca350ad8a..c7d7172a7 100644 --- a/transact-jooq-step-factory/src/main/java/dev/dbos/transact/jooq/JooqStepFactory.java +++ b/transact-jooq-step-factory/src/main/java/dev/dbos/transact/jooq/JooqStepFactory.java @@ -4,8 +4,10 @@ import dev.dbos.transact.json.DBOSSerializer; import dev.dbos.transact.json.SerializationUtil; import dev.dbos.transact.txstep.PostgresStepFactory; +import dev.dbos.transact.workflow.internal.StepResult; import java.util.Objects; +import java.util.Optional; import org.jooq.Configuration; import org.jooq.DSLContext; @@ -17,8 +19,18 @@ public class JooqStepFactory extends PostgresStepFactory { private final DSLContext dsl; @Override - protected T withConnection(ConnectionFn fn) { - return dsl.connectionResult(conn -> fn.apply(conn)); + protected Optional checkExecution(String workflowId, int stepId, String stepName) { + return dsl.fetchOptional(checkSql(), workflowId, stepId) + .map( + r -> + new StepResult( + workflowId, + stepId, + stepName, + r.get("output", String.class), + r.get("error", String.class), + null, + r.get("serialization", String.class))); } /** Creates a factory using the schema from the DBOS config. */ diff --git a/transact/src/main/java/dev/dbos/transact/txstep/JdbcStepFactory.java b/transact/src/main/java/dev/dbos/transact/txstep/JdbcStepFactory.java index e9279e663..6a0a39690 100644 --- a/transact/src/main/java/dev/dbos/transact/txstep/JdbcStepFactory.java +++ b/transact/src/main/java/dev/dbos/transact/txstep/JdbcStepFactory.java @@ -3,10 +3,12 @@ import dev.dbos.transact.DBOS; import dev.dbos.transact.json.DBOSSerializer; import dev.dbos.transact.json.SerializationUtil; +import dev.dbos.transact.workflow.internal.StepResult; import java.sql.Connection; import java.sql.SQLException; import java.util.Objects; +import java.util.Optional; import javax.sql.DataSource; @@ -32,15 +34,6 @@ public class JdbcStepFactory extends PostgresStepFactory { private final DataSource dataSource; - @Override - protected T withConnection(ConnectionFn fn) { - try (var conn = dataSource.getConnection()) { - return fn.apply(conn); - } catch (SQLException e) { - throw new RuntimeException(e); - } - } - /** Creates a factory using the schema from the DBOS config. */ public JdbcStepFactory(DBOS dbos, DataSource dataSource) throws SQLException { this(dbos, dataSource, null, null); @@ -64,6 +57,29 @@ public JdbcStepFactory(DBOS dbos, DataSource dataSource, String schema, DBOSSeri this.dataSource = Objects.requireNonNull(dataSource); } + @Override + protected Optional checkExecution(String workflowId, int stepId, String stepName) { + try (var conn = dataSource.getConnection(); + var stmt = conn.prepareStatement(checkSql())) { + stmt.setString(1, workflowId); + stmt.setInt(2, stepId); + try (var rs = stmt.executeQuery()) { + if (!rs.next()) return Optional.empty(); + return Optional.of( + new StepResult( + workflowId, + stepId, + stepName, + rs.getString("output"), + rs.getString("error"), + null, + rs.getString("serialization"))); + } + } catch (SQLException e) { + throw new RuntimeException(e); + } + } + @FunctionalInterface public interface TransactionalFunction { R execute(Connection conn) throws X; diff --git a/transact/src/main/java/dev/dbos/transact/txstep/PostgresStepFactory.java b/transact/src/main/java/dev/dbos/transact/txstep/PostgresStepFactory.java index 0eded1ba2..fa3fac0fc 100644 --- a/transact/src/main/java/dev/dbos/transact/txstep/PostgresStepFactory.java +++ b/transact/src/main/java/dev/dbos/transact/txstep/PostgresStepFactory.java @@ -78,7 +78,7 @@ PRIMARY KEY (workflow_id, step_id) @FunctionalInterface protected interface ConnectionOpener { - Connection open() throws Exception; + Connection open() throws SQLException; } protected PostgresStepFactory( @@ -91,50 +91,25 @@ protected PostgresStepFactory( ensurePostgres(conn); ensureSchema(conn, this.schema); ensureTxOutputTable(conn, this.schema); - } catch (Exception e) { - if (e instanceof RuntimeException re) { - throw re; - } + } catch (SQLException e) { throw new RuntimeException(e); } } - @FunctionalInterface - protected interface ConnectionFn { - T apply(Connection conn) throws SQLException; - } - - protected abstract T withConnection(ConnectionFn fn); - - protected Optional checkExecution(String workflowId, int stepId, String stepName) { - var sql = - """ + protected String checkSql() { + return """ SELECT output, error, serialization FROM "%s".tx_step_outputs WHERE workflow_id = ? AND step_id = ? """ - .formatted(schema); - return withConnection( - conn -> { - try (var stmt = conn.prepareStatement(sql)) { - stmt.setString(1, workflowId); - stmt.setInt(2, stepId); - try (var rs = stmt.executeQuery()) { - if (!rs.next()) return Optional.empty(); - return Optional.of( - new StepResult( - workflowId, - stepId, - stepName, - rs.getString("output"), - rs.getString("error"), - null, - rs.getString("serialization"))); - } - } - }); + .formatted(schema); } + protected abstract Optional checkExecution( + String workflowId, int stepId, String stepName); + + protected abstract void recordError(String workflowId, int stepId, Exception exception); + protected static void upsertResult( Connection conn, String schema, @@ -167,8 +142,6 @@ protected static void upsertResult( } } - protected abstract void recordError(String workflowId, int stepId, Exception exception); - @FunctionalInterface protected interface TxStepFunction { R execute(String workflowId, int stepId) throws X; From 6f9be8c83e76293cb1f332a61c2c9172f01ecbf5 Mon Sep 17 00:00:00 2001 From: Harry Pierson Date: Wed, 6 May 2026 16:23:21 -0700 Subject: [PATCH 26/29] push upsertResult down --- .../dbos/transact/jdbi/JdbiStepFactory.java | 36 +++++++++------- .../dbos/transact/jooq/JooqStepFactory.java | 41 +++++++++--------- .../dbos/transact/txstep/JdbcStepFactory.java | 25 +++++------ .../transact/txstep/PostgresStepFactory.java | 42 +++++-------------- 4 files changed, 62 insertions(+), 82 deletions(-) diff --git a/transact-jdbi-step-factory/src/main/java/dev/dbos/transact/jdbi/JdbiStepFactory.java b/transact-jdbi-step-factory/src/main/java/dev/dbos/transact/jdbi/JdbiStepFactory.java index 467e4a84c..d2c3a5af5 100644 --- a/transact-jdbi-step-factory/src/main/java/dev/dbos/transact/jdbi/JdbiStepFactory.java +++ b/transact-jdbi-step-factory/src/main/java/dev/dbos/transact/jdbi/JdbiStepFactory.java @@ -162,14 +162,7 @@ protected Optional checkExecution(String workflowId, int stepId, Str private void recordOutput(Handle handle, String workflowId, int stepId, R result) { var value = SerializationUtil.serializeValue(result, null, serializer); - upsertResult( - handle.getConnection(), - schema, - workflowId, - stepId, - value.serializedValue(), - null, - value.serialization()); + recordResult(handle, workflowId, stepId, value.serializedValue(), null, value.serialization()); } @Override @@ -177,13 +170,24 @@ protected void recordError(String workflowId, int stepId, Exception exception) { var value = SerializationUtil.serializeError(exception, null, serializer); jdbi.useTransaction( h -> - upsertResult( - h.getConnection(), - schema, - workflowId, - stepId, - null, - value.serializedValue(), - value.serialization())); + recordResult( + h, workflowId, stepId, null, value.serializedValue(), value.serialization())); + } + + private void recordResult( + Handle handle, + String workflowId, + int stepId, + String output, + String error, + String serialization) { + handle + .createUpdate(upsertSql()) + .bind(0, workflowId) + .bind(1, stepId) + .bind(2, output) + .bind(3, error) + .bind(4, serialization) + .execute(); } } diff --git a/transact-jooq-step-factory/src/main/java/dev/dbos/transact/jooq/JooqStepFactory.java b/transact-jooq-step-factory/src/main/java/dev/dbos/transact/jooq/JooqStepFactory.java index c7d7172a7..759807509 100644 --- a/transact-jooq-step-factory/src/main/java/dev/dbos/transact/jooq/JooqStepFactory.java +++ b/transact-jooq-step-factory/src/main/java/dev/dbos/transact/jooq/JooqStepFactory.java @@ -77,17 +77,8 @@ public void txStep(TransactionalRunnable transactional, String stepName) { private void recordOutput(Configuration trx, String workflowId, int stepId, R result) { var value = SerializationUtil.serializeValue(result, null, serializer); - trx.dsl() - .connection( - conn -> - upsertResult( - conn, - schema, - workflowId, - stepId, - value.serializedValue(), - null, - value.serialization())); + recordResult( + trx.dsl(), workflowId, stepId, value.serializedValue(), null, value.serialization()); } @Override @@ -95,16 +86,22 @@ protected void recordError(String workflowId, int stepId, Exception exception) { var value = SerializationUtil.serializeError(exception, null, serializer); dsl.transaction( trx -> - trx.dsl() - .connection( - conn -> - upsertResult( - conn, - schema, - workflowId, - stepId, - null, - value.serializedValue(), - value.serialization()))); + recordResult( + trx.dsl(), + workflowId, + stepId, + null, + value.serializedValue(), + value.serialization())); + } + + private void recordResult( + DSLContext ctx, + String workflowId, + int stepId, + String output, + String error, + String serialization) { + ctx.execute(upsertSql(), workflowId, stepId, output, error, serialization); } } diff --git a/transact/src/main/java/dev/dbos/transact/txstep/JdbcStepFactory.java b/transact/src/main/java/dev/dbos/transact/txstep/JdbcStepFactory.java index 6a0a39690..6b7f55ef2 100644 --- a/transact/src/main/java/dev/dbos/transact/txstep/JdbcStepFactory.java +++ b/transact/src/main/java/dev/dbos/transact/txstep/JdbcStepFactory.java @@ -165,8 +165,7 @@ private static void close(Connection conn) { private void recordOutput(Connection conn, String workflowId, int stepId, R result) { var value = SerializationUtil.serializeValue(result, null, serializer); - recordResult( - conn, schema, workflowId, stepId, value.serializedValue(), null, value.serialization()); + recordResult(conn, workflowId, stepId, value.serializedValue(), null, value.serialization()); } @Override @@ -176,25 +175,27 @@ protected void recordError(String workflowId, int stepId, Exception exception) { dataSource, (Connection conn) -> { recordResult( - conn, - schema, - workflowId, - stepId, - null, - value.serializedValue(), - value.serialization()); + conn, workflowId, stepId, null, value.serializedValue(), value.serialization()); return null; }); } - public static void recordResult( + private void recordResult( Connection conn, - String schema, String workflowId, int stepId, String output, String error, String serialization) { - upsertResult(conn, schema, workflowId, stepId, output, error, serialization); + try (var stmt = conn.prepareStatement(upsertSql())) { + stmt.setString(1, workflowId); + stmt.setInt(2, stepId); + stmt.setString(3, output); + stmt.setString(4, error); + stmt.setString(5, serialization); + stmt.executeUpdate(); + } catch (SQLException e) { + throw new RuntimeException(e); + } } } diff --git a/transact/src/main/java/dev/dbos/transact/txstep/PostgresStepFactory.java b/transact/src/main/java/dev/dbos/transact/txstep/PostgresStepFactory.java index fa3fac0fc..7673f1324 100644 --- a/transact/src/main/java/dev/dbos/transact/txstep/PostgresStepFactory.java +++ b/transact/src/main/java/dev/dbos/transact/txstep/PostgresStepFactory.java @@ -96,6 +96,16 @@ protected PostgresStepFactory( } } + protected String upsertSql() { + return """ + INSERT INTO "%s".tx_step_outputs + (workflow_id, step_id, output, error, serialization) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT DO NOTHING + """ + .formatted(schema); + } + protected String checkSql() { return """ SELECT output, error, serialization @@ -110,38 +120,6 @@ protected abstract Optional checkExecution( protected abstract void recordError(String workflowId, int stepId, Exception exception); - protected static void upsertResult( - Connection conn, - String schema, - String workflowId, - int stepId, - String output, - String error, - String serialization) { - if (output != null && error != null) { - throw new IllegalArgumentException("attempted to record non null output and error result"); - } - - var sql = - """ - INSERT INTO "%s".tx_step_outputs - (workflow_id, step_id, output, error, serialization) - VALUES (?, ?, ?, ?, ?) - ON CONFLICT DO NOTHING - """ - .formatted(schema); - try (var stmt = conn.prepareStatement(sql)) { - stmt.setString(1, workflowId); - stmt.setInt(2, stepId); - stmt.setString(3, output); - stmt.setString(4, error); - stmt.setString(5, serialization); - stmt.executeUpdate(); - } catch (SQLException e) { - throw new RuntimeException(e); - } - } - @FunctionalInterface protected interface TxStepFunction { R execute(String workflowId, int stepId) throws X; From deb821f5c4c46b5f8d4d28e21fa2a47a14f29caa Mon Sep 17 00:00:00 2001 From: Harry Pierson Date: Wed, 6 May 2026 16:51:14 -0700 Subject: [PATCH 27/29] safely/safeGet --- .../dbos/transact/txstep/JdbcStepFactory.java | 42 +++++++------------ 1 file changed, 16 insertions(+), 26 deletions(-) diff --git a/transact/src/main/java/dev/dbos/transact/txstep/JdbcStepFactory.java b/transact/src/main/java/dev/dbos/transact/txstep/JdbcStepFactory.java index 6b7f55ef2..3c18f9438 100644 --- a/transact/src/main/java/dev/dbos/transact/txstep/JdbcStepFactory.java +++ b/transact/src/main/java/dev/dbos/transact/txstep/JdbcStepFactory.java @@ -1,6 +1,8 @@ package dev.dbos.transact.txstep; import dev.dbos.transact.DBOS; +import dev.dbos.transact.execution.ThrowingRunnable; +import dev.dbos.transact.execution.ThrowingSupplier; import dev.dbos.transact.json.DBOSSerializer; import dev.dbos.transact.json.SerializationUtil; import dev.dbos.transact.workflow.internal.StepResult; @@ -116,48 +118,36 @@ public void txStep(final TransactionalRunnable callback private static R executeTransaction( final DataSource ds, TransactionalFunction func) throws X { - var conn = openTransaction(ds); + var conn = + safeGet( + () -> { + var c = ds.getConnection(); + c.setAutoCommit(false); + return c; + }); try { var result = func.execute(conn); - commit(conn); + safely(conn::commit); return result; } catch (Exception e) { - rollback(conn); + safely(conn::rollback); throw e; } finally { - close(conn); + safely(conn::close); } } - private static Connection openTransaction(DataSource ds) { + private static void safely(ThrowingRunnable op) { try { - var conn = ds.getConnection(); - conn.setAutoCommit(false); - return conn; + op.execute(); } catch (SQLException e) { throw new RuntimeException(e); } } - private static void commit(Connection conn) { + private static T safeGet(ThrowingSupplier supplier) { try { - conn.commit(); - } catch (SQLException e) { - throw new RuntimeException(e); - } - } - - private static void rollback(Connection conn) { - try { - conn.rollback(); - } catch (SQLException e) { - throw new RuntimeException(e); - } - } - - private static void close(Connection conn) { - try { - conn.close(); + return supplier.execute(); } catch (SQLException e) { throw new RuntimeException(e); } From 6eb881b4393cc8d4f9f15933f5f07462292ce97d Mon Sep 17 00:00:00 2001 From: Harry Pierson Date: Wed, 6 May 2026 17:17:03 -0700 Subject: [PATCH 28/29] java docs --- .../dbos/transact/jdbi/JdbiStepFactory.java | 2 +- .../dbos/transact/jooq/JooqStepFactory.java | 89 ++++++++++--- .../dbos/transact/txstep/JdbcStepFactory.java | 57 +++++++-- .../transact/txstep/PostgresStepFactory.java | 119 +++++++----------- 4 files changed, 170 insertions(+), 97 deletions(-) diff --git a/transact-jdbi-step-factory/src/main/java/dev/dbos/transact/jdbi/JdbiStepFactory.java b/transact-jdbi-step-factory/src/main/java/dev/dbos/transact/jdbi/JdbiStepFactory.java index d2c3a5af5..779f737c7 100644 --- a/transact-jdbi-step-factory/src/main/java/dev/dbos/transact/jdbi/JdbiStepFactory.java +++ b/transact-jdbi-step-factory/src/main/java/dev/dbos/transact/jdbi/JdbiStepFactory.java @@ -157,7 +157,7 @@ protected Optional checkExecution(String workflowId, int stepId, Str rs.getString("error"), null, rs.getString("serialization"))) - .findFirst()); + .findOne()); } private void recordOutput(Handle handle, String workflowId, int stepId, R result) { diff --git a/transact-jooq-step-factory/src/main/java/dev/dbos/transact/jooq/JooqStepFactory.java b/transact-jooq-step-factory/src/main/java/dev/dbos/transact/jooq/JooqStepFactory.java index 759807509..225fb8511 100644 --- a/transact-jooq-step-factory/src/main/java/dev/dbos/transact/jooq/JooqStepFactory.java +++ b/transact-jooq-step-factory/src/main/java/dev/dbos/transact/jooq/JooqStepFactory.java @@ -14,25 +14,28 @@ import org.jooq.TransactionalCallable; import org.jooq.TransactionalRunnable; +/** + * Runs idempotent transactional steps inside DBOS workflows using jOOQ {@link DSLContext} objects. + * + *

Construct one with a {@link DSLContext} connected to a PostgreSQL database. The constructor + * verifies the datasource is PostgreSQL and creates the {@code tx_step_outputs} table if needed. + * Lambdas passed to {@link #txStepResult} or {@link #txStep} receive a jOOQ {@link + * org.jooq.Configuration} with a transaction already open; they must not commit or close the + * underlying connection themselves. + * + * 

{@code
+ * JooqStepFactory factory = new JooqStepFactory(dbos, dslContext);
+ *
+ * // inside a @Workflow method:
+ * int count = factory.txStepResult(trx -> {
+ *     return trx.dsl().insertInto(...).execute();
+ * }, "myStep");
+ * }
+ */ public class JooqStepFactory extends PostgresStepFactory { private final DSLContext dsl; - @Override - protected Optional checkExecution(String workflowId, int stepId, String stepName) { - return dsl.fetchOptional(checkSql(), workflowId, stepId) - .map( - r -> - new StepResult( - workflowId, - stepId, - stepName, - r.get("output", String.class), - r.get("error", String.class), - null, - r.get("serialization", String.class))); - } - /** Creates a factory using the schema from the DBOS config. */ public JooqStepFactory(DBOS dbos, DSLContext dsl) { this(dbos, dsl, null, null); @@ -48,12 +51,39 @@ public JooqStepFactory(DBOS dbos, DSLContext dsl, DBOSSerializer serializer) { this(dbos, dsl, null, serializer); } - /** Creates a factory with a custom schema and serializer. */ + /** + * Creates a factory with a custom schema and serializer. + * + *

Connects to the database immediately to verify it is PostgreSQL and to create the {@code + * tx_step_outputs} table in the given schema if it does not already exist. + * + * @param dbos the DBOS runtime instance + * @param dsl a DSLContext connected to a PostgreSQL database + * @param schema the PostgreSQL schema to use for {@code tx_step_outputs}; {@code null} uses the + * schema from {@code dbos} configuration + * @param serializer the serializer to use for step outputs; {@code null} uses the serializer from + * {@code dbos} configuration + * @throws RuntimeException if the datasource is not PostgreSQL or the schema setup fails + */ public JooqStepFactory(DBOS dbos, DSLContext dsl, String schema, DBOSSerializer serializer) { super(dbos, schema, serializer, () -> dsl.configuration().connectionProvider().acquire()); this.dsl = Objects.requireNonNull(dsl); } + /** + * Executes {@code callback} as an idempotent DBOS step inside a jOOQ transaction. + * + *

If a result for this step is already recorded (e.g. on workflow retry), the callback is + * skipped and the cached result is returned. Otherwise the callback runs inside an open + * transaction; the output is recorded atomically with the database work so the step is + * exactly-once on success. + * + * @param the return type of the callback + * @param callback the database work to perform; receives a jOOQ {@link org.jooq.Configuration} + * with an open transaction and must not commit or close the underlying connection + * @param stepName a stable name that identifies this step within the workflow + * @return the value returned by {@code callback} + */ public T txStepResult(TransactionalCallable callback, String stepName) { return runTxStep( (wfId, stepId) -> @@ -66,6 +96,18 @@ public T txStepResult(TransactionalCallable callback, String stepName) { stepName); } + /** + * Executes {@code transactional} as an idempotent DBOS step inside a jOOQ transaction, with no + * return value. + * + *

Behaves identically to {@link #txStepResult} but accepts a {@link TransactionalRunnable} for + * callers that do not need to return a result. + * + * @param transactional the database work to perform; receives a jOOQ {@link + * org.jooq.Configuration} with an open transaction and must not commit or close the + * underlying connection + * @param stepName a stable name that identifies this step within the workflow + */ public void txStep(TransactionalRunnable transactional, String stepName) { txStepResult( c -> { @@ -75,6 +117,21 @@ public void txStep(TransactionalRunnable transactional, String stepName) { stepName); } + @Override + protected Optional checkExecution(String workflowId, int stepId, String stepName) { + return dsl.fetchOptional(checkSql(), workflowId, stepId) + .map( + r -> + new StepResult( + workflowId, + stepId, + stepName, + r.get("output", String.class), + r.get("error", String.class), + null, + r.get("serialization", String.class))); + } + private void recordOutput(Configuration trx, String workflowId, int stepId, R result) { var value = SerializationUtil.serializeValue(result, null, serializer); recordResult( diff --git a/transact/src/main/java/dev/dbos/transact/txstep/JdbcStepFactory.java b/transact/src/main/java/dev/dbos/transact/txstep/JdbcStepFactory.java index 3c18f9438..da17e8efc 100644 --- a/transact/src/main/java/dev/dbos/transact/txstep/JdbcStepFactory.java +++ b/transact/src/main/java/dev/dbos/transact/txstep/JdbcStepFactory.java @@ -37,24 +37,36 @@ public class JdbcStepFactory extends PostgresStepFactory { private final DataSource dataSource; /** Creates a factory using the schema from the DBOS config. */ - public JdbcStepFactory(DBOS dbos, DataSource dataSource) throws SQLException { + public JdbcStepFactory(DBOS dbos, DataSource dataSource) { this(dbos, dataSource, null, null); } /** Creates a factory using a custom schema for {@code tx_step_outputs}. */ - public JdbcStepFactory(DBOS dbos, DataSource dataSource, String schema) throws SQLException { + public JdbcStepFactory(DBOS dbos, DataSource dataSource, String schema) { this(dbos, dataSource, schema, null); } /** Creates a factory using a custom serializer. */ - public JdbcStepFactory(DBOS dbos, DataSource dataSource, DBOSSerializer serializer) - throws SQLException { + public JdbcStepFactory(DBOS dbos, DataSource dataSource, DBOSSerializer serializer) { this(dbos, dataSource, null, serializer); } - /** Creates a factory with a custom schema and serializer. */ - public JdbcStepFactory(DBOS dbos, DataSource dataSource, String schema, DBOSSerializer serializer) - throws SQLException { + /** + * Creates a factory with a custom schema and serializer. + * + *

Connects to the database immediately to verify it is PostgreSQL and to create the {@code + * tx_step_outputs} table in the given schema if it does not already exist. + * + * @param dbos the DBOS runtime instance + * @param dataSource a DataSource connected to a PostgreSQL database + * @param schema the PostgreSQL schema to use for {@code tx_step_outputs}; {@code null} uses the + * schema from {@code dbos} configuration + * @param serializer the serializer to use for step outputs; {@code null} uses the serializer from + * {@code dbos} configuration + * @throws RuntimeException if the datasource is not PostgreSQL or the schema setup fails + */ + public JdbcStepFactory( + DBOS dbos, DataSource dataSource, String schema, DBOSSerializer serializer) { super(dbos, schema, serializer, dataSource::getConnection); this.dataSource = Objects.requireNonNull(dataSource); } @@ -82,11 +94,28 @@ protected Optional checkExecution(String workflowId, int stepId, Str } } + /** Database work that runs inside a JDBC transaction and returns a result. */ @FunctionalInterface public interface TransactionalFunction { R execute(Connection conn) throws X; } + /** + * Executes {@code callback} as an idempotent DBOS step inside a JDBC transaction. + * + *

If a result for this step is already recorded (e.g. on workflow retry), the callback is + * skipped and the cached result is returned. Otherwise the callback runs inside an open + * transaction; the output is recorded atomically with the database work so the step is + * exactly-once on success. + * + * @param the return type of the callback + * @param the checked exception type the callback may throw + * @param callback the database work to perform; receives an open {@link Connection} and must not + * commit or close it + * @param stepName a stable name that identifies this step within the workflow + * @return the value returned by {@code callback} + * @throws X if the callback throws + */ public R txStep( final TransactionalFunction callback, String stepName) throws X { return runTxStep( @@ -101,11 +130,25 @@ public R txStep( stepName); } + /** Database work that runs inside a JDBC transaction without returning a result. */ @FunctionalInterface public interface TransactionalRunnable { void execute(Connection conn) throws X; } + /** + * Executes {@code callback} as an idempotent DBOS step inside a JDBC transaction, with no return + * value. + * + *

Behaves identically to {@link #txStep(TransactionalFunction, String)} but accepts a {@link + * TransactionalRunnable} for callers that do not need to return a result. + * + * @param the checked exception type the callback may throw + * @param callback the database work to perform; receives an open {@link Connection} and must not + * commit or close it + * @param stepName a stable name that identifies this step within the workflow + * @throws X if the callback throws + */ public void txStep(final TransactionalRunnable callback, String stepName) throws X { txStep( diff --git a/transact/src/main/java/dev/dbos/transact/txstep/PostgresStepFactory.java b/transact/src/main/java/dev/dbos/transact/txstep/PostgresStepFactory.java index 7673f1324..595b6e65f 100644 --- a/transact/src/main/java/dev/dbos/transact/txstep/PostgresStepFactory.java +++ b/transact/src/main/java/dev/dbos/transact/txstep/PostgresStepFactory.java @@ -10,68 +10,19 @@ import java.util.Objects; import java.util.Optional; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - +/** + * Abstract base for transactional step factories backed by a PostgreSQL database. + * + *

Subclasses provide a database-library-specific public API (e.g. plain JDBC {@link Connection}, + * JDBI {@code Handle}, jOOQ {@code DSLContext}) while this class owns the shared step lifecycle: + * idempotency checking, error recording, and the {@link #runTxStep} template method that integrates + * with the DBOS runtime. + * + *

The constructor verifies that the datasource is PostgreSQL and creates the {@code + * tx_step_outputs} table (and its enclosing schema) if they do not already exist. + */ public abstract class PostgresStepFactory { - private static final Logger logger = LoggerFactory.getLogger(PostgresStepFactory.class); - - protected static void ensurePostgres(Connection conn) throws SQLException { - var productName = conn.getMetaData().getDatabaseProductName(); - if (!productName.equalsIgnoreCase("PostgreSQL")) { - throw new IllegalArgumentException( - "TxStepFactory requires a PostgreSQL datasource, got: " + productName); - } - } - - protected static void ensureSchema(Connection conn, String schema) throws SQLException { - Objects.requireNonNull(schema, "schema must not be null"); - var sql = "SELECT schema_name FROM information_schema.schemata WHERE schema_name = ?"; - try (var stmt = conn.prepareStatement(sql)) { - stmt.setString(1, Objects.requireNonNull(schema, "schema must not be null")); - try (var rs = stmt.executeQuery()) { - if (rs.next()) { - return; - } - } - } - - try (var stmt = conn.createStatement()) { - stmt.execute("CREATE SCHEMA \"%s\"".formatted(schema)); - } - } - - protected static void ensureTxOutputTable(Connection conn, String schema) throws SQLException { - Objects.requireNonNull(schema, "schema must not be null"); - var sql = "SELECT 1 FROM information_schema.tables WHERE table_schema = ? AND table_name = ?"; - try (var stmt = conn.prepareStatement(sql)) { - stmt.setString(1, Objects.requireNonNull(schema, "schema must not be null")); - stmt.setString(2, "tx_step_outputs"); - try (var rs = stmt.executeQuery()) { - if (rs.next()) { - return; - } - } - } - - logger.debug("Creating tx_step_outputs table in schema={}", schema); - try (var stmt = conn.createStatement()) { - stmt.execute( - """ - CREATE TABLE "%1$s".tx_step_outputs ( - workflow_id TEXT NOT NULL, - step_id INT NOT NULL, - output TEXT, - error TEXT, - serialization TEXT, - created_at BIGINT NOT NULL DEFAULT (EXTRACT(EPOCH FROM now())*1000)::bigint, - PRIMARY KEY (workflow_id, step_id) - )""" - .formatted(schema)); - } - } - protected final DBOS dbos; protected final String schema; protected final DBOSSerializer serializer; @@ -87,25 +38,37 @@ protected PostgresStepFactory( var config = dbos.integration().config(); this.schema = SystemDatabase.sanitizeSchema(schema == null ? config.databaseSchema() : schema); this.serializer = serializer == null ? config.serializer() : serializer; + try (var conn = opener.open()) { - ensurePostgres(conn); - ensureSchema(conn, this.schema); - ensureTxOutputTable(conn, this.schema); + // ensure we're running on Postgres + var productName = conn.getMetaData().getDatabaseProductName(); + if (!productName.equalsIgnoreCase("PostgreSQL")) { + throw new IllegalArgumentException( + "TxStepFactory requires a PostgreSQL datasource, got: " + productName); + } + + // ensure provided schema and tx_step_outputs table exist + try (var stmt = conn.createStatement()) { + stmt.addBatch("CREATE SCHEMA IF NOT EXISTS \"%s\"".formatted(schema)); + stmt.addBatch( + """ + CREATE TABLE IF NOT EXISTS "%1$s".tx_step_outputs ( + workflow_id TEXT NOT NULL, + step_id INT NOT NULL, + output TEXT, + error TEXT, + serialization TEXT, + created_at BIGINT NOT NULL DEFAULT (EXTRACT(EPOCH FROM now())*1000)::bigint, + PRIMARY KEY (workflow_id, step_id) + )""" + .formatted(schema)); + stmt.executeBatch(); + } } catch (SQLException e) { throw new RuntimeException(e); } } - protected String upsertSql() { - return """ - INSERT INTO "%s".tx_step_outputs - (workflow_id, step_id, output, error, serialization) - VALUES (?, ?, ?, ?, ?) - ON CONFLICT DO NOTHING - """ - .formatted(schema); - } - protected String checkSql() { return """ SELECT output, error, serialization @@ -118,6 +81,16 @@ protected String checkSql() { protected abstract Optional checkExecution( String workflowId, int stepId, String stepName); + protected String upsertSql() { + return """ + INSERT INTO "%s".tx_step_outputs + (workflow_id, step_id, output, error, serialization) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT DO NOTHING + """ + .formatted(schema); + } + protected abstract void recordError(String workflowId, int stepId, Exception exception); @FunctionalInterface From 4b1c1e179d92bfa37e2704f01143325f213ebdb2 Mon Sep 17 00:00:00 2001 From: Harry Pierson Date: Wed, 6 May 2026 17:21:23 -0700 Subject: [PATCH 29/29] fix PG step factory ctor --- .../java/dev/dbos/transact/txstep/PostgresStepFactory.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/transact/src/main/java/dev/dbos/transact/txstep/PostgresStepFactory.java b/transact/src/main/java/dev/dbos/transact/txstep/PostgresStepFactory.java index 595b6e65f..8ba79b098 100644 --- a/transact/src/main/java/dev/dbos/transact/txstep/PostgresStepFactory.java +++ b/transact/src/main/java/dev/dbos/transact/txstep/PostgresStepFactory.java @@ -49,7 +49,7 @@ protected PostgresStepFactory( // ensure provided schema and tx_step_outputs table exist try (var stmt = conn.createStatement()) { - stmt.addBatch("CREATE SCHEMA IF NOT EXISTS \"%s\"".formatted(schema)); + stmt.addBatch("CREATE SCHEMA IF NOT EXISTS \"%s\"".formatted(this.schema)); stmt.addBatch( """ CREATE TABLE IF NOT EXISTS "%1$s".tx_step_outputs ( @@ -61,7 +61,7 @@ protected PostgresStepFactory( created_at BIGINT NOT NULL DEFAULT (EXTRACT(EPOCH FROM now())*1000)::bigint, PRIMARY KEY (workflow_id, step_id) )""" - .formatted(schema)); + .formatted(this.schema)); stmt.executeBatch(); } } catch (SQLException e) {