diff --git a/internal/protocol/SOURCE_COMMIT b/internal/protocol/SOURCE_COMMIT index 0ef1ed2..7018feb 100644 --- a/internal/protocol/SOURCE_COMMIT +++ b/internal/protocol/SOURCE_COMMIT @@ -1 +1 @@ -1caadbd7ecfdf5f2309acbeac28a3e36d16aa156 \ No newline at end of file +3145f9337fca9de57d2f89a6ff6f07150d34f1c2 diff --git a/internal/protocol/protos/orchestrator_service.proto b/internal/protocol/protos/orchestrator_service.proto index 0c34d98..3d9194a 100644 --- a/internal/protocol/protos/orchestrator_service.proto +++ b/internal/protocol/protos/orchestrator_service.proto @@ -25,6 +25,7 @@ message ActivityRequest { OrchestrationInstance orchestrationInstance = 4; int32 taskId = 5; TraceContext parentTraceContext = 6; + map tags = 7; } message ActivityResponse { @@ -320,6 +321,10 @@ message SendEntityMessageAction { } } +message RewindOrchestrationAction { + repeated HistoryEvent newHistory = 1; +} + message OrchestratorAction { int32 id = 1; oneof orchestratorActionType { @@ -330,6 +335,7 @@ message OrchestratorAction { CompleteOrchestrationAction completeOrchestration = 6; TerminateOrchestrationAction terminateOrchestration = 7; SendEntityMessageAction sendEntityMessage = 8; + RewindOrchestrationAction rewindOrchestration = 9; } } @@ -364,12 +370,14 @@ message OrchestratorResponse { // Whether or not a history is required to complete the original OrchestratorRequest and none was provided. bool requiresHistory = 7; + /* Chunking logic has since been deprecated and fields related to it are marked as such */ + // True if this is a partial (chunked) completion. The backend must keep the work item open until the final chunk (isPartial=false). - bool isPartial = 8; + bool isPartial = 8 [deprecated=true]; // Zero-based position of the current chunk within a chunked completion sequence. // This field is omitted for non-chunked completions. - google.protobuf.Int32Value chunkIndex = 9; + google.protobuf.Int32Value chunkIndex = 9 [deprecated=true]; } message CreateInstanceRequest { @@ -517,6 +525,7 @@ message PurgeInstanceFilter { google.protobuf.Timestamp createdTimeFrom = 1; google.protobuf.Timestamp createdTimeTo = 2; repeated OrchestrationStatus runtimeStatus = 3; + google.protobuf.Duration timeout = 4; } message PurgeInstancesResponse { diff --git a/packages/durabletask-js/src/proto/orchestrator_service_pb.d.ts b/packages/durabletask-js/src/proto/orchestrator_service_pb.d.ts index 1505288..59ac1b3 100644 --- a/packages/durabletask-js/src/proto/orchestrator_service_pb.d.ts +++ b/packages/durabletask-js/src/proto/orchestrator_service_pb.d.ts @@ -63,6 +63,9 @@ export class ActivityRequest extends jspb.Message { getParenttracecontext(): TraceContext | undefined; setParenttracecontext(value?: TraceContext): ActivityRequest; + getTagsMap(): jspb.Map; + clearTagsMap(): void; + serializeBinary(): Uint8Array; toObject(includeInstance?: boolean): ActivityRequest.AsObject; static toObject(includeInstance: boolean, msg: ActivityRequest): ActivityRequest.AsObject; @@ -81,6 +84,8 @@ export namespace ActivityRequest { orchestrationinstance?: OrchestrationInstance.AsObject, taskid: number, parenttracecontext?: TraceContext.AsObject, + + tagsMap: Array<[string, string]>, } } @@ -1613,6 +1618,28 @@ export namespace SendEntityMessageAction { } +export class RewindOrchestrationAction extends jspb.Message { + clearNewhistoryList(): void; + getNewhistoryList(): Array; + setNewhistoryList(value: Array): RewindOrchestrationAction; + addNewhistory(value?: HistoryEvent, index?: number): HistoryEvent; + + serializeBinary(): Uint8Array; + toObject(includeInstance?: boolean): RewindOrchestrationAction.AsObject; + static toObject(includeInstance: boolean, msg: RewindOrchestrationAction): RewindOrchestrationAction.AsObject; + static extensions: {[key: number]: jspb.ExtensionFieldInfo}; + static extensionsBinary: {[key: number]: jspb.ExtensionFieldBinaryInfo}; + static serializeBinaryToWriter(message: RewindOrchestrationAction, writer: jspb.BinaryWriter): void; + static deserializeBinary(bytes: Uint8Array): RewindOrchestrationAction; + static deserializeBinaryFromReader(message: RewindOrchestrationAction, reader: jspb.BinaryReader): RewindOrchestrationAction; +} + +export namespace RewindOrchestrationAction { + export type AsObject = { + newhistoryList: Array, + } +} + export class OrchestratorAction extends jspb.Message { getId(): number; setId(value: number): OrchestratorAction; @@ -1652,6 +1679,11 @@ export class OrchestratorAction extends jspb.Message { getSendentitymessage(): SendEntityMessageAction | undefined; setSendentitymessage(value?: SendEntityMessageAction): OrchestratorAction; + hasRewindorchestration(): boolean; + clearRewindorchestration(): void; + getRewindorchestration(): RewindOrchestrationAction | undefined; + setRewindorchestration(value?: RewindOrchestrationAction): OrchestratorAction; + getOrchestratoractiontypeCase(): OrchestratorAction.OrchestratoractiontypeCase; serializeBinary(): Uint8Array; @@ -1674,6 +1706,7 @@ export namespace OrchestratorAction { completeorchestration?: CompleteOrchestrationAction.AsObject, terminateorchestration?: TerminateOrchestrationAction.AsObject, sendentitymessage?: SendEntityMessageAction.AsObject, + rewindorchestration?: RewindOrchestrationAction.AsObject, } export enum OrchestratoractiontypeCase { @@ -1685,6 +1718,7 @@ export namespace OrchestratorAction { COMPLETEORCHESTRATION = 6, TERMINATEORCHESTRATION = 7, SENDENTITYMESSAGE = 8, + REWINDORCHESTRATION = 9, } } @@ -2557,6 +2591,11 @@ export class PurgeInstanceFilter extends jspb.Message { setRuntimestatusList(value: Array): PurgeInstanceFilter; addRuntimestatus(value: OrchestrationStatus, index?: number): OrchestrationStatus; + hasTimeout(): boolean; + clearTimeout(): void; + getTimeout(): google_protobuf_duration_pb.Duration | undefined; + setTimeout(value?: google_protobuf_duration_pb.Duration): PurgeInstanceFilter; + serializeBinary(): Uint8Array; toObject(includeInstance?: boolean): PurgeInstanceFilter.AsObject; static toObject(includeInstance: boolean, msg: PurgeInstanceFilter): PurgeInstanceFilter.AsObject; @@ -2572,6 +2611,7 @@ export namespace PurgeInstanceFilter { createdtimefrom?: google_protobuf_timestamp_pb.Timestamp.AsObject, createdtimeto?: google_protobuf_timestamp_pb.Timestamp.AsObject, runtimestatusList: Array, + timeout?: google_protobuf_duration_pb.Duration.AsObject, } } diff --git a/packages/durabletask-js/src/proto/orchestrator_service_pb.js b/packages/durabletask-js/src/proto/orchestrator_service_pb.js index fd84cc5..210f5cd 100644 --- a/packages/durabletask-js/src/proto/orchestrator_service_pb.js +++ b/packages/durabletask-js/src/proto/orchestrator_service_pb.js @@ -127,6 +127,7 @@ goog.exportSymbol('proto.ResumeRequest', null, global); goog.exportSymbol('proto.ResumeResponse', null, global); goog.exportSymbol('proto.RewindInstanceRequest', null, global); goog.exportSymbol('proto.RewindInstanceResponse', null, global); +goog.exportSymbol('proto.RewindOrchestrationAction', null, global); goog.exportSymbol('proto.ScheduleTaskAction', null, global); goog.exportSymbol('proto.SendEntityMessageAction', null, global); goog.exportSymbol('proto.SendEntityMessageAction.EntitymessagetypeCase', null, global); @@ -1039,6 +1040,27 @@ if (goog.DEBUG && !COMPILED) { */ proto.SendEntityMessageAction.displayName = 'proto.SendEntityMessageAction'; } +/** + * Generated by JsPbCodeGenerator. + * @param {Array=} opt_data Optional initial data array, typically from a + * server response, or constructed directly in Javascript. The array is used + * in place and becomes part of the constructed object. It is not cloned. + * If no data is provided, the constructed object will be empty, but still + * valid. + * @extends {jspb.Message} + * @constructor + */ +proto.RewindOrchestrationAction = function(opt_data) { + jspb.Message.initialize(this, opt_data, 0, -1, proto.RewindOrchestrationAction.repeatedFields_, null); +}; +goog.inherits(proto.RewindOrchestrationAction, jspb.Message); +if (goog.DEBUG && !COMPILED) { + /** + * @public + * @override + */ + proto.RewindOrchestrationAction.displayName = 'proto.RewindOrchestrationAction'; +} /** * Generated by JsPbCodeGenerator. * @param {Array=} opt_data Optional initial data array, typically from a @@ -2832,7 +2854,8 @@ proto.ActivityRequest.toObject = function(includeInstance, msg) { input: (f = msg.getInput()) && google_protobuf_wrappers_pb.StringValue.toObject(includeInstance, f), orchestrationinstance: (f = msg.getOrchestrationinstance()) && proto.OrchestrationInstance.toObject(includeInstance, f), taskid: jspb.Message.getFieldWithDefault(msg, 5, 0), - parenttracecontext: (f = msg.getParenttracecontext()) && proto.TraceContext.toObject(includeInstance, f) + parenttracecontext: (f = msg.getParenttracecontext()) && proto.TraceContext.toObject(includeInstance, f), + tagsMap: (f = msg.getTagsMap()) ? f.toObject(includeInstance, undefined) : [] }; if (includeInstance) { @@ -2897,6 +2920,12 @@ proto.ActivityRequest.deserializeBinaryFromReader = function(msg, reader) { reader.readMessage(value,proto.TraceContext.deserializeBinaryFromReader); msg.setParenttracecontext(value); break; + case 7: + var value = msg.getTagsMap(); + reader.readMessage(value, function(message, reader) { + jspb.Map.deserializeBinary(message, reader, jspb.BinaryReader.prototype.readString, jspb.BinaryReader.prototype.readString, null, "", ""); + }); + break; default: reader.skipField(); break; @@ -2972,6 +3001,10 @@ proto.ActivityRequest.serializeBinaryToWriter = function(message, writer) { proto.TraceContext.serializeBinaryToWriter ); } + f = message.getTagsMap(true); + if (f && f.getLength() > 0) { + f.serializeBinary(7, writer, jspb.BinaryWriter.prototype.writeString, jspb.BinaryWriter.prototype.writeString); + } }; @@ -3159,6 +3192,28 @@ proto.ActivityRequest.prototype.hasParenttracecontext = function() { }; +/** + * map tags = 7; + * @param {boolean=} opt_noLazyCreate Do not create the map if + * empty, instead returning `undefined` + * @return {!jspb.Map} + */ +proto.ActivityRequest.prototype.getTagsMap = function(opt_noLazyCreate) { + return /** @type {!jspb.Map} */ ( + jspb.Message.getMapField(this, 7, opt_noLazyCreate, + null)); +}; + + +/** + * Clears values from the map. The map will be non-null. + * @return {!proto.ActivityRequest} returns this + */ +proto.ActivityRequest.prototype.clearTagsMap = function() { + this.getTagsMap().clear(); + return this;}; + + @@ -14207,6 +14262,166 @@ proto.SendEntityMessageAction.prototype.hasEntityunlocksent = function() { +/** + * List of repeated fields within this message type. + * @private {!Array} + * @const + */ +proto.RewindOrchestrationAction.repeatedFields_ = [1]; + + + +if (jspb.Message.GENERATE_TO_OBJECT) { +/** + * Creates an object representation of this proto. + * Field names that are reserved in JavaScript and will be renamed to pb_name. + * Optional fields that are not set will be set to undefined. + * To access a reserved field use, foo.pb_, eg, foo.pb_default. + * For the list of reserved names please see: + * net/proto2/compiler/js/internal/generator.cc#kKeyword. + * @param {boolean=} opt_includeInstance Deprecated. whether to include the + * JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @return {!Object} + */ +proto.RewindOrchestrationAction.prototype.toObject = function(opt_includeInstance) { + return proto.RewindOrchestrationAction.toObject(opt_includeInstance, this); +}; + + +/** + * Static version of the {@see toObject} method. + * @param {boolean|undefined} includeInstance Deprecated. Whether to include + * the JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @param {!proto.RewindOrchestrationAction} msg The msg instance to transform. + * @return {!Object} + * @suppress {unusedLocalVariables} f is only used for nested messages + */ +proto.RewindOrchestrationAction.toObject = function(includeInstance, msg) { + var f, obj = { + newhistoryList: jspb.Message.toObjectList(msg.getNewhistoryList(), + proto.HistoryEvent.toObject, includeInstance) + }; + + if (includeInstance) { + obj.$jspbMessageInstance = msg; + } + return obj; +}; +} + + +/** + * Deserializes binary data (in protobuf wire format). + * @param {jspb.ByteSource} bytes The bytes to deserialize. + * @return {!proto.RewindOrchestrationAction} + */ +proto.RewindOrchestrationAction.deserializeBinary = function(bytes) { + var reader = new jspb.BinaryReader(bytes); + var msg = new proto.RewindOrchestrationAction; + return proto.RewindOrchestrationAction.deserializeBinaryFromReader(msg, reader); +}; + + +/** + * Deserializes binary data (in protobuf wire format) from the + * given reader into the given message object. + * @param {!proto.RewindOrchestrationAction} msg The message object to deserialize into. + * @param {!jspb.BinaryReader} reader The BinaryReader to use. + * @return {!proto.RewindOrchestrationAction} + */ +proto.RewindOrchestrationAction.deserializeBinaryFromReader = function(msg, reader) { + while (reader.nextField()) { + if (reader.isEndGroup()) { + break; + } + var field = reader.getFieldNumber(); + switch (field) { + case 1: + var value = new proto.HistoryEvent; + reader.readMessage(value,proto.HistoryEvent.deserializeBinaryFromReader); + msg.addNewhistory(value); + break; + default: + reader.skipField(); + break; + } + } + return msg; +}; + + +/** + * Serializes the message to binary data (in protobuf wire format). + * @return {!Uint8Array} + */ +proto.RewindOrchestrationAction.prototype.serializeBinary = function() { + var writer = new jspb.BinaryWriter(); + proto.RewindOrchestrationAction.serializeBinaryToWriter(this, writer); + return writer.getResultBuffer(); +}; + + +/** + * Serializes the given message to binary data (in protobuf wire + * format), writing to the given BinaryWriter. + * @param {!proto.RewindOrchestrationAction} message + * @param {!jspb.BinaryWriter} writer + * @suppress {unusedLocalVariables} f is only used for nested messages + */ +proto.RewindOrchestrationAction.serializeBinaryToWriter = function(message, writer) { + var f = undefined; + f = message.getNewhistoryList(); + if (f.length > 0) { + writer.writeRepeatedMessage( + 1, + f, + proto.HistoryEvent.serializeBinaryToWriter + ); + } +}; + + +/** + * repeated HistoryEvent newHistory = 1; + * @return {!Array} + */ +proto.RewindOrchestrationAction.prototype.getNewhistoryList = function() { + return /** @type{!Array} */ ( + jspb.Message.getRepeatedWrapperField(this, proto.HistoryEvent, 1)); +}; + + +/** + * @param {!Array} value + * @return {!proto.RewindOrchestrationAction} returns this +*/ +proto.RewindOrchestrationAction.prototype.setNewhistoryList = function(value) { + return jspb.Message.setRepeatedWrapperField(this, 1, value); +}; + + +/** + * @param {!proto.HistoryEvent=} opt_value + * @param {number=} opt_index + * @return {!proto.HistoryEvent} + */ +proto.RewindOrchestrationAction.prototype.addNewhistory = function(opt_value, opt_index) { + return jspb.Message.addToRepeatedWrapperField(this, 1, opt_value, proto.HistoryEvent, opt_index); +}; + + +/** + * Clears the list making it empty but non-null. + * @return {!proto.RewindOrchestrationAction} returns this + */ +proto.RewindOrchestrationAction.prototype.clearNewhistoryList = function() { + return this.setNewhistoryList([]); +}; + + + /** * Oneof group definitions for this message. Each group defines the field * numbers belonging to that group. When of these fields' value is set, all @@ -14215,7 +14430,7 @@ proto.SendEntityMessageAction.prototype.hasEntityunlocksent = function() { * @private {!Array>} * @const */ -proto.OrchestratorAction.oneofGroups_ = [[2,3,4,5,6,7,8]]; +proto.OrchestratorAction.oneofGroups_ = [[2,3,4,5,6,7,8,9]]; /** * @enum {number} @@ -14228,7 +14443,8 @@ proto.OrchestratorAction.OrchestratoractiontypeCase = { SENDEVENT: 5, COMPLETEORCHESTRATION: 6, TERMINATEORCHESTRATION: 7, - SENDENTITYMESSAGE: 8 + SENDENTITYMESSAGE: 8, + REWINDORCHESTRATION: 9 }; /** @@ -14276,7 +14492,8 @@ proto.OrchestratorAction.toObject = function(includeInstance, msg) { sendevent: (f = msg.getSendevent()) && proto.SendEventAction.toObject(includeInstance, f), completeorchestration: (f = msg.getCompleteorchestration()) && proto.CompleteOrchestrationAction.toObject(includeInstance, f), terminateorchestration: (f = msg.getTerminateorchestration()) && proto.TerminateOrchestrationAction.toObject(includeInstance, f), - sendentitymessage: (f = msg.getSendentitymessage()) && proto.SendEntityMessageAction.toObject(includeInstance, f) + sendentitymessage: (f = msg.getSendentitymessage()) && proto.SendEntityMessageAction.toObject(includeInstance, f), + rewindorchestration: (f = msg.getRewindorchestration()) && proto.RewindOrchestrationAction.toObject(includeInstance, f) }; if (includeInstance) { @@ -14352,6 +14569,11 @@ proto.OrchestratorAction.deserializeBinaryFromReader = function(msg, reader) { reader.readMessage(value,proto.SendEntityMessageAction.deserializeBinaryFromReader); msg.setSendentitymessage(value); break; + case 9: + var value = new proto.RewindOrchestrationAction; + reader.readMessage(value,proto.RewindOrchestrationAction.deserializeBinaryFromReader); + msg.setRewindorchestration(value); + break; default: reader.skipField(); break; @@ -14444,6 +14666,14 @@ proto.OrchestratorAction.serializeBinaryToWriter = function(message, writer) { proto.SendEntityMessageAction.serializeBinaryToWriter ); } + f = message.getRewindorchestration(); + if (f != null) { + writer.writeMessage( + 9, + f, + proto.RewindOrchestrationAction.serializeBinaryToWriter + ); + } }; @@ -14724,6 +14954,43 @@ proto.OrchestratorAction.prototype.hasSendentitymessage = function() { }; +/** + * optional RewindOrchestrationAction rewindOrchestration = 9; + * @return {?proto.RewindOrchestrationAction} + */ +proto.OrchestratorAction.prototype.getRewindorchestration = function() { + return /** @type{?proto.RewindOrchestrationAction} */ ( + jspb.Message.getWrapperField(this, proto.RewindOrchestrationAction, 9)); +}; + + +/** + * @param {?proto.RewindOrchestrationAction|undefined} value + * @return {!proto.OrchestratorAction} returns this +*/ +proto.OrchestratorAction.prototype.setRewindorchestration = function(value) { + return jspb.Message.setOneofWrapperField(this, 9, proto.OrchestratorAction.oneofGroups_[0], value); +}; + + +/** + * Clears the message field making it undefined. + * @return {!proto.OrchestratorAction} returns this + */ +proto.OrchestratorAction.prototype.clearRewindorchestration = function() { + return this.setRewindorchestration(undefined); +}; + + +/** + * Returns whether this field is set. + * @return {boolean} + */ +proto.OrchestratorAction.prototype.hasRewindorchestration = function() { + return jspb.Message.getField(this, 9) != null; +}; + + @@ -21065,7 +21332,8 @@ proto.PurgeInstanceFilter.toObject = function(includeInstance, msg) { var f, obj = { createdtimefrom: (f = msg.getCreatedtimefrom()) && google_protobuf_timestamp_pb.Timestamp.toObject(includeInstance, f), createdtimeto: (f = msg.getCreatedtimeto()) && google_protobuf_timestamp_pb.Timestamp.toObject(includeInstance, f), - runtimestatusList: (f = jspb.Message.getRepeatedField(msg, 3)) == null ? undefined : f + runtimestatusList: (f = jspb.Message.getRepeatedField(msg, 3)) == null ? undefined : f, + timeout: (f = msg.getTimeout()) && google_protobuf_duration_pb.Duration.toObject(includeInstance, f) }; if (includeInstance) { @@ -21118,6 +21386,11 @@ proto.PurgeInstanceFilter.deserializeBinaryFromReader = function(msg, reader) { msg.addRuntimestatus(values[i]); } break; + case 4: + var value = new google_protobuf_duration_pb.Duration; + reader.readMessage(value,google_protobuf_duration_pb.Duration.deserializeBinaryFromReader); + msg.setTimeout(value); + break; default: reader.skipField(); break; @@ -21170,6 +21443,14 @@ proto.PurgeInstanceFilter.serializeBinaryToWriter = function(message, writer) { f ); } + f = message.getTimeout(); + if (f != null) { + writer.writeMessage( + 4, + f, + google_protobuf_duration_pb.Duration.serializeBinaryToWriter + ); + } }; @@ -21284,6 +21565,43 @@ proto.PurgeInstanceFilter.prototype.clearRuntimestatusList = function() { }; +/** + * optional google.protobuf.Duration timeout = 4; + * @return {?proto.google.protobuf.Duration} + */ +proto.PurgeInstanceFilter.prototype.getTimeout = function() { + return /** @type{?proto.google.protobuf.Duration} */ ( + jspb.Message.getWrapperField(this, google_protobuf_duration_pb.Duration, 4)); +}; + + +/** + * @param {?proto.google.protobuf.Duration|undefined} value + * @return {!proto.PurgeInstanceFilter} returns this +*/ +proto.PurgeInstanceFilter.prototype.setTimeout = function(value) { + return jspb.Message.setWrapperField(this, 4, value); +}; + + +/** + * Clears the message field making it undefined. + * @return {!proto.PurgeInstanceFilter} returns this + */ +proto.PurgeInstanceFilter.prototype.clearTimeout = function() { + return this.setTimeout(undefined); +}; + + +/** + * Returns whether this field is set. + * @return {boolean} + */ +proto.PurgeInstanceFilter.prototype.hasTimeout = function() { + return jspb.Message.getField(this, 4) != null; +}; + + diff --git a/packages/durabletask-js/src/testing/in-memory-backend.ts b/packages/durabletask-js/src/testing/in-memory-backend.ts index 476f020..1d55189 100644 --- a/packages/durabletask-js/src/testing/in-memory-backend.ts +++ b/packages/durabletask-js/src/testing/in-memory-backend.ts @@ -248,6 +248,34 @@ export class InMemoryOrchestrationBackend { return true; } + /** + * Rewinds a failed orchestration instance. + * + * Validates the instance is in a failed state, then appends an ExecutionRewoundEvent to the + * pending events, resets the status to RUNNING, and re-enqueues the orchestration so the + * worker can replay it and produce a RewindOrchestrationAction with the corrected history. + * The actual history rewrite is performed by the SDK worker (see buildRewindResult); this + * backend merely applies the result. Any change to that rewrite must be mirrored here. + * + * @param instanceId The instance to rewind. + * @param reason Optional human-readable reason for the rewind. + * @throws Error with a "not found" message if the instance does not exist. + * @throws Error with a "not in a failed state" message if the instance is not FAILED. + */ + rewindInstance(instanceId: string, reason?: string): void { + const instance = this.instances.get(instanceId); + if (!instance) { + throw new Error(`Orchestration instance '${instanceId}' not found`); + } + + if (instance.status !== pb.OrchestrationStatus.ORCHESTRATION_STATUS_FAILED) { + throw new Error(`Orchestration instance '${instanceId}' is not in a failed state`); + } + + this.prepareRewind(instance, reason); + this.notifyWaiters(instanceId); + } + /** * Gets the next orchestration work item to process, if any. */ @@ -318,6 +346,23 @@ export class InMemoryOrchestrationBackend { this.processAction(instance, action); } + // Bookend a terminal orchestration with an executionCompleted event so the committed + // history records the terminal result. + // + // REQUIRED — do NOT remove this as an "optimization". It is also the precondition that lets + // the worker detect a rewind: the executor only short-circuits into buildRewindResult when an + // executionCompleted event is present in the committed (old) history + // (see OrchestrationExecutor.execute). Without this bookend, a rewind dispatch would silently + // fall through to plain replay and never produce a RewindOrchestrationAction. + // + // Continue-as-new resets status to PENDING and rewind resets it to RUNNING, so neither is + // terminal here and neither gets a bookend. + if (this.isTerminalStatus(instance.status)) { + instance.history.push( + pbh.newExecutionCompletedEvent(instance.status, instance.output, instance.failureDetails), + ); + } + // Update completion token for next execution instance.completionToken = this.nextCompletionToken++; @@ -489,6 +534,9 @@ export class InMemoryOrchestrationBackend { case pb.OrchestratorAction.OrchestratoractiontypeCase.SENDEVENT: this.processSendEventAction(action.getSendevent()!); break; + case pb.OrchestratorAction.OrchestratoractiontypeCase.REWINDORCHESTRATION: + this.processRewindOrchestrationAction(instance, action.getRewindorchestration()!); + break; default: throw new Error( `Unknown orchestrator action type '${actionType}' for orchestration '${instance.instanceId}'. ` + @@ -504,7 +552,10 @@ export class InMemoryOrchestrationBackend { const status = completeAction.getOrchestrationstatus(); instance.status = status; instance.output = completeAction.getResult()?.getValue(); - instance.failureDetails = completeAction.getFailuredetails(); + // Use an explicit presence check: a protobuf singular message accessor may materialize an + // empty message, so only record failureDetails when the action actually carries them. This + // keeps the bookended executionCompleted event clean for successful completions. + instance.failureDetails = completeAction.hasFailuredetails() ? completeAction.getFailuredetails() : undefined; if (status === pb.OrchestrationStatus.ORCHESTRATION_STATUS_CONTINUED_AS_NEW) { // Handle continue-as-new @@ -664,6 +715,97 @@ export class InMemoryOrchestrationBackend { }); } + private prepareRewind(instance: OrchestrationInstance, reason?: string): void { + // Reset instance state so it can be re-processed. + instance.status = pb.OrchestrationStatus.ORCHESTRATION_STATUS_RUNNING; + instance.output = undefined; + instance.failureDetails = undefined; + instance.lastUpdatedAt = new Date(); + + // Seed the pending events with exactly [orchestratorStarted, executionRewound]. The worker + // splits (history, pendingEvents) into (oldEvents, newEvents); buildRewindResult requires + // newEvents to be exactly those two events (orchestratorStarted followed by the + // executionRewound marker). Unlike the real sidecar, this backend does not auto-prepend an + // orchestratorStarted per dispatch, so it is supplied here. + instance.pendingEvents = [pbh.newOrchestratorStartedEvent(new Date()), pbh.newExecutionRewoundEvent(reason)]; + + // Refresh the completion token and enqueue. + instance.completionToken = this.nextCompletionToken++; + this.enqueueOrchestration(instance.instanceId); + } + + private processRewindOrchestrationAction( + instance: OrchestrationInstance, + rewindAction: pb.RewindOrchestrationAction, + ): void { + const newHistory = rewindAction.getNewhistoryList(); + + // Replace the history with the SDK-computed clean version. + instance.history = newHistory; + instance.status = pb.OrchestrationStatus.ORCHESTRATION_STATUS_RUNNING; + instance.output = undefined; + instance.failureDetails = undefined; + instance.lastUpdatedAt = new Date(); + + // Identify sub-orchestrations that were created but did not complete successfully — they + // need to be recursively rewound (buildRewindResult keeps subOrchestrationInstanceCreated + // and removes subOrchestrationInstanceFailed, so a "created but not completed" sub is a + // failed one). + const completedSubOrchTaskIds = new Set(); + const createdSubOrchEvents = new Map(); + for (const event of newHistory) { + if (event.hasSuborchestrationinstancecreated()) { + createdSubOrchEvents.set(event.getEventid(), event); + } else if (event.hasSuborchestrationinstancecompleted()) { + completedSubOrchTaskIds.add(event.getSuborchestrationinstancecompleted()!.getTaskscheduledid()); + } + } + + // Extract the rewind reason from the last executionRewound event. + let reason: string | undefined; + for (let i = newHistory.length - 1; i >= 0; i--) { + const event = newHistory[i]; + if (event.hasExecutionrewound()) { + const rewound = event.getExecutionrewound()!; + reason = rewound.hasReason() ? rewound.getReason()!.getValue() : undefined; + break; + } + } + + // Recursively rewind failed sub-orchestrations. If the sub was purged (no longer tracked), + // re-create it from the subOrchestrationInstanceCreated event so it runs fresh. + for (const [taskId, event] of createdSubOrchEvents) { + if (completedSubOrchTaskIds.has(taskId)) { + continue; + } + const subInfo = event.getSuborchestrationinstancecreated()!; + const subInstanceId = subInfo.getInstanceid(); + const subInstance = this.instances.get(subInstanceId); + if (!subInstance) { + // Sub-orchestration was purged — re-create it so it runs fresh. Pass the parent + // metadata (mirroring processCreateSubOrchestrationAction) so the re-created sub keeps + // its parentInstance link and can route its completion back to this orchestration. + this.createInstance(subInstanceId, subInfo.getName(), subInfo.getInput()?.getValue(), undefined, { + name: instance.name, + instanceId: instance.instanceId, + taskScheduledId: taskId, + }); + } else if (subInstance.status === pb.OrchestrationStatus.ORCHESTRATION_STATUS_FAILED) { + this.prepareRewind(subInstance, reason); + } + this.watchSubOrchestration(instance.instanceId, subInstanceId, taskId); + } + + // Re-enqueue so the orchestration replays with the clean history. The executionRewound + // event is already present in the clean history (kept by buildRewindResult), so it must + // NOT be re-sent as a pending event — doing so would duplicate it. A lone orchestratorStarted + // is enough to make the instance dispatchable; the worker replays normally because + // executionCompleted is no longer in the history. + instance.pendingEvents = [pbh.newOrchestratorStartedEvent(new Date())]; + instance.completionToken = this.nextCompletionToken++; + this.enqueueOrchestration(instance.instanceId); + } + private processSendEventAction(sendEvent: pb.SendEventAction): void { const targetInstanceId = sendEvent.getInstance()?.getInstanceid(); const eventName = sendEvent.getName(); diff --git a/packages/durabletask-js/src/testing/test-client.ts b/packages/durabletask-js/src/testing/test-client.ts index 5b8f448..09d1538 100644 --- a/packages/durabletask-js/src/testing/test-client.ts +++ b/packages/durabletask-js/src/testing/test-client.ts @@ -130,6 +130,16 @@ export class TestOrchestrationClient { return { deletedInstanceCount: deleted ? 1 : 0 }; } + /** + * Rewinds a failed orchestration so it re-runs from the point of failure. + * + * @param instanceId The instance to rewind. + * @param reason Optional human-readable reason for the rewind. + */ + async rewindOrchestration(instanceId: string, reason?: string): Promise { + this.backend.rewindInstance(instanceId, reason); + } + /** * Stops the client. No-op for in-memory backend. */ diff --git a/packages/durabletask-js/src/utils/pb-helper.util.ts b/packages/durabletask-js/src/utils/pb-helper.util.ts index 100f0b2..a26235c 100644 --- a/packages/durabletask-js/src/utils/pb-helper.util.ts +++ b/packages/durabletask-js/src/utils/pb-helper.util.ts @@ -288,6 +288,42 @@ export function newTerminatedEvent(encodedOutput?: string): pb.HistoryEvent { return event; } +export function newExecutionCompletedEvent( + status: pb.OrchestrationStatus, + encodedResult?: string, + failureDetails?: pb.TaskFailureDetails, +): pb.HistoryEvent { + const executionCompletedEvent = new pb.ExecutionCompletedEvent(); + executionCompletedEvent.setOrchestrationstatus(status); + executionCompletedEvent.setResult(getStringValue(encodedResult)); + if (failureDetails) { + executionCompletedEvent.setFailuredetails(failureDetails); + } + + const ts = new Timestamp(); + + const event = new pb.HistoryEvent(); + event.setEventid(-1); + event.setTimestamp(ts); + event.setExecutioncompleted(executionCompletedEvent); + + return event; +} + +export function newExecutionRewoundEvent(reason?: string): pb.HistoryEvent { + const executionRewoundEvent = new pb.ExecutionRewoundEvent(); + executionRewoundEvent.setReason(getStringValue(reason)); + + const ts = new Timestamp(); + + const event = new pb.HistoryEvent(); + event.setEventid(-1); + event.setTimestamp(ts); + event.setExecutionrewound(executionRewoundEvent); + + return event; +} + export function getStringValue(val?: string): StringValue | undefined { if (!val) { return; diff --git a/packages/durabletask-js/src/worker/orchestration-executor.ts b/packages/durabletask-js/src/worker/orchestration-executor.ts index e69a0dc..dfcb5df 100644 --- a/packages/durabletask-js/src/worker/orchestration-executor.ts +++ b/packages/durabletask-js/src/worker/orchestration-executor.ts @@ -25,6 +25,7 @@ import type { StringValue } from "google-protobuf/google/protobuf/wrappers_pb"; import { OrchestratorNotRegisteredError } from "./exception/orchestrator-not-registered-error"; import { StopIterationError } from "./exception/stop-iteration-error"; import { Registry } from "./registry"; +import { buildRewindResult } from "./rewind"; import { RuntimeOrchestrationContext } from "./runtime-orchestration-context"; import { EntityOperationFailedException, @@ -65,6 +66,23 @@ export class OrchestrationExecutor { throw new OrchestrationStateError("The new history event list must have at least one event in it"); } + // Check for rewind BEFORE replay. A rewind is indicated by an executionRewound event in + // newEvents. We look for an executionCompleted event in the committed history (oldEvents) + // to decide whether to rewind or replay: + // 1. executionCompleted IS present -> the orchestration reached a terminal state (e.g. + // failed). This is a *new* rewind that the worker must short-circuit by building + // clean history. + // 2. executionCompleted is NOT present -> the backend already processed the + // RewindOrchestrationAction and removed executionCompleted from the committed history. + // Here the executionRewound event in newEvents acts as a "jump-start": it wakes the + // orchestration so that normal replay re-emits scheduleTask actions for the removed + // activities, causing the previously-failed work to rerun. No further rewrite is + // needed, so we fall through to the normal replay path below. + const hasRewindInNew = newEvents.some((e) => e.hasExecutionrewound()); + if (hasRewindInNew && oldEvents.some((e) => e.hasExecutioncompleted())) { + return buildRewindResult(oldEvents, newEvents); + } + const ctx = new RuntimeOrchestrationContext(instanceId); try { @@ -175,6 +193,11 @@ export class OrchestrationExecutor { case pb.HistoryEvent.EventtypeCase.EXECUTIONTERMINATED: await this.handleExecutionTerminated(ctx, event); break; + case pb.HistoryEvent.EventtypeCase.EXECUTIONREWOUND: + // Informational event added when an orchestration is rewound. No action needed. + // (The rewind history rewrite happens in buildRewindResult before replay; when a + // rewound instance is re-dispatched this event simply jump-starts the replay.) + break; case pb.HistoryEvent.EventtypeCase.ENTITYOPERATIONCALLED: await this.handleEntityOperationCalled(ctx, event); break; diff --git a/packages/durabletask-js/src/worker/rewind.ts b/packages/durabletask-js/src/worker/rewind.ts new file mode 100644 index 0000000..48b968c --- /dev/null +++ b/packages/durabletask-js/src/worker/rewind.ts @@ -0,0 +1,131 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +import { randomUUID } from "crypto"; +import * as pb from "../proto/orchestrator_service_pb"; +import { getStringValue } from "../utils/pb-helper.util"; +import type { OrchestrationExecutionResult } from "./orchestration-executor"; + +/** + * Build an {@link OrchestrationExecutionResult} containing a {@link pb.RewindOrchestrationAction}. + * + * When the worker detects an `executionRewound` event in the new events (that does not yet + * appear in the committed history) it rewrites the history by removing failed task results + * (`taskFailed`) and failed sub-orchestration results (`subOrchestrationInstanceFailed`). The + * `executionRewound` event is kept so the backend knows why the rewind happened and so it + * remains in the history for audit purposes. + * + * For failed activities, the corresponding `taskScheduled` event is also removed so that the + * SDK will re-generate a `scheduleTask` action during the next replay, causing the backend to + * re-dispatch the activity. + * + * For failed sub-orchestrations, the `subOrchestrationInstanceCreated` event is kept so the + * backend can identify which sub-orchestration instances to recursively rewind. + * + * Known limitation (shared with the Core/.NET/Python rewind): timer events + * (`timerCreated` / `timerFired`) emitted between retry attempts of an activity scheduled + * with a `RetryPolicy` are not removed. On the next replay the regenerated `scheduleTask` + * action may not line up with those retained timer events, which can surface as a + * non-determinism mismatch. Rewinding an activity that used a retry policy is therefore not + * currently supported. + * + * WARNING!!!: + * If any changes are made to how this function modifies the orchestration's history, then + * corresponding changes *must* be made in the backend implementations that rely on this + * function for executing a rewind (e.g. the in-memory backend and the DTS backend). + * + * @param oldEvents - the committed history events for the instance + * @param newEvents - the newly received events, which must be exactly + * `[orchestratorStarted, executionRewound]` + * @returns an execution result whose single action is a `rewindOrchestration` action carrying + * the cleaned-up history + */ +export function buildRewindResult( + oldEvents: pb.HistoryEvent[], + newEvents: pb.HistoryEvent[], +): OrchestrationExecutionResult { + if (newEvents.length !== 2 || !newEvents[1].hasExecutionrewound()) { + throw new Error( + "When rewinding an orchestration, the new events list must contain exactly two events: " + + "orchestratorStarted and the executionRewound event.", + ); + } + + const rewindEvent = newEvents[1].getExecutionrewound() as pb.ExecutionRewoundEvent; + + const allEvents = [...oldEvents, ...newEvents]; + // Generate a new execution ID for the rewound execution. + const newExecutionId = randomUUID().replace(/-/g, ""); + + // First pass: collect the task-scheduled IDs that correspond to failed activities so we can + // remove the matching taskScheduled events in the second pass. + const failedTaskIds = new Set(); + for (const event of allEvents) { + if (event.hasTaskfailed()) { + failedTaskIds.add(event.getTaskfailed()!.getTaskscheduledid()); + } + } + + const parentExecutionId = + rewindEvent.hasParentexecutionid() && rewindEvent.getParentexecutionid()!.getValue() + ? rewindEvent.getParentexecutionid()!.getValue() + : undefined; + + // Second pass: build the clean history. + const cleanHistory: pb.HistoryEvent[] = []; + for (const event of allEvents) { + if (event.hasTaskfailed()) { + continue; + } + if (event.hasTaskscheduled() && failedTaskIds.has(event.getEventid())) { + continue; + } + if (event.hasSuborchestrationinstancefailed()) { + continue; + } + if (event.hasExecutioncompleted()) { + continue; + } + + // Modify the executionStarted event: assign a fresh execution ID and, for + // sub-orchestrations, update the parent's execution ID so it matches the parent's new run. + if (event.hasExecutionstarted()) { + const eventCopy = pb.HistoryEvent.deserializeBinary(event.serializeBinary()); + const startedEvent = eventCopy.getExecutionstarted()!; + + let orchestrationInstance = startedEvent.getOrchestrationinstance(); + if (!orchestrationInstance) { + orchestrationInstance = new pb.OrchestrationInstance(); + startedEvent.setOrchestrationinstance(orchestrationInstance); + } + orchestrationInstance.setExecutionid(getStringValue(newExecutionId)); + + // Only update the parent's execution ID when this is actually a sub-orchestration (its + // executionStarted has a parentInstance). Writing through parentInstance for a top-level + // orchestration would materialize an empty one. + if (parentExecutionId && startedEvent.hasParentinstance()) { + const parentInstance = startedEvent.getParentinstance()!; + let parentOrchestrationInstance = parentInstance.getOrchestrationinstance(); + if (!parentOrchestrationInstance) { + parentOrchestrationInstance = new pb.OrchestrationInstance(); + parentInstance.setOrchestrationinstance(parentOrchestrationInstance); + } + parentOrchestrationInstance.setExecutionid(getStringValue(parentExecutionId)); + } + + cleanHistory.push(eventCopy); + continue; + } + + cleanHistory.push(event); + } + + const rewindAction = new pb.RewindOrchestrationAction(); + rewindAction.setNewhistoryList(cleanHistory); + + const action = new pb.OrchestratorAction(); + action.setId(-1); + action.setRewindorchestration(rewindAction); + + return { actions: [action], customStatus: undefined }; +} diff --git a/packages/durabletask-js/test/build-rewind-result.spec.ts b/packages/durabletask-js/test/build-rewind-result.spec.ts new file mode 100644 index 0000000..0129c8b --- /dev/null +++ b/packages/durabletask-js/test/build-rewind-result.spec.ts @@ -0,0 +1,540 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +import { StringValue } from "google-protobuf/google/protobuf/wrappers_pb"; +import * as pb from "../src/proto/orchestrator_service_pb"; +import { + newExecutionCompletedEvent, + newFailureDetails, + newOrchestratorStartedEvent, + newSubOrchestrationCompletedEvent, + newSubOrchestrationCreatedEvent, + newSubOrchestrationFailedEvent, + newTaskCompletedEvent, + newTaskFailedEvent, + newTaskScheduledEvent, +} from "../src/utils/pb-helper.util"; +import { buildRewindResult } from "../src/worker/rewind"; + +const TEST_INSTANCE_ID = "rewind-test-instance"; +const ORIGINAL_EXECUTION_ID = "original-exec-id"; + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +/** Create an executionStarted event with an explicit execution ID. */ +function makeExecutionStarted( + name: string, + instanceId: string = TEST_INSTANCE_ID, + executionId: string = ORIGINAL_EXECUTION_ID, + parentInstance?: pb.ParentInstanceInfo, +): pb.HistoryEvent { + const orchestrationInstance = new pb.OrchestrationInstance(); + orchestrationInstance.setInstanceid(instanceId); + orchestrationInstance.setExecutionid(new StringValue().setValue(executionId)); + + const startedEvent = new pb.ExecutionStartedEvent(); + startedEvent.setName(name); + startedEvent.setOrchestrationinstance(orchestrationInstance); + if (parentInstance) { + startedEvent.setParentinstance(parentInstance); + } + + const event = new pb.HistoryEvent(); + event.setEventid(-1); + event.setExecutionstarted(startedEvent); + return event; +} + +/** Create a parentInstance info with an explicit parent execution ID. */ +function makeParentInstance( + name: string, + instanceId: string, + executionId: string, + taskScheduledId = 5, +): pb.ParentInstanceInfo { + const parentOrchestrationInstance = new pb.OrchestrationInstance(); + parentOrchestrationInstance.setInstanceid(instanceId); + parentOrchestrationInstance.setExecutionid(new StringValue().setValue(executionId)); + + const parentInstance = new pb.ParentInstanceInfo(); + parentInstance.setTaskscheduledid(taskScheduledId); + parentInstance.setName(new StringValue().setValue(name)); + parentInstance.setOrchestrationinstance(parentOrchestrationInstance); + return parentInstance; +} + +/** Create an executionRewound history event. */ +function makeExecutionRewound(reason = "test rewind", parentExecutionId?: string): pb.HistoryEvent { + const rewound = new pb.ExecutionRewoundEvent(); + rewound.setReason(new StringValue().setValue(reason)); + if (parentExecutionId !== undefined) { + rewound.setParentexecutionid(new StringValue().setValue(parentExecutionId)); + } + + const event = new pb.HistoryEvent(); + event.setEventid(-1); + event.setExecutionrewound(rewound); + return event; +} + +/** Create an orchestratorCompleted history event. */ +function makeOrchestratorCompleted(): pb.HistoryEvent { + const event = new pb.HistoryEvent(); + event.setEventid(-1); + event.setOrchestratorcompleted(new pb.OrchestratorCompletedEvent()); + return event; +} + +/** Extract the clean history from a RewindOrchestrationAction result. */ +function getCleanHistory(result: { actions: pb.OrchestratorAction[] }): pb.HistoryEvent[] { + expect(result.actions).toHaveLength(1); + const action = result.actions[0]; + expect(action.hasRewindorchestration()).toBe(true); + return action.getRewindorchestration()!.getNewhistoryList(); +} + +// --------------------------------------------------------------------------- +// Tests: execution ID changes +// --------------------------------------------------------------------------- + +describe("buildRewindResult", () => { + it("assignsNewExecutionId: the executionStarted event gets a new, different execution ID", () => { + const oldEvents = [ + newOrchestratorStartedEvent(), + makeExecutionStarted("my_orch"), + newTaskScheduledEvent(1, "my_activity"), + newTaskFailedEvent(1, new Error("boom")), + makeOrchestratorCompleted(), + newExecutionCompletedEvent( + pb.OrchestrationStatus.ORCHESTRATION_STATUS_FAILED, + undefined, + newFailureDetails(new Error("boom")), + ), + ]; + const newEvents = [newOrchestratorStartedEvent(), makeExecutionRewound("retry")]; + + const result = buildRewindResult(oldEvents, newEvents); + const clean = getCleanHistory(result); + + const started = clean.filter((e) => e.hasExecutionstarted()); + expect(started).toHaveLength(1); + + const newExecId = started[0].getExecutionstarted()!.getOrchestrationinstance()!.getExecutionid()!.getValue(); + expect(newExecId).not.toEqual(ORIGINAL_EXECUTION_ID); + expect(newExecId.length).toBeGreaterThan(0); + }); + + it("preservesExecutionStartedFields: name and instanceId are preserved, only execution ID changes", () => { + const oldEvents = [ + newOrchestratorStartedEvent(), + makeExecutionStarted("preserve_me"), + newTaskScheduledEvent(1, "act"), + newTaskFailedEvent(1, new Error("fail")), + makeOrchestratorCompleted(), + newExecutionCompletedEvent( + pb.OrchestrationStatus.ORCHESTRATION_STATUS_FAILED, + undefined, + newFailureDetails(new Error("fail")), + ), + ]; + const newEvents = [newOrchestratorStartedEvent(), makeExecutionRewound("retry")]; + + const result = buildRewindResult(oldEvents, newEvents); + const clean = getCleanHistory(result); + + const started = clean.filter((e) => e.hasExecutionstarted())[0].getExecutionstarted()!; + expect(started.getName()).toEqual("preserve_me"); + expect(started.getOrchestrationinstance()!.getInstanceid()).toEqual(TEST_INSTANCE_ID); + }); + + it("updatesParentExecutionId: parent execution ID is updated when the rewind event carries one", () => { + const parentNewExecId = "parent-new-exec-id"; + const parentInfo = makeParentInstance("parent_orch", "parent-instance", "parent-old-exec-id"); + + const oldEvents = [ + newOrchestratorStartedEvent(), + makeExecutionStarted("child_orch", TEST_INSTANCE_ID, ORIGINAL_EXECUTION_ID, parentInfo), + newTaskScheduledEvent(1, "child_act"), + newTaskFailedEvent(1, new Error("child fail")), + makeOrchestratorCompleted(), + newExecutionCompletedEvent( + pb.OrchestrationStatus.ORCHESTRATION_STATUS_FAILED, + undefined, + newFailureDetails(new Error("child fail")), + ), + ]; + const newEvents = [newOrchestratorStartedEvent(), makeExecutionRewound("parent rewind", parentNewExecId)]; + + const result = buildRewindResult(oldEvents, newEvents); + const clean = getCleanHistory(result); + + const started = clean.filter((e) => e.hasExecutionstarted())[0].getExecutionstarted()!; + const actualParentExecId = started.getParentinstance()!.getOrchestrationinstance()!.getExecutionid()!.getValue(); + expect(actualParentExecId).toEqual(parentNewExecId); + }); + + it("noParentExecutionIdLeavesParentUnchanged: parent is untouched when the rewind event has no parentExecutionId", () => { + const parentExecId = "parent-exec-id-unchanged"; + const parentInfo = makeParentInstance("parent_orch", "parent-instance", parentExecId); + + const oldEvents = [ + newOrchestratorStartedEvent(), + makeExecutionStarted("child_orch", TEST_INSTANCE_ID, ORIGINAL_EXECUTION_ID, parentInfo), + newTaskScheduledEvent(1, "child_act"), + newTaskFailedEvent(1, new Error("fail")), + makeOrchestratorCompleted(), + newExecutionCompletedEvent( + pb.OrchestrationStatus.ORCHESTRATION_STATUS_FAILED, + undefined, + newFailureDetails(new Error("fail")), + ), + ]; + // No parentExecutionId on the rewind event. + const newEvents = [newOrchestratorStartedEvent(), makeExecutionRewound("top-level rewind")]; + + const result = buildRewindResult(oldEvents, newEvents); + const clean = getCleanHistory(result); + + const started = clean.filter((e) => e.hasExecutionstarted())[0].getExecutionstarted()!; + const actualParentExecId = started.getParentinstance()!.getOrchestrationinstance()!.getExecutionid()!.getValue(); + expect(actualParentExecId).toEqual(parentExecId); + }); + + // ------------------------------------------------------------------------- + // Tests: failed activity cleanup + // ------------------------------------------------------------------------- + + it("removesFailedActivityEvents: taskFailed and its taskScheduled are removed", () => { + const oldEvents = [ + newOrchestratorStartedEvent(), + makeExecutionStarted("orch"), + newTaskScheduledEvent(1, "my_act"), + newTaskFailedEvent(1, new Error("boom")), + makeOrchestratorCompleted(), + newExecutionCompletedEvent( + pb.OrchestrationStatus.ORCHESTRATION_STATUS_FAILED, + undefined, + newFailureDetails(new Error("boom")), + ), + ]; + const newEvents = [newOrchestratorStartedEvent(), makeExecutionRewound("retry")]; + + const result = buildRewindResult(oldEvents, newEvents); + const clean = getCleanHistory(result); + + expect(clean.some((e) => e.hasTaskfailed())).toBe(false); + expect(clean.some((e) => e.hasTaskscheduled() && e.getEventid() === 1)).toBe(false); + }); + + it("preservesSuccessfulActivity: successful taskScheduled + taskCompleted remain", () => { + const oldEvents = [ + newOrchestratorStartedEvent(), + makeExecutionStarted("orch"), + // Activity 1 succeeds + newTaskScheduledEvent(1, "good_act"), + newTaskCompletedEvent(1, JSON.stringify("ok")), + makeOrchestratorCompleted(), + // Activity 2 fails + newOrchestratorStartedEvent(), + newTaskScheduledEvent(2, "bad_act"), + newTaskFailedEvent(2, new Error("fail")), + makeOrchestratorCompleted(), + newExecutionCompletedEvent( + pb.OrchestrationStatus.ORCHESTRATION_STATUS_FAILED, + undefined, + newFailureDetails(new Error("fail")), + ), + ]; + const newEvents = [newOrchestratorStartedEvent(), makeExecutionRewound("retry")]; + + const result = buildRewindResult(oldEvents, newEvents); + const clean = getCleanHistory(result); + + // Activity 1's taskScheduled and taskCompleted should still be present. + expect(clean.some((e) => e.hasTaskscheduled() && e.getEventid() === 1)).toBe(true); + expect(clean.some((e) => e.hasTaskcompleted() && e.getTaskcompleted()!.getTaskscheduledid() === 1)).toBe(true); + // Activity 2's taskScheduled and taskFailed should be removed. + expect(clean.some((e) => e.hasTaskscheduled() && e.getEventid() === 2)).toBe(false); + expect(clean.some((e) => e.hasTaskfailed())).toBe(false); + }); + + // ------------------------------------------------------------------------- + // Tests: failed sub-orchestration cleanup + // ------------------------------------------------------------------------- + + it("removesFailedSubOrchEvents: subOrchestrationInstanceFailed removed, created kept", () => { + const oldEvents = [ + newOrchestratorStartedEvent(), + makeExecutionStarted("parent_orch"), + newSubOrchestrationCreatedEvent(1, "child_orch", "child-id"), + makeOrchestratorCompleted(), + newOrchestratorStartedEvent(), + newSubOrchestrationFailedEvent(1, new Error("child exploded")), + makeOrchestratorCompleted(), + newExecutionCompletedEvent( + pb.OrchestrationStatus.ORCHESTRATION_STATUS_FAILED, + undefined, + newFailureDetails(new Error("child exploded")), + ), + ]; + const newEvents = [newOrchestratorStartedEvent(), makeExecutionRewound("rewind child")]; + + const result = buildRewindResult(oldEvents, newEvents); + const clean = getCleanHistory(result); + + expect(clean.some((e) => e.hasSuborchestrationinstancefailed())).toBe(false); + // Sub-orchestrations use subOrchestrationInstanceCreated, not taskScheduled. + expect(clean.some((e) => e.hasTaskscheduled())).toBe(false); + // The created event is preserved so the backend can identify which sub-orch to rewind. + expect(clean.some((e) => e.hasSuborchestrationinstancecreated())).toBe(true); + }); + + it("preservesSuccessfulSubOrchestration: successful sub-orch events preserved", () => { + const oldEvents = [ + newOrchestratorStartedEvent(), + makeExecutionStarted("parent_orch"), + // Sub-orch 1 succeeds + newSubOrchestrationCreatedEvent(1, "child_ok", "child-ok-id"), + makeOrchestratorCompleted(), + newOrchestratorStartedEvent(), + newSubOrchestrationCompletedEvent(1, JSON.stringify("child result")), + // Sub-orch 2 fails + newSubOrchestrationCreatedEvent(2, "child_fail", "child-fail-id"), + makeOrchestratorCompleted(), + newOrchestratorStartedEvent(), + newSubOrchestrationFailedEvent(2, new Error("fail")), + makeOrchestratorCompleted(), + newExecutionCompletedEvent( + pb.OrchestrationStatus.ORCHESTRATION_STATUS_FAILED, + undefined, + newFailureDetails(new Error("fail")), + ), + ]; + const newEvents = [newOrchestratorStartedEvent(), makeExecutionRewound("retry")]; + + const result = buildRewindResult(oldEvents, newEvents); + const clean = getCleanHistory(result); + + const createdIds = clean + .filter((e) => e.hasSuborchestrationinstancecreated()) + .map((e) => e.getSuborchestrationinstancecreated()!.getInstanceid()); + expect(createdIds).toContain("child-ok-id"); + const completedSubIds = clean + .filter((e) => e.hasSuborchestrationinstancecompleted()) + .map((e) => e.getSuborchestrationinstancecompleted()!.getTaskscheduledid()); + expect(completedSubIds).toContain(1); + // Sub-orch 2's failed event should be removed. + expect(clean.some((e) => e.hasSuborchestrationinstancefailed())).toBe(false); + // Sub-orch 2's created event should be kept (for backend recursive rewind). + expect(createdIds).toContain("child-fail-id"); + }); + + // ------------------------------------------------------------------------- + // Tests: executionCompleted removal + // ------------------------------------------------------------------------- + + it("removesExecutionCompleted: executionCompleted events are stripped", () => { + const oldEvents = [ + newOrchestratorStartedEvent(), + makeExecutionStarted("orch"), + newTaskScheduledEvent(1, "act"), + newTaskFailedEvent(1, new Error("fail")), + makeOrchestratorCompleted(), + newExecutionCompletedEvent( + pb.OrchestrationStatus.ORCHESTRATION_STATUS_FAILED, + undefined, + newFailureDetails(new Error("fail")), + ), + ]; + const newEvents = [newOrchestratorStartedEvent(), makeExecutionRewound("retry")]; + + const result = buildRewindResult(oldEvents, newEvents); + const clean = getCleanHistory(result); + + expect(clean.some((e) => e.hasExecutioncompleted())).toBe(false); + }); + + // ------------------------------------------------------------------------- + // Tests: orchestratorStarted/Completed preservation + // ------------------------------------------------------------------------- + + it("keepsOrchestratorStartedAndCompleted: bookend events are preserved", () => { + const oldEvents = [ + newOrchestratorStartedEvent(), + makeExecutionStarted("orch"), + newTaskScheduledEvent(1, "act"), + makeOrchestratorCompleted(), + newOrchestratorStartedEvent(), + newTaskFailedEvent(1, new Error("fail")), + makeOrchestratorCompleted(), + newExecutionCompletedEvent( + pb.OrchestrationStatus.ORCHESTRATION_STATUS_FAILED, + undefined, + newFailureDetails(new Error("fail")), + ), + ]; + const newEvents = [newOrchestratorStartedEvent(), makeExecutionRewound("retry")]; + + const result = buildRewindResult(oldEvents, newEvents); + const clean = getCleanHistory(result); + + const startedCount = clean.filter((e) => e.hasOrchestratorstarted()).length; + const completedCount = clean.filter((e) => e.hasOrchestratorcompleted()).length; + // old_events has 2 orchestratorStarted + 2 orchestratorCompleted, + // new_events adds 1 orchestratorStarted. All should be kept. + expect(startedCount).toBeGreaterThanOrEqual(2); + expect(completedCount).toBeGreaterThanOrEqual(2); + }); + + // ------------------------------------------------------------------------- + // Tests: executionRewound event preserved + // ------------------------------------------------------------------------- + + it("keepsExecutionRewoundEvent: the executionRewound event itself remains", () => { + const oldEvents = [ + newOrchestratorStartedEvent(), + makeExecutionStarted("orch"), + newTaskScheduledEvent(1, "act"), + newTaskFailedEvent(1, new Error("fail")), + makeOrchestratorCompleted(), + newExecutionCompletedEvent( + pb.OrchestrationStatus.ORCHESTRATION_STATUS_FAILED, + undefined, + newFailureDetails(new Error("fail")), + ), + ]; + const newEvents = [newOrchestratorStartedEvent(), makeExecutionRewound("rewind reason")]; + + const result = buildRewindResult(oldEvents, newEvents); + const clean = getCleanHistory(result); + + const rewoundEvents = clean.filter((e) => e.hasExecutionrewound()); + expect(rewoundEvents).toHaveLength(1); + expect(rewoundEvents[0].getExecutionrewound()!.getReason()!.getValue()).toEqual("rewind reason"); + }); + + // ------------------------------------------------------------------------- + // Tests: mixed scenario + // ------------------------------------------------------------------------- + + it("mixedActivitiesAndSubOrchestrations: only failed items cleaned, execution ID updated", () => { + const oldEvents = [ + newOrchestratorStartedEvent(), + makeExecutionStarted("complex_orch"), + // Activity 1 succeeds (eventId=1) + newTaskScheduledEvent(1, "good_activity"), + newTaskCompletedEvent(1, JSON.stringify("good")), + // Sub-orch A succeeds (eventId=2) + newSubOrchestrationCreatedEvent(2, "child_ok", "child-ok-id"), + makeOrchestratorCompleted(), + newOrchestratorStartedEvent(), + newSubOrchestrationCompletedEvent(2, JSON.stringify("child ok result")), + // Activity 3 fails (eventId=3) + newTaskScheduledEvent(3, "bad_activity"), + newTaskFailedEvent(3, new Error("act fail")), + makeOrchestratorCompleted(), + // Sub-orch B fails (eventId=4) + newOrchestratorStartedEvent(), + newSubOrchestrationCreatedEvent(4, "child_fail", "child-fail-id"), + makeOrchestratorCompleted(), + newOrchestratorStartedEvent(), + newSubOrchestrationFailedEvent(4, new Error("sub orch fail")), + makeOrchestratorCompleted(), + newExecutionCompletedEvent( + pb.OrchestrationStatus.ORCHESTRATION_STATUS_FAILED, + undefined, + newFailureDetails(new Error("overall fail")), + ), + ]; + const newEvents = [newOrchestratorStartedEvent(), makeExecutionRewound("fix everything")]; + + const result = buildRewindResult(oldEvents, newEvents); + const clean = getCleanHistory(result); + + // --- Execution ID changed --- + const started = clean.filter((e) => e.hasExecutionstarted()); + expect(started).toHaveLength(1); + expect(started[0].getExecutionstarted()!.getOrchestrationinstance()!.getExecutionid()!.getValue()).not.toEqual( + ORIGINAL_EXECUTION_ID, + ); + + // --- Successful activity 1 preserved --- + expect(clean.some((e) => e.hasTaskscheduled() && e.getEventid() === 1)).toBe(true); + expect(clean.some((e) => e.hasTaskcompleted() && e.getTaskcompleted()!.getTaskscheduledid() === 1)).toBe(true); + + // --- Failed activity 3 removed --- + expect(clean.some((e) => e.hasTaskscheduled() && e.getEventid() === 3)).toBe(false); + expect(clean.some((e) => e.hasTaskfailed())).toBe(false); + + // --- Successful sub-orch A preserved --- + const createdIds = clean + .filter((e) => e.hasSuborchestrationinstancecreated()) + .map((e) => e.getSuborchestrationinstancecreated()!.getInstanceid()); + expect(createdIds).toContain("child-ok-id"); + const completedSubIds = clean + .filter((e) => e.hasSuborchestrationinstancecompleted()) + .map((e) => e.getSuborchestrationinstancecompleted()!.getTaskscheduledid()); + expect(completedSubIds).toContain(2); + + // --- Failed sub-orch B: failed event removed, created kept --- + expect(clean.some((e) => e.hasSuborchestrationinstancefailed())).toBe(false); + expect(createdIds).toContain("child-fail-id"); + + // --- executionCompleted removed --- + expect(clean.some((e) => e.hasExecutioncompleted())).toBe(false); + + // --- executionRewound preserved --- + expect(clean.some((e) => e.hasExecutionrewound())).toBe(true); + }); + + it("doesNotMutateOriginalEvents: original history events are not modified in place", () => { + const esEvent = makeExecutionStarted("orch"); + const originalExecId = esEvent.getExecutionstarted()!.getOrchestrationinstance()!.getExecutionid()!.getValue(); + + const oldEvents = [ + newOrchestratorStartedEvent(), + esEvent, + newTaskScheduledEvent(1, "act"), + newTaskFailedEvent(1, new Error("fail")), + makeOrchestratorCompleted(), + newExecutionCompletedEvent( + pb.OrchestrationStatus.ORCHESTRATION_STATUS_FAILED, + undefined, + newFailureDetails(new Error("fail")), + ), + ]; + const newEvents = [newOrchestratorStartedEvent(), makeExecutionRewound("retry")]; + + buildRewindResult(oldEvents, newEvents); + + // The original executionStarted event should NOT be mutated. + const actual = esEvent.getExecutionstarted()!.getOrchestrationinstance()!.getExecutionid()!.getValue(); + expect(actual).toEqual(originalExecId); + }); + + it("resultActionStructure: exactly one action with id=-1 and a rewindOrchestration field", () => { + const oldEvents = [ + newOrchestratorStartedEvent(), + makeExecutionStarted("orch"), + newTaskScheduledEvent(1, "act"), + newTaskFailedEvent(1, new Error("fail")), + makeOrchestratorCompleted(), + newExecutionCompletedEvent( + pb.OrchestrationStatus.ORCHESTRATION_STATUS_FAILED, + undefined, + newFailureDetails(new Error("fail")), + ), + ]; + const newEvents = [newOrchestratorStartedEvent(), makeExecutionRewound("retry")]; + + const result = buildRewindResult(oldEvents, newEvents); + + expect(result.actions).toHaveLength(1); + const action = result.actions[0]; + expect(action.getId()).toEqual(-1); + expect(action.hasRewindorchestration()).toBe(true); + expect(result.customStatus).toBeUndefined(); + }); +}); diff --git a/packages/durabletask-js/test/rewind-e2e.spec.ts b/packages/durabletask-js/test/rewind-e2e.spec.ts new file mode 100644 index 0000000..6841587 --- /dev/null +++ b/packages/durabletask-js/test/rewind-e2e.spec.ts @@ -0,0 +1,311 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +import { + InMemoryOrchestrationBackend, + TestOrchestrationClient, + TestOrchestrationWorker, + OrchestrationStatus, + ActivityContext, + OrchestrationContext, + TOrchestrator, +} from "../src"; + +describe("Rewind (in-memory e2e)", () => { + let backend: InMemoryOrchestrationBackend; + let client: TestOrchestrationClient; + let worker: TestOrchestrationWorker; + + beforeEach(() => { + backend = new InMemoryOrchestrationBackend(); + client = new TestOrchestrationClient(backend); + worker = new TestOrchestrationWorker(backend); + }); + + afterEach(async () => { + if (worker) { + try { + await worker.stop(); + } catch { + // Ignore if not running + } + } + backend.reset(); + }); + + it("rewindFailedActivity: a failed activity re-runs and the orchestration completes", async () => { + let activityCallCount = 0; + let shouldFail = true; + + const failingActivity = (_: ActivityContext, input: string): string => { + activityCallCount += 1; + if (shouldFail) { + throw new Error("Simulated failure"); + } + return `Hello, ${input}!`; + }; + + const orchestrator: TOrchestrator = async function* (ctx: OrchestrationContext, input: string): any { + return yield ctx.callActivity(failingActivity, input); + }; + + worker.addOrchestrator(orchestrator); + worker.addActivity(failingActivity); + await worker.start(); + + const id = await client.scheduleNewOrchestration(orchestrator, "World"); + let state = await client.waitForOrchestrationCompletion(id, true, 10); + + // The orchestration should have failed. + expect(state).toBeDefined(); + expect(state?.runtimeStatus).toEqual(OrchestrationStatus.FAILED); + + // Fix the activity so it now succeeds, then rewind. + shouldFail = false; + await client.rewindOrchestration(id, "retry after fix"); + + state = await client.waitForOrchestrationCompletion(id, true, 10); + + expect(state).toBeDefined(); + expect(state?.runtimeStatus).toEqual(OrchestrationStatus.COMPLETED); + expect(state?.serializedOutput).toEqual(JSON.stringify("Hello, World!")); + expect(state?.failureDetails).toBeUndefined(); + // Activity was called twice (once failed, once succeeded after rewind). + expect(activityCallCount).toEqual(2); + }); + + it("rewindPreservesSuccessfulResults: only the failed activity re-runs; successful result is replayed", async () => { + const callTracker = { first: 0, second: 0 }; + let shouldFailSecond = true; + + const firstActivity = (_: ActivityContext, input: string): string => { + callTracker.first += 1; + return `first:${input}`; + }; + + const secondActivity = (_: ActivityContext, input: string): string => { + callTracker.second += 1; + if (shouldFailSecond) { + throw new Error("Temporary failure"); + } + return `second:${input}`; + }; + + const orchestrator: TOrchestrator = async function* (ctx: OrchestrationContext, input: string): any { + const r1 = yield ctx.callActivity(firstActivity, input); + const r2 = yield ctx.callActivity(secondActivity, input); + return [r1, r2]; + }; + + worker.addOrchestrator(orchestrator); + worker.addActivity(firstActivity); + worker.addActivity(secondActivity); + await worker.start(); + + const id = await client.scheduleNewOrchestration(orchestrator, "test"); + let state = await client.waitForOrchestrationCompletion(id, true, 10); + + // The orchestration should have failed (secondActivity fails). + expect(state).toBeDefined(); + expect(state?.runtimeStatus).toEqual(OrchestrationStatus.FAILED); + + // Fix secondActivity so it now succeeds, then rewind. + shouldFailSecond = false; + await client.rewindOrchestration(id, "retry"); + state = await client.waitForOrchestrationCompletion(id, true, 10); + + expect(state).toBeDefined(); + expect(state?.runtimeStatus).toEqual(OrchestrationStatus.COMPLETED); + expect(state?.serializedOutput).toEqual(JSON.stringify(["first:test", "second:test"])); + expect(state?.failureDetails).toBeUndefined(); + // firstActivity should NOT be re-executed — its result is replayed from history. + expect(callTracker.first).toEqual(1); + // secondActivity was called at least twice (once failed, once succeeded). + expect(callTracker.second).toBeGreaterThanOrEqual(2); + }); + + it("rewindNotFound: rewinding a non-existent instance rejects", async () => { + await worker.start(); + await expect(client.rewindOrchestration("nonexistent-id")).rejects.toThrow(); + }); + + it("rewindNonFailedInstance: rewinding a completed (non-failed) instance rejects", async () => { + const orchestrator: TOrchestrator = async (_: OrchestrationContext) => "done"; + + worker.addOrchestrator(orchestrator); + await worker.start(); + + const id = await client.scheduleNewOrchestration(orchestrator); + const state = await client.waitForOrchestrationCompletion(id, true, 10); + expect(state).toBeDefined(); + expect(state?.runtimeStatus).toEqual(OrchestrationStatus.COMPLETED); + + await expect(client.rewindOrchestration(id)).rejects.toThrow(); + }); + + it("rewindWithSubOrchestration: recursively rewinds a failed sub-orchestration", async () => { + let subCallCount = 0; + + const childActivity = (_: ActivityContext, input: string): string => { + subCallCount += 1; + if (subCallCount === 1) { + throw new Error("Child failure"); + } + return `child:${input}`; + }; + + const childOrchestrator: TOrchestrator = async function* (ctx: OrchestrationContext, input: string): any { + return yield ctx.callActivity(childActivity, input); + }; + + const parentOrchestrator: TOrchestrator = async function* (ctx: OrchestrationContext, input: string): any { + const result = yield ctx.callSubOrchestrator(childOrchestrator, input); + return `parent:${result}`; + }; + + worker.addOrchestrator(parentOrchestrator); + worker.addOrchestrator(childOrchestrator); + worker.addActivity(childActivity); + await worker.start(); + + const id = await client.scheduleNewOrchestration(parentOrchestrator, "data"); + let state = await client.waitForOrchestrationCompletion(id, true, 10); + + // Parent should fail because child failed. + expect(state).toBeDefined(); + expect(state?.runtimeStatus).toEqual(OrchestrationStatus.FAILED); + + // Rewind — childActivity will succeed on retry. + await client.rewindOrchestration(id, "sub-orch fix"); + state = await client.waitForOrchestrationCompletion(id, true, 10); + + expect(state).toBeDefined(); + expect(state?.runtimeStatus).toEqual(OrchestrationStatus.COMPLETED); + expect(state?.serializedOutput).toEqual(JSON.stringify("parent:child:data")); + expect(subCallCount).toEqual(2); + }); + + it("rewindWithoutReason: rewind works when no reason is provided", async () => { + let callCount = 0; + + const flakyActivity = (_: ActivityContext): string => { + callCount += 1; + if (callCount === 1) { + throw new Error("Boom"); + } + return "ok"; + }; + + const orchestrator: TOrchestrator = async function* (ctx: OrchestrationContext): any { + return yield ctx.callActivity(flakyActivity); + }; + + worker.addOrchestrator(orchestrator); + worker.addActivity(flakyActivity); + await worker.start(); + + const id = await client.scheduleNewOrchestration(orchestrator); + let state = await client.waitForOrchestrationCompletion(id, true, 10); + expect(state).toBeDefined(); + expect(state?.runtimeStatus).toEqual(OrchestrationStatus.FAILED); + + // Rewind without a reason. + await client.rewindOrchestration(id); + state = await client.waitForOrchestrationCompletion(id, true, 10); + + expect(state).toBeDefined(); + expect(state?.runtimeStatus).toEqual(OrchestrationStatus.COMPLETED); + expect(state?.serializedOutput).toEqual(JSON.stringify("ok")); + }); + + it("rewindPurgedSubOrchestration: a purged sub-orchestration is re-run when the parent is rewound", async () => { + let childCallCount = 0; + + const childActivity = (_: ActivityContext, input: string): string => { + childCallCount += 1; + if (childCallCount === 1) { + throw new Error("Child failure"); + } + return `child:${input}`; + }; + + const childOrchestrator: TOrchestrator = async function* (ctx: OrchestrationContext, input: string): any { + return yield ctx.callActivity(childActivity, input); + }; + + const parentOrchestrator: TOrchestrator = async function* (ctx: OrchestrationContext, input: string): any { + const result = yield ctx.callSubOrchestrator(childOrchestrator, input, { + instanceId: "sub-orch-to-purge", + }); + return `parent:${result}`; + }; + + worker.addOrchestrator(parentOrchestrator); + worker.addOrchestrator(childOrchestrator); + worker.addActivity(childActivity); + await worker.start(); + + const id = await client.scheduleNewOrchestration(parentOrchestrator, "data"); + let state = await client.waitForOrchestrationCompletion(id, true, 10); + + // Parent should fail because child failed. + expect(state).toBeDefined(); + expect(state?.runtimeStatus).toEqual(OrchestrationStatus.FAILED); + + // Purge the sub-orchestration so it must be completely re-run. + await client.purgeOrchestration("sub-orch-to-purge"); + + // Rewind the parent — child will be re-scheduled and succeed. + await client.rewindOrchestration(id, "purge and retry"); + state = await client.waitForOrchestrationCompletion(id, true, 10); + + expect(state).toBeDefined(); + expect(state?.runtimeStatus).toEqual(OrchestrationStatus.COMPLETED); + expect(state?.serializedOutput).toEqual(JSON.stringify("parent:child:data")); + expect(childCallCount).toEqual(2); + }); + + it("rewindTwice: the same orchestration can be rewound twice after failing twice", async () => { + let callCount = 0; + + const flakyActivity = (_: ActivityContext, input: string): string => { + callCount += 1; + // Fail on the 1st and 2nd calls; succeed on the 3rd. + if (callCount <= 2) { + throw new Error(`Failure #${callCount}`); + } + return `Hello, ${input}!`; + }; + + const orchestrator: TOrchestrator = async function* (ctx: OrchestrationContext, input: string): any { + return yield ctx.callActivity(flakyActivity, input); + }; + + worker.addOrchestrator(orchestrator); + worker.addActivity(flakyActivity); + await worker.start(); + + const id = await client.scheduleNewOrchestration(orchestrator, "World"); + let state = await client.waitForOrchestrationCompletion(id, true, 10); + + // First failure. + expect(state).toBeDefined(); + expect(state?.runtimeStatus).toEqual(OrchestrationStatus.FAILED); + + // First rewind — activity fails again (callCount === 2). + await client.rewindOrchestration(id, "first rewind"); + state = await client.waitForOrchestrationCompletion(id, true, 10); + + expect(state).toBeDefined(); + expect(state?.runtimeStatus).toEqual(OrchestrationStatus.FAILED); + + // Second rewind — activity succeeds (callCount === 3). + await client.rewindOrchestration(id, "second rewind"); + state = await client.waitForOrchestrationCompletion(id, true, 10); + + expect(state).toBeDefined(); + expect(state?.runtimeStatus).toEqual(OrchestrationStatus.COMPLETED); + expect(state?.serializedOutput).toEqual(JSON.stringify("Hello, World!")); + expect(callCount).toEqual(3); + }); +}); diff --git a/test/e2e-azuremanaged/rewind.spec.ts b/test/e2e-azuremanaged/rewind.spec.ts index 8f696ee..8bd53f3 100644 --- a/test/e2e-azuremanaged/rewind.spec.ts +++ b/test/e2e-azuremanaged/rewind.spec.ts @@ -103,7 +103,7 @@ describe("Rewind Instance E2E Tests", () => { }); // Skip these tests if the backend doesn't support rewind (emulator returns UNIMPLEMENTED) - it.skip("should rewind a failed orchestration instance (requires backend support)", async () => { + it("should rewind a failed orchestration instance (requires backend support)", async () => { const instanceId = generateUniqueInstanceId("rewind-test"); taskHubWorker.addOrchestrator(failOnceOrchestrator); @@ -127,9 +127,9 @@ describe("Rewind Instance E2E Tests", () => { expect(rewindedState).toBeDefined(); expect(rewindedState?.runtimeStatus).toBe(OrchestrationStatus.COMPLETED); expect(rewindedState?.serializedOutput).toContain("Success on attempt"); - }); + }, 90000); - it.skip("should rewind a failed orchestration with a descriptive reason (requires backend support)", async () => { + it("should rewind a failed orchestration with a descriptive reason (requires backend support)", async () => { const instanceId = generateUniqueInstanceId("rewind-reason"); const rewindReason = "Rewinding due to transient network failure"; @@ -147,7 +147,7 @@ describe("Rewind Instance E2E Tests", () => { // Verify it can complete after rewind const rewindedState = await taskHubClient.waitForOrchestrationCompletion(instanceId, true, 30); expect(rewindedState?.runtimeStatus).toBe(OrchestrationStatus.COMPLETED); - }); + }, 90000); }); describe("rewindInstance - negative cases", () => { @@ -157,8 +157,8 @@ describe("Rewind Instance E2E Tests", () => { }; // An orchestrator that waits for an event (stays in Running state) - const waitingOrchestrator: TOrchestrator = async (ctx: OrchestrationContext) => { - const approval = await ctx.waitForExternalEvent("approval"); + const waitingOrchestrator: TOrchestrator = async function* (ctx: OrchestrationContext): any { + const approval = yield ctx.waitForExternalEvent("approval"); return `Approved: ${approval}`; }; @@ -185,7 +185,7 @@ describe("Rewind Instance E2E Tests", () => { await expect(taskHubClient.rewindInstance(instanceId, "Test rewind")).rejects.toThrow(); }, 60000); - it.skip("should throw an error when rewinding a running orchestration (requires backend support)", async () => { + it("should throw an error when rewinding a running orchestration (requires backend support)", async () => { const instanceId = generateUniqueInstanceId("rewind-running"); taskHubWorker.addOrchestrator(waitingOrchestrator); @@ -208,9 +208,9 @@ describe("Rewind Instance E2E Tests", () => { await taskHubClient.terminateOrchestration(instanceId, "Test cleanup"); await taskHubClient.waitForOrchestrationCompletion(instanceId, false, 30); } - }); + }, 60000); - it.skip("should throw an error when rewinding a terminated orchestration (requires backend support)", async () => { + it("should throw an error when rewinding a terminated orchestration (requires backend support)", async () => { const instanceId = generateUniqueInstanceId("rewind-terminated"); taskHubWorker.addOrchestrator(waitingOrchestrator); @@ -227,7 +227,7 @@ describe("Rewind Instance E2E Tests", () => { // Try to rewind a terminated orchestration - should fail await expect(taskHubClient.rewindInstance(instanceId, "Test rewind")).rejects.toThrow(); - }); + }, 60000); it("should throw an error when instanceId is empty", async () => { await expect(taskHubClient.rewindInstance("", "Test rewind")).rejects.toThrow("instanceId is required");