Merge #158350

158350: jobs: record reason in job status during for-cause pauses r=dt a=dt

When pausing for a specified reason, that reason should be shown as the secondary, informative status of the job aka its 'running_status'. This is done by setting the job's status using StatusStorage. Furthermore, a status set in pause-requested should be kept when the job moves to paused, unlike a status from running, as the status in pause-requested explains why the job moved to paused via the pause-request.

Release note (ops change): Jobs that are paused due to a specific reason, including jobs which pause themselves when encountering errors such as running out of disk space, now record that reason in their displayed status field of SHOW JOBS. Epic: none.

Co-authored-by: David Taylor <davidt@davidt.io>

Author: craig[bot]

pkg/jobs/job_info_storage_test.go

@@ -896,9 +896,15 @@ func TestJobPauseStateTransitionsRecorded(t *testing.T) {
 	require.NoError(t, err)
 	jobutils.WaitForJobToRun(t, sql, job.ID())
 
-	sql.Exec(t, "PAUSE JOB $1", job.ID())
+	sql.Exec(t, "PAUSE JOB $1 WITH REASON = 'test pause reason'", job.ID())
 	jobutils.WaitForJobToPause(t, sql, job.ID())
 
+	// Verify the running_status shows the pause reason via SHOW JOB.
+	sql.CheckQueryResults(t,
+		fmt.Sprintf("SELECT running_status FROM [SHOW JOB %d]", job.ID()),
+		[][]string{{"pausing: test pause reason"}},
+	)
+
 	sql.Exec(t, "RESUME JOB $1", job.ID())
 	jobutils.WaitForJobToRun(t, sql, job.ID())
 

pkg/jobs/update.go

@@ -243,8 +243,12 @@ WHERE id = $1
 			return err
 		}
 		// If we are changing state, we should clear out the status, unless
-		// we are about to set it to something instead.
-		if progress == nil || progress.StatusMessage == "" {
+		// we are about to set it to something instead or if we are coming from a
+		// pause requested state, in which case we already cleared it out once when
+		// we entered the pause requested state and may have since set it to a pause
+		// reason which we now want to preserve.
+		noNewStatus := progress == nil || progress.StatusMessage == ""
+		if noNewStatus && ju.md.State != StatePauseRequested {
 			if err := j.StatusStorage().Clear(ctx, u.txn); err != nil {
 				return err
 			}
@@ -403,6 +407,11 @@ func (ju *JobUpdater) PauseRequestedWithFunc(
 			return err
 		}
 	}
+	if reason != "" {
+		if err := StatusStorage(md.ID).Set(ctx, txn, fmt.Sprintf("pausing: %s", reason)); err != nil {
+			return err
+		}
+	}
 	ju.UpdateState(StatePauseRequested)
 	md.Payload.PauseReason = reason
 	ju.UpdatePayload(md.Payload)