Soft manual steps (allowed to fail) (#15)

* Soft manual steps (allowed to fail)

* Soft manual steps (allowed to fail) - Readme

---------

Co-authored-by: Max Soloviov <max.soloviov@keenethics.com>

Author: soloviovmax

README.md

@@ -30,6 +30,20 @@ test('example with manual step', async ({ page, manualStep }) => {
 
 When `manualStep` is called the test pauses and the Cyborg Test UI window appears. Use the `✅ Step passed` or `❌ Step failed` buttons to resume the test. Failing a step throws an error so your CI can detect it.
 
+## Soft Fail for Manual Steps
+
+You can use `manualStep.soft` to mark a manual step as a soft fail. If a soft manual step fails, the test will continue running, and the failure will be annotated as a soft fail (warning) in the report.
+
+**Usage:**
+
+```ts
+await manualStep('This is a hard manual step'); // Test fails if this step fails
+await manualStep.soft('This is a soft manual step'); // Test continues if this step fails, and a warning is shown
+```
+
+- Soft fails are shown as warnings in the UI and annotated in the test report.
+- Use soft fails for non-critical manual verifications where you want to highlight issues but not fail the entire test.
+
 ## Analytics Configuration
 
 The package includes Google Analytics integration that is enabled by default. The following data is collected:

package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "@cyborgtests/test",
-  "version": "0.0.6",
+  "version": "0.0.7",
   "description": "Powerfull extension for Playwright, that allows you to include manual verification steps in your automated test flow",
   "main": "dist/index.umd.js",
   "module": "dist/index.es.js",

package.json

@@ -2,6 +2,7 @@ import {
   Browser,
   BrowserContext,
   Page,
+  expect,
   test as pwTest,
 } from "@playwright/test";
 import { chromium } from "playwright";
@@ -59,21 +60,23 @@ const test = pwTest.extend<{
     server.kill();
   },
   manualStep: async ({ testControl, page, browser, context }, use) => {
-    const manualStep = async (stepName: string) =>
+    const manualStep = async (stepName: string, params: { isSoft?: boolean } = {}) =>
       await test.step(
-        stepName,
+        `✋ [MANUAL] ${stepName}`,
         async () => {
           await testControl.page.evaluate((_testName) => {
             (window as any)?.testUtils?.setTestName(_testName);
           }, test.info().title);
 
           // Write current step name
-          await testControl.page.evaluate((_stepName) => {
-            (window as any)?.testUtils?.addStep(_stepName);
-          }, stepName);
+          await testControl.page.evaluate(
+            ({ stepName, params }) => {
+              (window as any).testUtils?.addStep(stepName, params);
+            },
+            { stepName, params }
+          );
 
           // Pause for manual step
-
           await testControl.page.pause();
 
           // If last step failed, throw error
@@ -85,7 +88,32 @@ const test = pwTest.extend<{
             return false;
           });
           if (hasFailed) {
-            throw new TestFailedError(stepName as string);
+            const reason = await testControl.page.evaluate(() => {
+              const reason = (window as any).testUtils?.failedReason || '';
+              delete (window as any).testUtils.failedReason;
+              return reason;
+            });
+            const errorMessage = `${stepName}${reason ? ` - ${reason}` : ''}`;
+            throw new TestFailedError(errorMessage);
+          }
+        },
+        { box: true }
+      );
+    manualStep.soft = async (stepName: string) =>
+      await test.step(
+        `✋ [MANUAL][SOFT] ${stepName}`,
+        async () => {
+          try {
+            await manualStep(stepName, { isSoft: true });
+          } catch (err) {
+            test.info().annotations.push({
+              type: 'softFail',
+              description: `Soft fail in manual step: ${(err as Error).message}`,
+            });
+            // This will mark the step as failed, but not fail the test
+            await expect.soft(false, `Soft fail in manual step: ${(err as Error).message}`).toBeTruthy();
+            // Optionally log
+            console.warn(`Soft fail in manual step: ${stepName}`, err);
           }
         },
         { box: true }

src/test.ts

@@ -1,25 +1,33 @@
 import { useTestStore } from '../store/TestStore';
 import React, { Fragment } from 'react';
 
+const getClassName = (status: string) => {
+  if (status === 'pass') {
+    return 'step-pass';
+  } else if (status === 'fail') {
+    return 'step-fail';
+  } else if (status === 'warning') {
+    return 'step-warning';
+  } else {
+    return '';
+  }
+};
+
 export default function StepsList() {
   const { state } = useTestStore();
+
   return (
     <Fragment>
       <h4>Steps:</h4>
       <ol id="stepsList">
         {state.steps.map((step, idx) => (
           <li
             key={idx}
-            className={
-              step.status === 'pass'
-                ? 'step-pass'
-                : step.status === 'fail'
-                ? 'step-fail'
-                : ''
-            }
+            className={getClassName(step.status)}
           >
             {step.status === 'pass' && '✅ '}
             {step.status === 'fail' && '❌ '}
+            {step.status === 'warning' && '⚠️ '}
             {step.text}
             {step.status === 'fail' && step.reason ? ` - ${step.reason}` : ''}
           </li>

src/ui/components/StepsList.tsx

@@ -13,8 +13,8 @@ export default function TestControlPanel() {
       setTestName: (name: string) => {
         dispatch({ type: 'SET_TEST_NAME', payload: name });
       },
-      addStep: (step: string) => {
-        dispatch({ type: 'ADD_STEP', payload: step });
+      addStep: (step: string, params: { isSoft?: boolean } = {}) => {
+        dispatch({ type: 'ADD_STEP', payload: { step, isSoft: params.isSoft || false } });
       },
     };
     // Cleanup

src/ui/components/TestControlPanel.tsx

@@ -3,18 +3,24 @@ import { useTestStore } from '../store/TestStore';
 import { trackEvent } from '../../utils/analytics';
 
 export default function TestControls() {
+  const { state } = useTestStore();
+
   const { dispatch } = useTestStore();
   const [failureReason, setFailureReason] = useState('');
 
   const trackButtonClick = (buttonName: string) => {
     trackEvent(`app_${buttonName}_click`);
   };
 
+  const controlButtonsAreDisabled = state.steps[state.steps.length - 1]?.status !== 'pending';
+
   return (
     <Fragment>
       <button
-        className="btn-success"
+        className="btn btn-success"
+        disabled={controlButtonsAreDisabled}
         onClick={() => {
+          if (controlButtonsAreDisabled) return;
           dispatch({ type: 'PASS_STEP' });
           (window as any).playwright?.resume();
           trackButtonClick('pass_step');
@@ -31,10 +37,13 @@ export default function TestControls() {
       />
 
       <button
-        className="btn-danger"
+        className="btn btn-danger"
+        disabled={controlButtonsAreDisabled}
         onClick={() => {
+          if (controlButtonsAreDisabled) return;
           dispatch({ type: 'FAIL_STEP', payload: failureReason || 'No failure reason provided' });
           (window as any).testUtils.hasFailed = true;
+          (window as any).testUtils.failedReason = failureReason;
           (window as any).playwright?.resume();
           setFailureReason('');
           trackButtonClick('fail_step');

src/ui/components/TestControls.tsx

@@ -3,8 +3,9 @@ import React, { createContext, useContext, ReactNode, Dispatch } from 'react';
 // Types
 export type Step = {
   text: string;
-  status: 'pending' | 'pass' | 'fail';
+  status: 'pending' | 'pass' | 'fail' | 'warning';
   reason?: string;
+  isSoft?: boolean;
 };
 
 interface State {
@@ -20,7 +21,7 @@ const initialState: State = {
 // Actions
 export type Action =
   | { type: 'SET_TEST_NAME'; payload: string }
-  | { type: 'ADD_STEP'; payload: string }
+  | { type: 'ADD_STEP'; payload: { step: string; isSoft?: boolean } }
   | { type: 'PASS_STEP' }
   | { type: 'FAIL_STEP'; payload: string };
 
@@ -31,7 +32,7 @@ function reducer(state: State, action: Action): State {
     case 'ADD_STEP':
       return {
         ...state,
-        steps: [...state.steps, { text: action.payload, status: 'pending' }],
+        steps: [...state.steps, { text: action.payload.step, status: 'pending', isSoft: action.payload.isSoft }],
       };
     case 'PASS_STEP': {
       const steps = [...state.steps];
@@ -48,7 +49,7 @@ function reducer(state: State, action: Action): State {
       if (steps.length > 0) {
         steps[steps.length - 1] = {
           ...steps[steps.length - 1],
-          status: 'fail',
+          status: steps[steps.length - 1].isSoft ? 'warning' : 'fail',
           reason: action.payload,
         };
       }

src/ui/store/TestStore.tsx

@@ -42,6 +42,10 @@ button {
   font-weight: 600;
   transition: background-color 0.2s;
 }
+button:disabled {
+  opacity: 0.7;
+  cursor: progress;
+}
 .btn-success {
   background-color: #2ecc71;
   color: white;
@@ -78,4 +82,7 @@ input:focus {
 }
 .step-fail {
   color: #e74c3c;
+}
+.step-warning {
+  color: #f1c40f;
 }