diff --git a/packages/core/src/lib/implementation/execute-plugin.ts b/packages/core/src/lib/implementation/execute-plugin.ts index 8f0a07860..c84efd6ad 100644 --- a/packages/core/src/lib/implementation/execute-plugin.ts +++ b/packages/core/src/lib/implementation/execute-plugin.ts @@ -14,6 +14,7 @@ import { groupByStatus, logMultipleResults, pluralizeToken, + scoreAuditsWithTarget, } from '@code-pushup/utils'; import { executePluginRunner, @@ -57,6 +58,7 @@ export async function executePlugin( description, docsUrl, groups, + scoreTargets, ...pluginMeta } = pluginConfig; const { write: cacheWrite = false, read: cacheRead = false } = cache; @@ -76,8 +78,13 @@ export async function executePlugin( }); } + // transform audit scores to 1 when they meet/exceed their targets + const scoredAuditsWithTarget = scoreTargets + ? scoreAuditsWithTarget(audits, scoreTargets) + : audits; + // enrich `AuditOutputs` to `AuditReport` - const auditReports: AuditReport[] = audits.map( + const auditReports: AuditReport[] = scoredAuditsWithTarget.map( (auditOutput: AuditOutput) => ({ ...auditOutput, ...(pluginConfigAudits.find( diff --git a/packages/core/src/lib/implementation/execute-plugin.unit.test.ts b/packages/core/src/lib/implementation/execute-plugin.unit.test.ts index 8067f180c..e3c1ab929 100644 --- a/packages/core/src/lib/implementation/execute-plugin.unit.test.ts +++ b/packages/core/src/lib/implementation/execute-plugin.unit.test.ts @@ -124,6 +124,67 @@ describe('executePlugin', () => { MINIMAL_PLUGIN_CONFIG_MOCK, ); }); + + it('should apply a single score target to all audits', async () => { + const pluginConfig: PluginConfig = { + ...MINIMAL_PLUGIN_CONFIG_MOCK, + scoreTargets: 0.8, + audits: [ + { + slug: 'speed-index', + title: 'Speed Index', + }, + { + slug: 'total-blocking-time', + title: 'Total Blocking Time', + }, + ], + runner: () => [ + { slug: 'speed-index', score: 0.9, value: 1300 }, + { slug: 'total-blocking-time', score: 0.3, value: 600 }, + ], + }; + + const result = await executePlugin(pluginConfig, { + persist: { outputDir: '' }, + cache: { read: false, write: false }, + }); + + expect(result.audits).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + slug: 'speed-index', + score: 1, + scoreTarget: 0.8, + }), + expect.objectContaining({ + slug: 'total-blocking-time', + score: 0.3, + scoreTarget: 0.8, + }), + ]), + ); + }); + + it('should apply per-audit score targets', async () => { + const pluginConfig: PluginConfig = { + ...MINIMAL_PLUGIN_CONFIG_MOCK, // returns node-version audit with score 0.3 + scoreTargets: { + 'node-version': 0.2, + }, + }; + + const result = await executePlugin(pluginConfig, { + persist: { outputDir: '' }, + cache: { read: false, write: false }, + }); + + expect(result.audits[0]).toMatchObject({ + slug: 'node-version', + score: 1, + scoreTarget: 0.2, + }); + }); }); describe('executePlugins', () => { diff --git a/packages/models/docs/models-reference.md b/packages/models/docs/models-reference.md index cad1b40d4..f11d358f8 100644 --- a/packages/models/docs/models-reference.md +++ b/packages/models/docs/models-reference.md @@ -49,6 +49,7 @@ _Object containing the following properties:_ | `displayValue` | Formatted value (e.g. '0.9 s', '2.1 MB') | `string` | | **`value`** (\*) | Raw numeric value | `number` (_≥0_) | | **`score`** (\*) | Value between 0 and 1 | `number` (_≥0, ≤1_) | +| `scoreTarget` | Pass/fail score threshold (0-1) | `number` (_≥0, ≤1_) | | `details` | Detailed information | [AuditDetails](#auditdetails) | _(\*) Required._ @@ -73,6 +74,7 @@ _Object containing the following properties:_ | `displayValue` | Formatted value (e.g. '0.9 s', '2.1 MB') | `string` | | **`value`** (\*) | Raw numeric value | `number` (_≥0_) | | **`score`** (\*) | Value between 0 and 1 | `number` (_≥0, ≤1_) | +| `scoreTarget` | Pass/fail score threshold (0-1) | `number` (_≥0, ≤1_) | | `details` | Detailed information | [AuditDetails](#auditdetails) | _(\*) Required._ @@ -1282,20 +1284,21 @@ _(\*) Required._ _Object containing the following properties:_ -| Property | Description | Type | -| :---------------- | :---------------------------------------- | :------------------------------------------------------------------- | -| `packageName` | NPM package name | `string` | -| `version` | NPM version of the package | `string` | -| **`title`** (\*) | Descriptive name | `string` (_max length: 256_) | -| `description` | Description (markdown) | `string` (_max length: 65536_) | -| `docsUrl` | Plugin documentation site | `string` (_url_) (_optional_) _or_ `''` | -| `isSkipped` | | `boolean` | -| **`slug`** (\*) | Unique plugin slug within core config | `string` (_regex: `/^[a-z\d]+(?:-[a-z\d]+)*$/`, max length: 128_) | -| **`icon`** (\*) | Icon from VSCode Material Icons extension | [MaterialIcon](#materialicon) | -| **`runner`** (\*) | | [RunnerConfig](#runnerconfig) _or_ [RunnerFunction](#runnerfunction) | -| **`audits`** (\*) | List of audits maintained in a plugin | _Array of at least 1 [Audit](#audit) items_ | -| `groups` | List of groups | _Array of [Group](#group) items_ | -| `context` | Plugin-specific context data for helpers | [PluginContext](#plugincontext) | +| Property | Description | Type | +| :---------------- | :--------------------------------------------------------------------------------------------------------------- | :------------------------------------------------------------------------------------------------------------------------- | +| `packageName` | NPM package name | `string` | +| `version` | NPM version of the package | `string` | +| **`title`** (\*) | Descriptive name | `string` (_max length: 256_) | +| `description` | Description (markdown) | `string` (_max length: 65536_) | +| `docsUrl` | Plugin documentation site | `string` (_url_) (_optional_) _or_ `''` | +| `isSkipped` | | `boolean` | +| **`slug`** (\*) | Unique plugin slug within core config | `string` (_regex: `/^[a-z\d]+(?:-[a-z\d]+)*$/`, max length: 128_) | +| **`icon`** (\*) | Icon from VSCode Material Icons extension | [MaterialIcon](#materialicon) | +| **`runner`** (\*) | | [RunnerConfig](#runnerconfig) _or_ [RunnerFunction](#runnerfunction) | +| **`audits`** (\*) | List of audits maintained in a plugin | _Array of at least 1 [Audit](#audit) items_ | +| `groups` | List of groups | _Array of [Group](#group) items_ | +| `scoreTargets` | Score targets that trigger a perfect score. Number for all audits or record { slug: target } for specific audits | `number` (_≥0, ≤1_) (_optional_) _or_ _Object with dynamic keys of type_ `string` _and values of type_ `number` (_≥0, ≤1_) | +| `context` | Plugin-specific context data for helpers | [PluginContext](#plugincontext) | _(\*) Required._ diff --git a/packages/models/src/index.ts b/packages/models/src/index.ts index a305ea673..0b2510395 100644 --- a/packages/models/src/index.ts +++ b/packages/models/src/index.ts @@ -80,6 +80,7 @@ export { type PluginConfig, type PluginContext, type PluginMeta, + type PluginScoreTargets, } from './lib/plugin-config.js'; export { auditReportSchema, diff --git a/packages/models/src/lib/audit-output.ts b/packages/models/src/lib/audit-output.ts index a5b431f15..878451acb 100644 --- a/packages/models/src/lib/audit-output.ts +++ b/packages/models/src/lib/audit-output.ts @@ -3,6 +3,7 @@ import { createDuplicateSlugsCheck } from './implementation/checks.js'; import { nonnegativeNumberSchema, scoreSchema, + scoreTargetSchema, slugSchema, } from './implementation/schemas.js'; import { issueSchema } from './issue.js'; @@ -34,6 +35,7 @@ export const auditOutputSchema = z displayValue: auditDisplayValueSchema, value: auditValueSchema, score: scoreSchema, + scoreTarget: scoreTargetSchema, details: auditDetailsSchema.optional(), }) .describe('Audit information'); diff --git a/packages/models/src/lib/audit-output.unit.test.ts b/packages/models/src/lib/audit-output.unit.test.ts index 7a9515ded..09baf066a 100644 --- a/packages/models/src/lib/audit-output.unit.test.ts +++ b/packages/models/src/lib/audit-output.unit.test.ts @@ -87,6 +87,18 @@ describe('auditOutputSchema', () => { ).not.toThrow(); }); + it('should accept a valid audit output with a score target', () => { + expect(() => + auditOutputSchema.parse({ + slug: 'total-blocking-time', + score: 0.91, + scoreTarget: 0.9, + value: 183.5, + displayValue: '180 ms', + } satisfies AuditOutput), + ).not.toThrow(); + }); + it('should accept a decimal value', () => { expect(() => auditOutputSchema.parse({ diff --git a/packages/models/src/lib/audit.ts b/packages/models/src/lib/audit.ts index 695ad5ef9..92ca66c9a 100644 --- a/packages/models/src/lib/audit.ts +++ b/packages/models/src/lib/audit.ts @@ -6,14 +6,14 @@ export const auditSchema = z .object({ slug: slugSchema.describe('ID (unique within plugin)'), }) - .merge( + .extend( metaSchema({ titleDescription: 'Descriptive name', descriptionDescription: 'Description (markdown)', docsUrlDescription: 'Link to documentation (rationale)', description: 'List of scorable metrics for the given plugin', isSkippedDescription: 'Indicates whether the audit is skipped', - }), + }).shape, ); export type Audit = z.infer; diff --git a/packages/models/src/lib/category-config.ts b/packages/models/src/lib/category-config.ts index f6a14e432..ff618b19b 100644 --- a/packages/models/src/lib/category-config.ts +++ b/packages/models/src/lib/category-config.ts @@ -5,8 +5,8 @@ import { } from './implementation/checks.js'; import { metaSchema, - nonnegativeNumberSchema, scorableSchema, + scoreTargetSchema, slugSchema, weightedRefSchema, } from './implementation/schemas.js'; @@ -44,12 +44,7 @@ export const categoryConfigSchema = scorableSchema( description: 'Meta info for category', }).shape, ) - .extend({ - scoreTarget: nonnegativeNumberSchema - .max(1) - .describe('Pass/fail score threshold (0-1)') - .optional(), - }); + .extend({ scoreTarget: scoreTargetSchema }); export type CategoryConfig = z.infer; diff --git a/packages/models/src/lib/category-config.unit.test.ts b/packages/models/src/lib/category-config.unit.test.ts index 7e7ffd091..a24b40de6 100644 --- a/packages/models/src/lib/category-config.unit.test.ts +++ b/packages/models/src/lib/category-config.unit.test.ts @@ -121,6 +121,30 @@ describe('categoryConfigSchema', () => { ).not.toThrow(); }); + it('should accept a valid category configuration with a score target', () => { + expect(() => + categoryConfigSchema.parse({ + slug: 'core-web-vitals', + title: 'Core Web Vitals', + scoreTarget: 0.9, + refs: [ + { + plugin: 'lighthouse', + slug: 'largest-contentful-paint', + type: 'audit', + weight: 3, + }, + { + plugin: 'lighthouse', + slug: 'first-input-delay', + type: 'audit', + weight: 2, + }, + ], + } satisfies CategoryConfig), + ).not.toThrow(); + }); + it('should throw for an empty category', () => { expect(() => categoryConfigSchema.parse({ diff --git a/packages/models/src/lib/core-config.unit.test.ts b/packages/models/src/lib/core-config.unit.test.ts index ddc0e0072..8561ae575 100644 --- a/packages/models/src/lib/core-config.unit.test.ts +++ b/packages/models/src/lib/core-config.unit.test.ts @@ -170,7 +170,13 @@ describe('coreConfigSchema', () => { slug: 'lighthouse', title: 'Lighthouse', icon: 'lighthouse', - runner: { command: 'npm run lint', outputFile: 'output.json' }, + runner: async () => [ + { + slug: 'csp-xss', + score: 1, + value: 1, + }, + ], audits: [ { slug: 'csp-xss', diff --git a/packages/models/src/lib/implementation/schemas.ts b/packages/models/src/lib/implementation/schemas.ts index 720dec5c4..735952511 100644 --- a/packages/models/src/lib/implementation/schemas.ts +++ b/packages/models/src/lib/implementation/schemas.ts @@ -172,6 +172,12 @@ export function packageVersionSchema< }>; } +/** Schema for a binary score threshold */ +export const scoreTargetSchema = nonnegativeNumberSchema + .max(1) + .describe('Pass/fail score threshold (0-1)') + .optional(); + /** Schema for a weight */ export const weightSchema = nonnegativeNumberSchema.describe( 'Coefficient for the given score (use weight 0 if only for display)', diff --git a/packages/models/src/lib/plugin-config.ts b/packages/models/src/lib/plugin-config.ts index 3697ed4cf..865a6c7d4 100644 --- a/packages/models/src/lib/plugin-config.ts +++ b/packages/models/src/lib/plugin-config.ts @@ -6,6 +6,7 @@ import { materialIconSchema, metaSchema, packageVersionSchema, + scoreTargetSchema, slugSchema, } from './implementation/schemas.js'; import { formatSlugsList, hasMissingStrings } from './implementation/utils.js'; @@ -18,31 +19,42 @@ export const pluginContextSchema = z export type PluginContext = z.infer; export const pluginMetaSchema = packageVersionSchema() - .merge( + .extend( metaSchema({ titleDescription: 'Descriptive name', descriptionDescription: 'Description (markdown)', docsUrlDescription: 'Plugin documentation site', description: 'Plugin metadata', - }), + }).shape, ) - .merge( - z.object({ - slug: slugSchema.describe('Unique plugin slug within core config'), - icon: materialIconSchema, - }), - ); + .extend({ + slug: slugSchema.describe('Unique plugin slug within core config'), + icon: materialIconSchema, + }); export type PluginMeta = z.infer; +export const pluginScoreTargetsSchema = z + .union([ + scoreTargetSchema, + z.record(z.string(), scoreTargetSchema.nonoptional()), + ]) + .describe( + 'Score targets that trigger a perfect score. Number for all audits or record { slug: target } for specific audits', + ) + .optional(); + +export type PluginScoreTargets = z.infer; + export const pluginDataSchema = z.object({ runner: z.union([runnerConfigSchema, runnerFunctionSchema]), audits: pluginAuditsSchema, groups: groupsSchema, + scoreTargets: pluginScoreTargetsSchema, context: pluginContextSchema, }); export const pluginConfigSchema = pluginMetaSchema - .merge(pluginDataSchema) + .extend(pluginDataSchema.shape) .check(createCheck(findMissingSlugsInGroupRefs)); export type PluginConfig = z.infer; diff --git a/packages/models/src/lib/plugin-config.unit.test.ts b/packages/models/src/lib/plugin-config.unit.test.ts index f7409c92a..69ccaa9ff 100644 --- a/packages/models/src/lib/plugin-config.unit.test.ts +++ b/packages/models/src/lib/plugin-config.unit.test.ts @@ -40,6 +40,35 @@ describe('pluginConfigSchema', () => { ).not.toThrow(); }); + it('should accept a valid plugin configuration with a score target', () => { + expect(() => + pluginConfigSchema.parse({ + slug: 'lighthouse', + title: 'Lighthouse', + icon: 'lighthouse', + runner: async () => [ + { + slug: 'first-contentful-paint', + score: 0.28, + value: 3752, + displayValue: '3.8 s', + }, + { + slug: 'total-blocking-time', + score: 0.91, + value: 183.5, + displayValue: '180 ms', + }, + ], + scoreTarget: { 'total-blocking-time': 0.9 }, + audits: [ + { slug: 'first-contentful-paint', title: 'First Contentful Paint' }, + { slug: 'total-blocking-time', title: 'Total Blocking Time' }, + ], + } satisfies PluginConfig), + ).not.toThrow(); + }); + it('should throw for a plugin configuration without audits', () => { expect(() => pluginConfigSchema.parse({ diff --git a/packages/utils/src/index.ts b/packages/utils/src/index.ts index 90d8a58eb..2ae826a06 100644 --- a/packages/utils/src/index.ts +++ b/packages/utils/src/index.ts @@ -102,7 +102,7 @@ export { } from './lib/reports/generate-md-reports-diff.js'; export { loadReport } from './lib/reports/load-report.js'; export { logStdoutSummary } from './lib/reports/log-stdout-summary.js'; -export { scoreReport } from './lib/reports/scoring.js'; +export { scoreReport, scoreAuditsWithTarget } from './lib/reports/scoring.js'; export { sortReport } from './lib/reports/sorting.js'; export type { ScoredCategoryConfig, diff --git a/packages/utils/src/lib/reports/scoring.ts b/packages/utils/src/lib/reports/scoring.ts index 7f22fd92b..98697dc3d 100644 --- a/packages/utils/src/lib/reports/scoring.ts +++ b/packages/utils/src/lib/reports/scoring.ts @@ -1,7 +1,9 @@ import type { + AuditOutput, AuditReport, CategoryRef, GroupRef, + PluginScoreTargets, Report, } from '@code-pushup/models'; import { deepClone } from '../transform.js'; @@ -117,3 +119,37 @@ function parseScoringParameters( return scoredRefs; } + +/** + * Sets audit score to 1 if it meets target. + * @param audit audit output to evaluate + * @param scoreTarget threshold for perfect score (0-1) + * @returns Audit with scoreTarget field + */ +export function scoreAuditWithTarget( + audit: AuditOutput, + scoreTarget: number, +): AuditOutput { + return audit.score >= scoreTarget + ? { ...audit, score: 1, scoreTarget } + : { ...audit, scoreTarget }; +} + +/** + * Sets audit scores to 1 when targets are met. + * @param audits audit outputs from plugin execution + * @param scoreTargets number or { slug: target } mapping + * @returns Transformed audits with scoreTarget field + */ +export function scoreAuditsWithTarget( + audits: AuditOutput[], + scoreTargets: PluginScoreTargets, +): AuditOutput[] { + if (typeof scoreTargets === 'number') { + return audits.map(audit => scoreAuditWithTarget(audit, scoreTargets)); + } + return audits.map(audit => { + const target = scoreTargets?.[audit.slug]; + return target == null ? audit : scoreAuditWithTarget(audit, target); + }); +} diff --git a/packages/utils/src/lib/reports/scoring.unit.test.ts b/packages/utils/src/lib/reports/scoring.unit.test.ts index e273eee8a..fe0862f31 100644 --- a/packages/utils/src/lib/reports/scoring.unit.test.ts +++ b/packages/utils/src/lib/reports/scoring.unit.test.ts @@ -1,6 +1,11 @@ import { describe, expect } from 'vitest'; import { REPORT_MOCK } from '@code-pushup/test-utils'; -import { calculateScore, scoreReport } from './scoring.js'; +import { + calculateScore, + scoreAuditWithTarget, + scoreAuditsWithTarget, + scoreReport, +} from './scoring.js'; describe('calculateScore', () => { it('should calculate the same score for one reference', () => { @@ -136,3 +141,108 @@ describe('scoreReport', () => { ); }); }); + +describe('scoreAuditWithTarget', () => { + it('should add scoreTarget and increase an audit score to 1 when the target is reached', () => { + expect( + scoreAuditWithTarget( + { slug: 'speed-index', score: 0.9, value: 1300 }, + 0.8, + ), + ).toEqual({ + slug: 'speed-index', + score: 1, + value: 1300, + scoreTarget: 0.8, + }); + }); + + it('should only add scoreTarget when the target is not reached', () => { + expect( + scoreAuditWithTarget( + { slug: 'largest-contentful-paint', score: 0.6, value: 3000 }, + 0.8, + ), + ).toEqual({ + slug: 'largest-contentful-paint', + score: 0.6, + value: 3000, + scoreTarget: 0.8, + }); + }); +}); + +describe('scoreAuditsWithTarget', () => { + it('should apply a single score target to all audits', () => { + const audits = [ + { slug: 'first-contentful-paint', score: 0.8, value: 1200 }, + { slug: 'largest-contentful-paint', score: 0.6, value: 3000 }, + { slug: 'speed-index', score: 0.9, value: 1300 }, + ]; + + expect(scoreAuditsWithTarget(audits, 0.75)).toEqual([ + { + slug: 'first-contentful-paint', + score: 1, + value: 1200, + scoreTarget: 0.75, + }, + { + slug: 'largest-contentful-paint', + score: 0.6, + value: 3000, + scoreTarget: 0.75, + }, + { slug: 'speed-index', score: 1, value: 1300, scoreTarget: 0.75 }, + ]); + }); + + it('should apply per-audit score targets', () => { + const audits = [ + { slug: 'first-contentful-paint', score: 0.8, value: 1200 }, + { slug: 'largest-contentful-paint', score: 0.6, value: 3000 }, + { slug: 'speed-index', score: 0.9, value: 1300 }, + ]; + + expect( + scoreAuditsWithTarget(audits, { + 'first-contentful-paint': 0.85, + 'largest-contentful-paint': 0.5, + }), + ).toEqual([ + { + slug: 'first-contentful-paint', + score: 0.8, + value: 1200, + scoreTarget: 0.85, + }, + { + slug: 'largest-contentful-paint', + score: 1, + value: 3000, + scoreTarget: 0.5, + }, + { slug: 'speed-index', score: 0.9, value: 1300 }, + ]); + }); + + it('should set an audit score to 1 when the original score equals the target', () => { + const audits = [{ slug: 'speed-index', score: 0.9, value: 1300 }]; + + expect(scoreAuditsWithTarget(audits, 0.9)).toEqual([ + { slug: 'speed-index', score: 1, value: 1300, scoreTarget: 0.9 }, + ]); + }); + + it('should handle an empty audits array', () => { + expect(scoreAuditsWithTarget([], 0.8)).toEqual([]); + }); + + it('should handle an empty score target record', () => { + const audits = [{ slug: 'speed-index', score: 0.9, value: 1300 }]; + + expect(scoreAuditsWithTarget(audits, {})).toEqual([ + { slug: 'speed-index', score: 0.9, value: 1300 }, + ]); + }); +});