From 9cab1e7c200838505813f2b0dd8806a10efc32f5 Mon Sep 17 00:00:00 2001 From: martinboulais <31805063+martinboulais@users.noreply.github.com> Date: Tue, 4 Mar 2025 11:44:52 +0100 Subject: [PATCH 01/18] Extract dedicated services for GAQ and QC summaries --- lib/server/controllers/qcFlag.controller.js | 8 +- .../services/qualityControlFlag/GaqService.js | 113 +++++ .../qualityControlFlag/QcFlagService.js | 408 ++++-------------- .../QcFlagSummaryService.js | 198 +++++++++ lib/usecases/run/GetAllRunsUseCase.js | 4 +- .../qualityControlFlag/QcFlagService.test.js | 28 +- 6 files changed, 413 insertions(+), 346 deletions(-) create mode 100644 lib/server/services/qualityControlFlag/GaqService.js create mode 100644 lib/server/services/qualityControlFlag/QcFlagSummaryService.js diff --git a/lib/server/controllers/qcFlag.controller.js b/lib/server/controllers/qcFlag.controller.js index 6938d9092e..5ec00816f5 100644 --- a/lib/server/controllers/qcFlag.controller.js +++ b/lib/server/controllers/qcFlag.controller.js @@ -19,6 +19,8 @@ const { PaginationDto } = require('../../domain/dtos'); const { ApiConfig } = require('../../config'); const { countedItemsToHttpView } = require('../utilities/countedItemsToHttpView'); const { qcFlagService } = require('../services/qualityControlFlag/QcFlagService.js'); +const { gaqService } = require('../services/qualityControlFlag/GaqService.js'); +const { qcFlagSummaryService } = require('../services/qualityControlFlag/QcFlagSummaryService.js'); // eslint-disable-next-line valid-jsdoc /** @@ -324,7 +326,7 @@ const getQcFlagsSummaryHandler = async (req, res) => { mcReproducibleAsNotBad = false, } = validatedDTO.query; - const data = await qcFlagService.getQcFlagsSummary({ dataPassId, simulationPassId, lhcPeriodId }, { mcReproducibleAsNotBad }); + const data = await qcFlagSummaryService.getQcFlagsSummary({ dataPassId, simulationPassId, lhcPeriodId }, { mcReproducibleAsNotBad }); res.json({ data }); } catch (error) { updateExpressResponseFromNativeError(res, error); @@ -349,7 +351,7 @@ const getGaqQcFlagsHandler = async (request, response) => { try { const { dataPassId, runNumber } = validatedDTO.query; - const data = await qcFlagService.getGaqFlags(dataPassId, runNumber); + const data = await gaqService.getFlagsForDataPassAndRun(dataPassId, runNumber); response.json({ data }); } catch (error) { updateExpressResponseFromNativeError(response, error); @@ -374,7 +376,7 @@ const getGaqSummaryHandler = async (request, response) => { try { const { dataPassId, mcReproducibleAsNotBad = false } = validatedDTO.query; - const data = await qcFlagService.getGaqSummary(dataPassId, { mcReproducibleAsNotBad }); + const data = await gaqService.getSummary(dataPassId, { mcReproducibleAsNotBad }); response.json({ data }); } catch (error) { updateExpressResponseFromNativeError(response, error); diff --git a/lib/server/services/qualityControlFlag/GaqService.js b/lib/server/services/qualityControlFlag/GaqService.js new file mode 100644 index 0000000000..00bf00657b --- /dev/null +++ b/lib/server/services/qualityControlFlag/GaqService.js @@ -0,0 +1,113 @@ +/** + * @license + * Copyright CERN and copyright holders of ALICE O2. This software is + * distributed under the terms of the GNU General Public License v3 (GPL + * Version 3), copied verbatim in the file "COPYING". + * + * See http://alice-o2.web.cern.ch/license for full licensing information. + * + * In applying this license CERN does not waive the privileges and immunities + * granted to it by virtue of its status as an Intergovernmental Organization + * or submit itself to any jurisdiction. + */ +import { getOneDataPassOrFail } from '../dataPasses/getOneDataPassOrFail.js'; +import { Op } from 'sequelize'; +import { qcFlagAdapter } from '../../../database/adapters/index.js'; +import { QcFlagRepository } from '../../../database/repositories/index.js'; +import { QcFlagSummaryService } from './QcFlagSummaryService.js'; + +const QC_SUMMARY_PROPERTIES = { + badEffectiveRunCoverage: 'badEffectiveRunCoverage', + explicitlyNotBadEffectiveRunCoverage: 'explicitlyNotBadEffectiveRunCoverage', + missingVerificationsCount: 'missingVerificationsCount', + mcReproducible: 'mcReproducible', +}; + +/** + * Globally aggregated quality (QC flags aggregated for a predefined list of detectors per runs) service + */ +export class GaqService { + /** + * Get GAQ summary + * + * @param {number} dataPassId id of data pass id + * @param {object} [options] additional options + * @param {boolean} [options.mcReproducibleAsNotBad = false] if set to true, + * `Limited Acceptance MC Reproducible` flag type is treated as good one + * @return {Promise} Resolves with the GAQ Summary + */ + async getSummary(dataPassId, { mcReproducibleAsNotBad = false } = {}) { + await getOneDataPassOrFail({ id: dataPassId }); + const runGaqSubSummaries = await QcFlagRepository.getRunGaqSubSummaries(dataPassId, { mcReproducibleAsNotBad }); + + const summary = {}; + const flagsAndVerifications = {}; + + // Fold list of subSummaries into one summary + for (const subSummary of runGaqSubSummaries) { + const { + runNumber, + flagsIds, + verifiedFlagsIds, + } = subSummary; + + if (!summary[runNumber]) { + summary[runNumber] = { [QC_SUMMARY_PROPERTIES.mcReproducible]: false }; + } + if (!flagsAndVerifications[runNumber]) { + flagsAndVerifications[runNumber] = {}; + } + + const runSummary = summary[runNumber]; + + const distinctRunFlagsIds = flagsAndVerifications[runNumber]?.distinctFlagsIds ?? []; + const distinctRunVerifiedFlagsIds = flagsAndVerifications[runNumber]?.distinctVerifiedFlagsIds ?? []; + + flagsAndVerifications[runNumber] = { + distinctFlagsIds: new Set([...distinctRunFlagsIds, ...flagsIds]), + distinctVerifiedFlagsIds: new Set([...distinctRunVerifiedFlagsIds, ...verifiedFlagsIds]), + }; + + QcFlagSummaryService.mergeIntoSummaryUnit(runSummary, subSummary); + } + + for (const [runNumber, { distinctFlagsIds, distinctVerifiedFlagsIds }] of Object.entries(flagsAndVerifications)) { + summary[runNumber][QC_SUMMARY_PROPERTIES.missingVerificationsCount] = distinctFlagsIds.size - distinctVerifiedFlagsIds.size; + } + + return summary; + } + + /** + * Find QC flags in GAQ effective periods for given data pass and run + * + * @param {number} dataPassId id od data pass + * @param {number} runNumber run number + * @return {Promise} promise of aggregated QC flags + */ + async getFlagsForDataPassAndRun(dataPassId, runNumber) { + const gaqPeriods = await QcFlagRepository.findGaqPeriods(dataPassId, runNumber); + const qcFlags = (await QcFlagRepository.findAll({ + where: { id: { [Op.in]: gaqPeriods.flatMap(({ contributingFlagIds }) => contributingFlagIds) } }, + include: [ + { association: 'flagType' }, + { association: 'createdBy' }, + { association: 'verifications', include: [{ association: 'createdBy' }] }, + ], + })).map(qcFlagAdapter.toEntity); + + const idToFlag = Object.fromEntries(qcFlags.map((flag) => [flag.id, flag])); + + return gaqPeriods.map(({ + contributingFlagIds, + from, + to, + }) => ({ + from, + to, + contributingFlags: contributingFlagIds.map((id) => idToFlag[id]), + })); + } +} + +export const gaqService = new GaqService(); diff --git a/lib/server/services/qualityControlFlag/QcFlagService.js b/lib/server/services/qualityControlFlag/QcFlagService.js index 99cd93c86c..d562dc47ef 100644 --- a/lib/server/services/qualityControlFlag/QcFlagService.js +++ b/lib/server/services/qualityControlFlag/QcFlagService.js @@ -63,20 +63,6 @@ const validateUserDetectorAccess = (userRoles, detectorName) => { } }; -/** - * @typedef RunDetectorQcSummary - * @property {number} badEffectiveRunCoverage - fraction of run's data, marked explicitly with bad QC flag - * @property {number} explicitlyNotBadEffectiveRunCoverage - fraction of run's data, marked explicitly with good QC flag - * @property {number} missingVerificationsCount - number of not verified QC flags which are not discarded - * @property {boolean} mcReproducible - states whether some Limited Acceptance MC Reproducible flag was assigned - */ -const QC_SUMMARY_PROPERTIES = { - badEffectiveRunCoverage: 'badEffectiveRunCoverage', - explicitlyNotBadEffectiveRunCoverage: 'explicitlyNotBadEffectiveRunCoverage', - missingVerificationsCount: 'missingVerificationsCount', - mcReproducible: 'mcReproducible', -}; - /** * @typedef RunQcSummary * @type {Object} dplDetectorID to RunDetectorQcSummary mappings @@ -103,6 +89,14 @@ const QC_SUMMARY_PROPERTIES = { * @property {boolean} mcReproducible - states whether some aggregation of QC flags is Limited Acceptance MC Reproducible */ +/** + * @typedef RunDetectorQcSummary + * @property {number} badEffectiveRunCoverage - fraction of run's data, marked explicitly with bad QC flag + * @property {number} explicitlyNotBadEffectiveRunCoverage - fraction of run's data, marked explicitly with good QC flag + * @property {number} missingVerificationsCount - number of not verified QC flags which are not discarded + * @property {boolean} mcReproducible - states whether some Limited Acceptance MC Reproducible flag was assigned + */ + /** * @typedef GaqSummary aggregated global quality summaries for given data pass * @type {Object} runNumber to RunGaqSummary mapping @@ -120,7 +114,7 @@ class QcFlagService { } /** - * Find an Quality Control Flag by its id + * Find a Quality Control Flag by its id * @param {number} id identifier of Quality Control Flag * @return {QcFlag} a Quality Control Flag */ @@ -137,7 +131,7 @@ class QcFlagService { } /** - * Find an Quality Control Flag by its id + * Find a Quality Control Flag by its id * @param {number} id id of Quality Control Flag * @throws {NotFoundError} in case there is no Quality Control Flag with given id * @return {Promise} a Quality Control Flag @@ -150,204 +144,6 @@ class QcFlagService { return qcFlag; } - /** - * Validate QC flag timestamps - * If null timestamp was provided, given timestamp is replaced by run's startTime or endTime - * @param {Partial} timestamps QC flag timestamps - * @param {Run} targetRun run which for QC flag is to be set - * @return {{from: (number|null), to: (number|null)}} prepared timestamps - * @throws {BadParameterError} - */ - _prepareQcFlagPeriod(timestamps, targetRun) { - const { timeTrgStart, timeO2Start, firstTfTimestamp, timeTrgEnd, timeO2End, lastTfTimestamp } = targetRun; - - let lowerBound = timeTrgStart?.getTime() ?? timeO2Start?.getTime() ?? null; - if (firstTfTimestamp) { - lowerBound = lowerBound ? Math.min(firstTfTimestamp.getTime(), lowerBound) : firstTfTimestamp.getTime(); - } - - let upperBound = timeTrgEnd?.getTime() ?? timeO2End?.getTime() ?? null; - if (lastTfTimestamp) { - upperBound = upperBound ? Math.max(lastTfTimestamp.getTime(), upperBound) : lastTfTimestamp.getTime(); - } - - const from = timestamps.from ?? lowerBound ?? null; - const to = timestamps.to ?? upperBound ?? null; - - if (from && to && from >= to) { - throw new BadParameterError('Parameter "to" timestamp must be greater than "from" timestamp'); - } - const isFromOutOfRange = from && (lowerBound && from < lowerBound || upperBound && upperBound <= from); - const isToOutOfRange = to && (lowerBound && to <= lowerBound || upperBound && upperBound < to); - if (isFromOutOfRange || isToOutOfRange) { - throw new BadParameterError(`Given QC flag period (${from}, ${to}) is out of run (${lowerBound}, ${upperBound}) period`); - } - - return { from, to }; - } - - /** - * Remove a time segment from a list of QC flags effective periods - * - * @param {Period} intersectingPeriod time segment that should be removed from given periods - * @param {SequelizeQcFlagEffectivePeriod} periods periods to be updated or discarded - * @return {Promise} resolve once all periods are updated - */ - async _removeEffectivePeriodsAndPeriodIntersection({ from: newerPeriodFrom, to: newerPeriodTo }, periods) { - for (const effectivePeriod of periods) { - const { id: effectivePeriodId, from: effectiveFrom, to: effectiveTo } = effectivePeriod; - - const effectiveToIsLesserOrEqNewerPeriodTo = newerPeriodTo === null || effectiveTo !== null && effectiveTo <= newerPeriodTo; - - if (newerPeriodFrom <= effectiveFrom - && effectiveToIsLesserOrEqNewerPeriodTo) { // Old flag is fully covered by new one - await QcFlagEffectivePeriodRepository.removeOne({ where: { id: effectivePeriodId } }); - } else if (effectiveFrom < newerPeriodFrom && !effectiveToIsLesserOrEqNewerPeriodTo) { - // New flag's period is included in the old one's period. - await QcFlagEffectivePeriodRepository.update(effectivePeriod, { to: newerPeriodFrom }); - await QcFlagEffectivePeriodRepository.insert({ - flagId: effectivePeriod.flagId, - from: newerPeriodTo, - to: effectiveTo, - }); - } else if (effectiveFrom < newerPeriodFrom) { - await QcFlagEffectivePeriodRepository.update(effectivePeriod, { to: newerPeriodFrom }); - } else if (!effectiveToIsLesserOrEqNewerPeriodTo) { - await QcFlagEffectivePeriodRepository.update(effectivePeriod, { from: newerPeriodTo }); - } else { - throw new Error('Incorrect state'); - } - } - } - - /** - * Get QC summary for given data/simulation pass or synchronous QC flags for given LHC period - * - * @param {scope} scope of the QC flag - * @param {number} [scope.dataPassId] data pass id - exclusive with other options - * @param {number} [scope.simulationPassId] simulation pass id - exclusive with other options - * @param {number} [scope.lhcPeriodId] id of LHC Period - exclusive with other options - * @param {object} [options] additional options - * @param {boolean} [options.mcReproducibleAsNotBad = false] if set to true, `Limited Acceptance MC Reproducible` flag type is treated as - * good one - * @return {Promise} summary - */ - async getQcFlagsSummary({ dataPassId, simulationPassId, lhcPeriodId }, { mcReproducibleAsNotBad = false } = {}) { - if (Boolean(dataPassId) + Boolean(simulationPassId) + Boolean(lhcPeriodId) > 1) { - throw new BadParameterError('`dataPassId`, `simulationPassId` and `lhcPeriodId` are exclusive options'); - } - - const queryBuilder = dataSource.createQueryBuilder() - .where('deleted').is(false); - - if (dataPassId) { - queryBuilder - .whereAssociation('dataPasses', 'id').is(dataPassId) - .include({ association: 'run', attributes: [] }); - } else if (simulationPassId) { - queryBuilder - .whereAssociation('simulationPasses', 'id').is(simulationPassId) - .include({ association: 'run', attributes: [] }); - } else { - queryBuilder.include({ association: 'dataPasses', required: false }) - .include({ association: 'simulationPasses', required: false }) - .where('$dataPasses.id$').is(null) - .where('$simulationPasses.id$').is(null) - .include({ association: 'run', attributes: [], where: { lhcPeriodId } }); - } - - queryBuilder - .include({ association: 'effectivePeriods', attributes: [], required: true }) - .include({ association: 'flagType', attributes: [] }) - .set('attributes', (sequelize) => [ - 'runNumber', - 'detectorId', - [sequelize.literal(`IF(\`flagType\`.monte_carlo_reproducible AND ${mcReproducibleAsNotBad}, false, \`flagType\`.bad)`), 'bad'], - - [ - sequelize.literal(` - IF( - run.time_start IS NULL OR run.time_end IS NULL, - IF( - effectivePeriods.\`from\` IS NULL AND effectivePeriods.\`to\` IS NULL, - 1, - null - ), - SUM( - UNIX_TIMESTAMP(COALESCE(effectivePeriods.\`to\`, run.time_end)) - - UNIX_TIMESTAMP(COALESCE(effectivePeriods.\`from\`, run.time_start)) - ) / ( - UNIX_TIMESTAMP(run.time_end) - UNIX_TIMESTAMP(run.time_start) - ) - ) - `), - 'effectiveRunCoverage', - ], - [ - sequelize.literal('GROUP_CONCAT( DISTINCT `QcFlag`.id )'), - 'flagIds', - ], - [ - sequelize.literal('SUM( `flagType`.monte_carlo_reproducible ) > 0'), - 'mcReproducible', - ], - ]) - .groupBy('runNumber') - .groupBy('detectorId') - .groupBy((sequelize) => sequelize.literal(` - IF(\`flagType\`.monte_carlo_reproducible AND ${mcReproducibleAsNotBad}, false, \`flagType\`.bad) - `)); - - const runDetectorSummaryList = (await QcFlagRepository.findAll(queryBuilder)) - .map((summaryDb) => - ({ - runNumber: summaryDb.runNumber, - detectorId: summaryDb.detectorId, - effectiveRunCoverage: parseFloat(summaryDb.get('effectiveRunCoverage'), 10) || null, - bad: Boolean(summaryDb.get('bad')), - flagIds: (summaryDb.get('flagIds')?.split(',') ?? []).map((id) => parseInt(id, 10)), - mcReproducible: Boolean(summaryDb.get('mcReproducible')), - })); - - const allFlagsIds = new Set(runDetectorSummaryList.flatMap(({ flagIds }) => flagIds)); - const notVerifiedFlagsIds = new Set((await QcFlagRepository.findAll({ - attributes: ['id'], - include: [{ association: 'verifications', required: false, attributes: [] }], - where: { - id: { [Op.in]: [...allFlagsIds] }, - '$verifications.id$': { [Op.is]: null }, - }, - })).map(({ id }) => id)); - - const summary = {}; - - // Fold list of summaries into nested object - for (const runDetectorSummaryForFlagTypesClass of runDetectorSummaryList) { - const { - runNumber, - detectorId, - flagIds, - } = runDetectorSummaryForFlagTypesClass; - const missingVerificationsCount = flagIds.filter((id) => notVerifiedFlagsIds.has(id)).length; - - if (!summary[runNumber]) { - summary[runNumber] = {}; - } - if (!summary[runNumber][detectorId]) { - summary[runNumber][detectorId] = { [QC_SUMMARY_PROPERTIES.mcReproducible]: false }; - } - - const runDetectorSummary = summary[runNumber][detectorId]; - - runDetectorSummary[QC_SUMMARY_PROPERTIES.missingVerificationsCount] = - (runDetectorSummary[QC_SUMMARY_PROPERTIES.missingVerificationsCount] ?? 0) + missingVerificationsCount; - - this._mergeIntoSummaryUnit(runDetectorSummary, runDetectorSummaryForFlagTypesClass); - } - - return summary; - } - /** * Create new instance of quality control flags, * asynchronous for data/simulation pass or synchronous @@ -658,120 +454,6 @@ class QcFlagService { }; } - /** - * Find QC flags in GAQ effective periods for given data pass and run - * - * @param {number} dataPassId id od data pass - * @param {number} runNumber run number - * @return {Promise} promise of aggregated QC flags - */ - async getGaqFlags(dataPassId, runNumber) { - const gaqPeriods = await QcFlagRepository.findGaqPeriods(dataPassId, runNumber); - const qcFlags = (await QcFlagRepository.findAll({ - where: { id: { [Op.in]: gaqPeriods.flatMap(({ contributingFlagIds }) => contributingFlagIds) } }, - include: [ - { association: 'flagType' }, - { association: 'createdBy' }, - { association: 'verifications', include: [{ association: 'createdBy' }] }, - ], - })).map(qcFlagAdapter.toEntity); - - const idToFlag = Object.fromEntries(qcFlags.map((flag) => [flag.id, flag])); - - return gaqPeriods.map(({ - contributingFlagIds, - from, - to, - }) => ({ - from, - to, - contributingFlags: contributingFlagIds.map((id) => idToFlag[id]), - })); - } - - /** - * Get GAQ summary - * - * @param {number} dataPassId id of data pass id - * @param {object} [options] additional options - * @param {boolean} [options.mcReproducibleAsNotBad = false] if set to true, - * `Limited Acceptance MC Reproducible` flag type is treated as good one - * @return {Promise} Resolves with the GAQ Summary - */ - async getGaqSummary(dataPassId, { mcReproducibleAsNotBad = false } = {}) { - await getOneDataPassOrFail({ id: dataPassId }); - const runGaqSubSummaries = await QcFlagRepository.getRunGaqSubSummaries(dataPassId, { mcReproducibleAsNotBad }); - - const summary = {}; - const flagsAndVerifications = {}; - - // Fold list of subSummaries into one summary - for (const subSummary of runGaqSubSummaries) { - const { - runNumber, - flagsIds, - verifiedFlagsIds, - } = subSummary; - - if (!summary[runNumber]) { - summary[runNumber] = { [QC_SUMMARY_PROPERTIES.mcReproducible]: false }; - } - if (!flagsAndVerifications[runNumber]) { - flagsAndVerifications[runNumber] = {}; - } - - const runSummary = summary[runNumber]; - - const distinctRunFlagsIds = flagsAndVerifications[runNumber]?.distinctFlagsIds ?? []; - const distinctRunVerifiedFlagsIds = flagsAndVerifications[runNumber]?.distinctVerifiedFlagsIds ?? []; - - flagsAndVerifications[runNumber] = { - distinctFlagsIds: new Set([...distinctRunFlagsIds, ...flagsIds]), - distinctVerifiedFlagsIds: new Set([...distinctRunVerifiedFlagsIds, ...verifiedFlagsIds]), - }; - - this._mergeIntoSummaryUnit(runSummary, subSummary); - } - - for (const [runNumber, { distinctFlagsIds, distinctVerifiedFlagsIds }] of Object.entries(flagsAndVerifications)) { - summary[runNumber][QC_SUMMARY_PROPERTIES.missingVerificationsCount] = distinctFlagsIds.size - distinctVerifiedFlagsIds.size; - } - - return summary; - } - - /** - * Update RunDetectorQcSummary or RunGaqSummary with new information - * - * @param {RunDetectorQcSummary|RunGaqSummary} summaryUnit RunDetectorQcSummary or RunGaqSummary - * @param {{ bad: boolean, effectiveRunCoverage: number, mcReproducible: boolean}} partialSummaryUnit new properties - * to be applied to the summary object - * @return {void} - */ - _mergeIntoSummaryUnit(summaryUnit, partialSummaryUnit) { - const { - bad, - effectiveRunCoverage, - mcReproducible, - } = partialSummaryUnit; - - if (bad) { - summaryUnit[QC_SUMMARY_PROPERTIES.badEffectiveRunCoverage] = effectiveRunCoverage; - summaryUnit[QC_SUMMARY_PROPERTIES.mcReproducible] = - mcReproducible || summaryUnit[QC_SUMMARY_PROPERTIES.mcReproducible]; - } else { - summaryUnit[QC_SUMMARY_PROPERTIES.explicitlyNotBadEffectiveRunCoverage] = effectiveRunCoverage; - summaryUnit[QC_SUMMARY_PROPERTIES.mcReproducible] = - mcReproducible || summaryUnit[QC_SUMMARY_PROPERTIES.mcReproducible]; - } - if (summaryUnit[QC_SUMMARY_PROPERTIES.badEffectiveRunCoverage] === undefined) { - summaryUnit[QC_SUMMARY_PROPERTIES.badEffectiveRunCoverage] = 0; - } - if (summaryUnit[QC_SUMMARY_PROPERTIES.explicitlyNotBadEffectiveRunCoverage] === undefined) { - summaryUnit[QC_SUMMARY_PROPERTIES.explicitlyNotBadEffectiveRunCoverage] = 0; - } - } - /** * Return a paginated list of QC flags related to a given simulation pass, run and dpl detector * @@ -883,6 +565,76 @@ class QcFlagService { .orderBy('createdAt', 'DESC', 'verifications'); } + /** + * Validate QC flag timestamps + * If null timestamp was provided, given timestamp is replaced by run's startTime or endTime + * @param {Partial} timestamps QC flag timestamps + * @param {Run} targetRun run which for QC flag is to be set + * @return {{from: (number|null), to: (number|null)}} prepared timestamps + * @throws {BadParameterError} + */ + _prepareQcFlagPeriod(timestamps, targetRun) { + const { timeTrgStart, timeO2Start, firstTfTimestamp, timeTrgEnd, timeO2End, lastTfTimestamp } = targetRun; + + let lowerBound = timeTrgStart?.getTime() ?? timeO2Start?.getTime() ?? null; + if (firstTfTimestamp) { + lowerBound = lowerBound ? Math.min(firstTfTimestamp.getTime(), lowerBound) : firstTfTimestamp.getTime(); + } + + let upperBound = timeTrgEnd?.getTime() ?? timeO2End?.getTime() ?? null; + if (lastTfTimestamp) { + upperBound = upperBound ? Math.max(lastTfTimestamp.getTime(), upperBound) : lastTfTimestamp.getTime(); + } + + const from = timestamps.from ?? lowerBound ?? null; + const to = timestamps.to ?? upperBound ?? null; + + if (from && to && from >= to) { + throw new BadParameterError('Parameter "to" timestamp must be greater than "from" timestamp'); + } + const isFromOutOfRange = from && (lowerBound && from < lowerBound || upperBound && upperBound <= from); + const isToOutOfRange = to && (lowerBound && to <= lowerBound || upperBound && upperBound < to); + if (isFromOutOfRange || isToOutOfRange) { + throw new BadParameterError(`Given QC flag period (${from}, ${to}) is out of run (${lowerBound}, ${upperBound}) period`); + } + + return { from, to }; + } + + /** + * Remove a time segment from a list of QC flags effective periods + * + * @param {Period} intersectingPeriod time segment that should be removed from given periods + * @param {SequelizeQcFlagEffectivePeriod} periods periods to be updated or discarded + * @return {Promise} resolve once all periods are updated + */ + async _removeEffectivePeriodsAndPeriodIntersection({ from: newerPeriodFrom, to: newerPeriodTo }, periods) { + for (const effectivePeriod of periods) { + const { id: effectivePeriodId, from: effectiveFrom, to: effectiveTo } = effectivePeriod; + + const effectiveToIsLesserOrEqNewerPeriodTo = newerPeriodTo === null || effectiveTo !== null && effectiveTo <= newerPeriodTo; + + if (newerPeriodFrom <= effectiveFrom + && effectiveToIsLesserOrEqNewerPeriodTo) { // Old flag is fully covered by new one + await QcFlagEffectivePeriodRepository.removeOne({ where: { id: effectivePeriodId } }); + } else if (effectiveFrom < newerPeriodFrom && !effectiveToIsLesserOrEqNewerPeriodTo) { + // New flag's period is included in the old one's period. + await QcFlagEffectivePeriodRepository.update(effectivePeriod, { to: newerPeriodFrom }); + await QcFlagEffectivePeriodRepository.insert({ + flagId: effectivePeriod.flagId, + from: newerPeriodTo, + to: effectiveTo, + }); + } else if (effectiveFrom < newerPeriodFrom) { + await QcFlagEffectivePeriodRepository.update(effectivePeriod, { to: newerPeriodFrom }); + } else if (!effectiveToIsLesserOrEqNewerPeriodTo) { + await QcFlagEffectivePeriodRepository.update(effectivePeriod, { from: newerPeriodTo }); + } else { + throw new Error('Incorrect state'); + } + } + } + /** * Find a run with a given run number and including a given detector. * Throw if none exists diff --git a/lib/server/services/qualityControlFlag/QcFlagSummaryService.js b/lib/server/services/qualityControlFlag/QcFlagSummaryService.js new file mode 100644 index 0000000000..7c5bddb98b --- /dev/null +++ b/lib/server/services/qualityControlFlag/QcFlagSummaryService.js @@ -0,0 +1,198 @@ +/** + * @license + * Copyright CERN and copyright holders of ALICE O2. This software is + * distributed under the terms of the GNU General Public License v3 (GPL + * Version 3), copied verbatim in the file "COPYING". + * + * See http://alice-o2.web.cern.ch/license for full licensing information. + * + * In applying this license CERN does not waive the privileges and immunities + * granted to it by virtue of its status as an Intergovernmental Organization + * or submit itself to any jurisdiction. + */ +import { BadParameterError } from '../../errors/BadParameterError.js'; +import { dataSource } from '../../../database/DataSource.js'; +import { Op } from 'sequelize'; +import { QcFlagRepository } from '../../../database/repositories/index.js'; + +/** + * @typedef RunDetectorQcSummary + * @property {number} badEffectiveRunCoverage - fraction of run's data, marked explicitly with bad QC flag + * @property {number} explicitlyNotBadEffectiveRunCoverage - fraction of run's data, marked explicitly with good QC flag + * @property {number} missingVerificationsCount - number of not verified QC flags which are not discarded + * @property {boolean} mcReproducible - states whether some Limited Acceptance MC Reproducible flag was assigned + */ + +const QC_SUMMARY_PROPERTIES = { + badEffectiveRunCoverage: 'badEffectiveRunCoverage', + explicitlyNotBadEffectiveRunCoverage: 'explicitlyNotBadEffectiveRunCoverage', + missingVerificationsCount: 'missingVerificationsCount', + mcReproducible: 'mcReproducible', +}; + +/** + * QC flag summary service + */ +export class QcFlagSummaryService { + /** + * Update RunDetectorQcSummary or RunGaqSummary with new information + * + * @param {RunDetectorQcSummary|RunGaqSummary} summaryUnit RunDetectorQcSummary or RunGaqSummary + * @param {{ bad: boolean, effectiveRunCoverage: number, mcReproducible: boolean}} partialSummaryUnit new properties + * to be applied to the summary object + * @return {void} + */ + static mergeIntoSummaryUnit(summaryUnit, partialSummaryUnit) { + const { + bad, + effectiveRunCoverage, + mcReproducible, + } = partialSummaryUnit; + + if (bad) { + summaryUnit[QC_SUMMARY_PROPERTIES.badEffectiveRunCoverage] = effectiveRunCoverage; + summaryUnit[QC_SUMMARY_PROPERTIES.mcReproducible] = + mcReproducible || summaryUnit[QC_SUMMARY_PROPERTIES.mcReproducible]; + } else { + summaryUnit[QC_SUMMARY_PROPERTIES.explicitlyNotBadEffectiveRunCoverage] = effectiveRunCoverage; + summaryUnit[QC_SUMMARY_PROPERTIES.mcReproducible] = + mcReproducible || summaryUnit[QC_SUMMARY_PROPERTIES.mcReproducible]; + } + if (summaryUnit[QC_SUMMARY_PROPERTIES.badEffectiveRunCoverage] === undefined) { + summaryUnit[QC_SUMMARY_PROPERTIES.badEffectiveRunCoverage] = 0; + } + if (summaryUnit[QC_SUMMARY_PROPERTIES.explicitlyNotBadEffectiveRunCoverage] === undefined) { + summaryUnit[QC_SUMMARY_PROPERTIES.explicitlyNotBadEffectiveRunCoverage] = 0; + } + } + + /** + * Get QC summary for given data/simulation pass or synchronous QC flags for given LHC period + * + * @param {scope} scope of the QC flag + * @param {number} [scope.dataPassId] data pass id - exclusive with other options + * @param {number} [scope.simulationPassId] simulation pass id - exclusive with other options + * @param {number} [scope.lhcPeriodId] id of LHC Period - exclusive with other options + * @param {object} [options] additional options + * @param {boolean} [options.mcReproducibleAsNotBad = false] if set to true, `Limited Acceptance MC Reproducible` flag type is treated as + * good one + * @return {Promise} summary + */ + async getQcFlagsSummary({ dataPassId, simulationPassId, lhcPeriodId }, { mcReproducibleAsNotBad = false } = {}) { + if (Boolean(dataPassId) + Boolean(simulationPassId) + Boolean(lhcPeriodId) > 1) { + throw new BadParameterError('`dataPassId`, `simulationPassId` and `lhcPeriodId` are exclusive options'); + } + + const queryBuilder = dataSource.createQueryBuilder() + .where('deleted').is(false); + + if (dataPassId) { + queryBuilder + .whereAssociation('dataPasses', 'id').is(dataPassId) + .include({ association: 'run', attributes: [] }); + } else if (simulationPassId) { + queryBuilder + .whereAssociation('simulationPasses', 'id').is(simulationPassId) + .include({ association: 'run', attributes: [] }); + } else { + queryBuilder.include({ association: 'dataPasses', required: false }) + .include({ association: 'simulationPasses', required: false }) + .where('$dataPasses.id$').is(null) + .where('$simulationPasses.id$').is(null) + .include({ association: 'run', attributes: [], where: { lhcPeriodId } }); + } + + queryBuilder + .include({ association: 'effectivePeriods', attributes: [], required: true }) + .include({ association: 'flagType', attributes: [] }) + .set('attributes', (sequelize) => [ + 'runNumber', + 'detectorId', + [sequelize.literal(`IF(\`flagType\`.monte_carlo_reproducible AND ${mcReproducibleAsNotBad}, false, \`flagType\`.bad)`), 'bad'], + + [ + sequelize.literal(` + IF( + run.time_start IS NULL OR run.time_end IS NULL, + IF( + effectivePeriods.\`from\` IS NULL AND effectivePeriods.\`to\` IS NULL, + 1, + null + ), + SUM( + UNIX_TIMESTAMP(COALESCE(effectivePeriods.\`to\`, run.time_end)) + - UNIX_TIMESTAMP(COALESCE(effectivePeriods.\`from\`, run.time_start)) + ) / ( + UNIX_TIMESTAMP(run.time_end) - UNIX_TIMESTAMP(run.time_start) + ) + ) + `), + 'effectiveRunCoverage', + ], + [ + sequelize.literal('GROUP_CONCAT( DISTINCT `QcFlag`.id )'), + 'flagIds', + ], + [ + sequelize.literal('SUM( `flagType`.monte_carlo_reproducible ) > 0'), + 'mcReproducible', + ], + ]) + .groupBy('runNumber') + .groupBy('detectorId') + .groupBy((sequelize) => sequelize.literal(` + IF(\`flagType\`.monte_carlo_reproducible AND ${mcReproducibleAsNotBad}, false, \`flagType\`.bad) + `)); + + const runDetectorSummaryList = (await QcFlagRepository.findAll(queryBuilder)) + .map((summaryDb) => + ({ + runNumber: summaryDb.runNumber, + detectorId: summaryDb.detectorId, + effectiveRunCoverage: parseFloat(summaryDb.get('effectiveRunCoverage'), 10) || null, + bad: Boolean(summaryDb.get('bad')), + flagIds: (summaryDb.get('flagIds')?.split(',') ?? []).map((id) => parseInt(id, 10)), + mcReproducible: Boolean(summaryDb.get('mcReproducible')), + })); + + const allFlagsIds = new Set(runDetectorSummaryList.flatMap(({ flagIds }) => flagIds)); + const notVerifiedFlagsIds = new Set((await QcFlagRepository.findAll({ + attributes: ['id'], + include: [{ association: 'verifications', required: false, attributes: [] }], + where: { + id: { [Op.in]: [...allFlagsIds] }, + '$verifications.id$': { [Op.is]: null }, + }, + })).map(({ id }) => id)); + + const summary = {}; + + // Fold list of summaries into nested object + for (const runDetectorSummaryForFlagTypesClass of runDetectorSummaryList) { + const { + runNumber, + detectorId, + flagIds, + } = runDetectorSummaryForFlagTypesClass; + const missingVerificationsCount = flagIds.filter((id) => notVerifiedFlagsIds.has(id)).length; + + if (!summary[runNumber]) { + summary[runNumber] = {}; + } + if (!summary[runNumber][detectorId]) { + summary[runNumber][detectorId] = { [QC_SUMMARY_PROPERTIES.mcReproducible]: false }; + } + + const runDetectorSummary = summary[runNumber][detectorId]; + + runDetectorSummary[QC_SUMMARY_PROPERTIES.missingVerificationsCount] = + (runDetectorSummary[QC_SUMMARY_PROPERTIES.missingVerificationsCount] ?? 0) + missingVerificationsCount; + + QcFlagSummaryService.mergeIntoSummaryUnit(runDetectorSummary, runDetectorSummaryForFlagTypesClass); + } + + return summary; + } +} + +export const qcFlagSummaryService = new QcFlagSummaryService(); diff --git a/lib/usecases/run/GetAllRunsUseCase.js b/lib/usecases/run/GetAllRunsUseCase.js index 5e8c8385de..4fbaf0151a 100644 --- a/lib/usecases/run/GetAllRunsUseCase.js +++ b/lib/usecases/run/GetAllRunsUseCase.js @@ -20,7 +20,7 @@ const sequelize = require('sequelize'); const { EorReasonRepository } = require('../../database/repositories'); const { PhysicalConstant } = require('../../domain/enums/PhysicalConstant'); const { BadParameterError } = require('../../server/errors/BadParameterError'); -const { qcFlagService } = require('../../server/services/qualityControlFlag/QcFlagService'); +const { gaqService } = require('../../server/services/qualityControlFlag/GaqService.js'); /** * GetAllRunsUseCase @@ -343,7 +343,7 @@ class GetAllRunsUseCase { } const { mcReproducibleAsNotBad = false } = gaq; const [dataPassId] = dataPassIds; - const gaqSummary = await qcFlagService.getGaqSummary(dataPassId, { mcReproducibleAsNotBad }); + const gaqSummary = await gaqService.getSummary(dataPassId, { mcReproducibleAsNotBad }); for (const [operator, value] of Object.entries(gaq.notBadFraction)) { const runNumbers = Object.entries(gaqSummary) diff --git a/test/lib/server/services/qualityControlFlag/QcFlagService.test.js b/test/lib/server/services/qualityControlFlag/QcFlagService.test.js index 04396dc0a3..59d5739129 100644 --- a/test/lib/server/services/qualityControlFlag/QcFlagService.test.js +++ b/test/lib/server/services/qualityControlFlag/QcFlagService.test.js @@ -21,6 +21,8 @@ const { Op } = require('sequelize'); const { qcFlagAdapter } = require('../../../../../lib/database/adapters'); const { runService } = require('../../../../../lib/server/services/run/RunService'); const { gaqDetectorService } = require('../../../../../lib/server/services/gaq/GaqDetectorsService'); +const { gaqService } = require('../../../../../lib/server/services/qualityControlFlag/GaqService.js'); +const { qcFlagSummaryService } = require('../../../../../lib/server/services/qualityControlFlag/QcFlagSummaryService.js'); /** * Get effective part and periods of Qc flag @@ -157,7 +159,7 @@ module.exports = () => { describe('Get QC flags summary', () => { it('should successfully get non-empty QC flag summary for data pass', async () => { - expect(await qcFlagService.getQcFlagsSummary({ dataPassId: 1 })).to.be.eql({ + expect(await qcFlagSummaryService.getQcFlagsSummary({ dataPassId: 1 })).to.be.eql({ 106: { 1: { missingVerificationsCount: 3, @@ -176,7 +178,7 @@ module.exports = () => { }); it('should successfully get non-empty QC flag summary with MC.Reproducible interpreted as not-bad for data pass', async () => { - expect(await qcFlagService.getQcFlagsSummary({ dataPassId: 1 }, { mcReproducibleAsNotBad: true })).to.be.eql({ + expect(await qcFlagSummaryService.getQcFlagsSummary({ dataPassId: 1 }, { mcReproducibleAsNotBad: true })).to.be.eql({ 106: { 1: { missingVerificationsCount: 3, @@ -220,7 +222,7 @@ module.exports = () => { .filter(({ flag: { flagType: { bad } } }) => !bad) .reduce((coverage, { from, to }) => coverage + (to - from), 0); - expect(await qcFlagService.getQcFlagsSummary({ dataPassId })).to.be.eql({ + expect(await qcFlagSummaryService.getQcFlagsSummary({ dataPassId })).to.be.eql({ 1: { 1: { missingVerificationsCount: 0, @@ -237,7 +239,7 @@ module.exports = () => { // Verify flag and fetch summary one more time const relations = { user: { roles: ['admin'], externalUserId: 456 } }; await qcFlagService.verifyFlag({ flagId: 4 }, relations); - expect(await qcFlagService.getQcFlagsSummary({ dataPassId })).to.be.eql({ + expect(await qcFlagSummaryService.getQcFlagsSummary({ dataPassId })).to.be.eql({ 1: { 1: { missingVerificationsCount: 0, @@ -250,11 +252,11 @@ module.exports = () => { }); it('should successfully get empty QC flag summary for data pass', async () => { - expect(await qcFlagService.getQcFlagsSummary({ dataPassId: 3 })).to.be.eql({}); + expect(await qcFlagSummaryService.getQcFlagsSummary({ dataPassId: 3 })).to.be.eql({}); }); it('should successfully get non-empty QC flag summary for simulation pass', async () => { - expect(await qcFlagService.getQcFlagsSummary({ simulationPassId: 1 })).to.be.eql({ + expect(await qcFlagSummaryService.getQcFlagsSummary({ simulationPassId: 1 })).to.be.eql({ 106: { 1: { missingVerificationsCount: 1, @@ -267,11 +269,11 @@ module.exports = () => { }); it('should successfully get empty QC flag summary for simulation pass', async () => { - expect(await qcFlagService.getQcFlagsSummary({ simulationPassId: 2 })).to.be.eql({}); + expect(await qcFlagSummaryService.getQcFlagsSummary({ simulationPassId: 2 })).to.be.eql({}); }); it('should successfully get QC summary of synchronous QC flags for one LHC Period', async () => { - expect(await qcFlagService.getQcFlagsSummary({ lhcPeriodId: 1 })).to.be.eql({ + expect(await qcFlagSummaryService.getQcFlagsSummary({ lhcPeriodId: 1 })).to.be.eql({ 56: { // FT0 7: { @@ -714,7 +716,7 @@ module.exports = () => { createdFlagIds.push(qcFlag.id); expect(effectivePeriods.map(({ from, to }) => ({ from, to }))).to.be.eql([{ from, to }]); - qcSummary = await qcFlagService.getQcFlagsSummary({ dataPassId }); + qcSummary = await qcFlagSummaryService.getQcFlagsSummary({ dataPassId }); expect(qcSummary).to.be.eql({ 106: { 1: { @@ -762,7 +764,7 @@ module.exports = () => { effectivePeriods = await getEffectivePeriodsOfQcFlag(createdFlagIds[0]); expect(effectivePeriods.map(({ from, to }) => ({ from, to }))).to.have.all.deep.members([{ from: to, to: null }]); - qcSummary = await qcFlagService.getQcFlagsSummary({ dataPassId }); + qcSummary = await qcFlagSummaryService.getQcFlagsSummary({ dataPassId }); expect(qcSummary).to.be.eql({ 106: { 1: { @@ -1569,7 +1571,7 @@ module.exports = () => { { from: t('10:00:00'), to: t('14:00:00'), flagTypeId: goodFlagTypeId }, ], scopeFDD, relations)).map(({ id }) => id); - const gaqFlags = await qcFlagService.getGaqFlags(dataPassId, runNumber); + const gaqFlags = await gaqService.getFlagsForDataPassAndRun(dataPassId, runNumber); const data = gaqFlags.map(({ from, to, @@ -1623,7 +1625,7 @@ module.exports = () => { expectedGaqSummary.explicitlyNotBadEffectiveRunCoverage /= timeTrgEnd - timeTrgStart; expectedGaqSummary.missingVerificationsCount = 11; - const { [runNumber]: runGaqSummary } = await qcFlagService.getGaqSummary(dataPassId); + const { [runNumber]: runGaqSummary } = await gaqService.getSummary(dataPassId); expect(runGaqSummary).to.be.eql(expectedGaqSummary); const scope = { @@ -1652,7 +1654,7 @@ module.exports = () => { relations, ); - const gaqSummary = await qcFlagService.getGaqSummary(dataPassId); + const gaqSummary = await gaqService.getSummary(dataPassId); expect(gaqSummary).to.be.eql({ [runNumber]: expectedGaqSummary, 56: { From 65fa6a2ce237437e30ac281b0286e48cdce120ea Mon Sep 17 00:00:00 2001 From: martinboulais <31805063+martinboulais@users.noreply.github.com> Date: Wed, 5 Mar 2025 16:04:14 +0100 Subject: [PATCH 02/18] Update seeders so run 106 do not have first TF timestamp --- lib/database/seeders/20200713103855-runs.js | 4 ++-- .../ccdb/CcdbSynchronizer.test.js | 4 ++-- test/lib/server/externalServicesSynchronization/index.js | 2 +- test/public/qcFlags/forDataPassCreation.test.js | 6 +++--- test/public/qcFlags/forSimulationPassCreation.test.js | 6 +++--- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/lib/database/seeders/20200713103855-runs.js b/lib/database/seeders/20200713103855-runs.js index c994bd1f49..9d904287c4 100644 --- a/lib/database/seeders/20200713103855-runs.js +++ b/lib/database/seeders/20200713103855-runs.js @@ -2658,7 +2658,6 @@ module.exports = { time_o2_end: '2019-08-09 14:00:00', time_trg_start: '2019-08-08 13:00:00', time_trg_end: '2019-08-09 14:00:00', - first_tf_timestamp: '2019-08-09 13:00:00', run_type_id: 12, run_quality: 'good', n_detectors: 15, @@ -2734,9 +2733,10 @@ module.exports = { id: 108, run_number: 108, time_o2_start: '2019-08-08 13:00:00', - time_o2_end: '2019-08-09 14:00:00', + first_tf_timestamp: '2019-08-08 13:00:00', time_trg_start: '2019-08-08 13:00:00', time_trg_end: '2019-08-09 14:00:00', + time_o2_end: '2019-08-09 14:00:00', run_type_id: 12, run_quality: 'good', n_detectors: 15, diff --git a/test/lib/server/externalServicesSynchronization/ccdb/CcdbSynchronizer.test.js b/test/lib/server/externalServicesSynchronization/ccdb/CcdbSynchronizer.test.js index a852130547..f9c9269c3a 100644 --- a/test/lib/server/externalServicesSynchronization/ccdb/CcdbSynchronizer.test.js +++ b/test/lib/server/externalServicesSynchronization/ccdb/CcdbSynchronizer.test.js @@ -48,7 +48,7 @@ const dummyCcdbRunInfo = JSON.parse(`{ module.exports = () => { it('Should successfully update good physics runs first and last TF timestamps', async () => { - const expectedRunNumber = 108; + const expectedRunNumber = 106; const synchronizeAfter = new Date('2019-08-09T00:00:00'); const goodPhysicsRuns = await getGoodPhysicsRunsWithMissingTfTimestamps(synchronizeAfter); @@ -66,7 +66,7 @@ module.exports = () => { const updatedGoodPhysicsRuns = await getGoodPhysicsRunsWithMissingTfTimestamps(new Date('2019-08-09T00:00:00')); expect(updatedGoodPhysicsRuns).to.length(0); - const updatedRun = await runService.getOrFail({ runNumber: 108 }); + const updatedRun = await runService.getOrFail({ runNumber: 106 }); expect(updatedRun.firstTfTimestamp).to.equal(1565262000000); expect(updatedRun.lastTfTimestamp).to.equal(1565265600000); }); diff --git a/test/lib/server/externalServicesSynchronization/index.js b/test/lib/server/externalServicesSynchronization/index.js index 73955921a0..7a6c57e6cf 100644 --- a/test/lib/server/externalServicesSynchronization/index.js +++ b/test/lib/server/externalServicesSynchronization/index.js @@ -3,5 +3,5 @@ const CcdbSuite = require('./ccdb/index.js'); module.exports = () => { describe('MonALISA', MonalisaSuite); - describe('MonALISA', CcdbSuite); + describe('CCDB', CcdbSuite); }; diff --git a/test/public/qcFlags/forDataPassCreation.test.js b/test/public/qcFlags/forDataPassCreation.test.js index 7fba4b8f1e..d388adcb00 100644 --- a/test/public/qcFlags/forDataPassCreation.test.js +++ b/test/public/qcFlags/forDataPassCreation.test.js @@ -112,7 +112,7 @@ module.exports = () => { }); await page.waitForSelector('button#submit[disabled]'); - await expectInnerText(page, 'table > tbody > tr > td:nth-child(3) > div', '09/08/2019\n13:00:00'); + await expectInnerText(page, 'table > tbody > tr > td:nth-child(3) > div', '08/08/2019\n13:00:00'); await expectInnerText(page, 'table > tbody > tr > td:nth-child(4) > div', '09/08/2019\n14:00:00'); await page.waitForSelector('input[type="time"]', { hidden: true, timeout: 250 }); @@ -144,7 +144,7 @@ module.exports = () => { }); await page.waitForSelector('button#submit[disabled]'); - await expectInnerText(page, 'table > tbody > tr > td:nth-child(3) > div', '09/08/2019\n13:00:00'); + await expectInnerText(page, 'table > tbody > tr > td:nth-child(3) > div', '08/08/2019\n13:00:00'); await expectInnerText(page, 'table > tbody > tr > td:nth-child(4) > div', '09/08/2019\n14:00:00'); await page.waitForSelector('input[type="time"]', { hidden: true }); @@ -168,7 +168,7 @@ module.exports = () => { await expectRowValues(page, 1, { flagType: 'Limited acceptance', - from: '09/08/2019\n13:01:01', + from: '08/08/2019\n13:01:01', to: '09/08/2019\n13:50:59', }); }); diff --git a/test/public/qcFlags/forSimulationPassCreation.test.js b/test/public/qcFlags/forSimulationPassCreation.test.js index d2bb11448f..fce9fd2b51 100644 --- a/test/public/qcFlags/forSimulationPassCreation.test.js +++ b/test/public/qcFlags/forSimulationPassCreation.test.js @@ -98,7 +98,7 @@ module.exports = () => { }); await page.waitForSelector('button#submit[disabled]'); - await expectInnerText(page, 'table > tbody > tr > td:nth-child(3) > div', '09/08/2019\n13:00:00'); + await expectInnerText(page, 'table > tbody > tr > td:nth-child(3) > div', '08/08/2019\n13:00:00'); await expectInnerText(page, 'table > tbody > tr > td:nth-child(4) > div', '09/08/2019\n14:00:00'); await page.waitForSelector('input[type="time"]', { hidden: true, timeout: 250 }); @@ -130,7 +130,7 @@ module.exports = () => { }); await page.waitForSelector('button#submit[disabled]'); - await expectInnerText(page, 'table > tbody > tr > td:nth-child(3) > div', '09/08/2019\n13:00:00'); + await expectInnerText(page, 'table > tbody > tr > td:nth-child(3) > div', '08/08/2019\n13:00:00'); await expectInnerText(page, 'table > tbody > tr > td:nth-child(4) > div', '09/08/2019\n14:00:00'); await page.waitForSelector('input[type="time"]', { hidden: true, timeout: 250 }); @@ -154,7 +154,7 @@ module.exports = () => { await expectRowValues(page, 1, { flagType: 'Limited acceptance', - from: '09/08/2019\n13:01:01', + from: '08/08/2019\n13:01:01', to: '09/08/2019\n13:50:59', }); }); From 95ed19e536bbf7ef2d704989d666b444f9e3d680 Mon Sep 17 00:00:00 2001 From: martinboulais <31805063+martinboulais@users.noreply.github.com> Date: Fri, 7 Mar 2025 11:55:44 +0100 Subject: [PATCH 03/18] WIP --- lib/database/adapters/QcFlagAdapter.js | 2 +- lib/database/adapters/RunAdapter.js | 6 + lib/database/models/run.js | 6 + .../models/typedefs/SequelizeQcFlag.js | 4 +- lib/database/models/typedefs/SequelizeRun.js | 2 + .../QcFlagEffectivePeriodRepository.js | 3 +- lib/database/repositories/QcFlagRepository.js | 60 +++--- lib/domain/entities/QcFlag.js | 4 +- lib/domain/entities/Run.js | 2 + .../services/qualityControlFlag/GaqService.js | 21 ++ .../qualityControlFlag/QcFlagService.js | 193 ++++++++++-------- .../QcFlagSummaryService.js | 44 +++- .../qualityControlFlag/QcSummaryUnit.js | 18 ++ .../qualityControlFlag/QcFlagService.test.js | 163 +++++---------- 14 files changed, 287 insertions(+), 241 deletions(-) create mode 100644 lib/server/services/qualityControlFlag/QcSummaryUnit.js diff --git a/lib/database/adapters/QcFlagAdapter.js b/lib/database/adapters/QcFlagAdapter.js index d3bacab437..58c27a5de0 100644 --- a/lib/database/adapters/QcFlagAdapter.js +++ b/lib/database/adapters/QcFlagAdapter.js @@ -30,7 +30,7 @@ class QcFlagAdapter { /** * Converts the given database object to an entity object. * - * @param {SequelizeQualityControlFlag} databaseObject Object to convert. + * @param {SequelizeQcFlag} databaseObject Object to convert. * @returns {QcFlag} Converted entity object. */ toEntity(databaseObject) { diff --git a/lib/database/adapters/RunAdapter.js b/lib/database/adapters/RunAdapter.js index e10f251a77..5f05058630 100644 --- a/lib/database/adapters/RunAdapter.js +++ b/lib/database/adapters/RunAdapter.js @@ -115,6 +115,8 @@ class RunAdapter { userIdO2Stop, startTime, endTime, + qcTimeStart, + qcTimeEnd, runDuration, tags, updatedAt, @@ -183,6 +185,8 @@ class RunAdapter { userIdO2Stop, startTime, endTime, + qcTimeStart, + qcTimeEnd, runDuration, environmentId, updatedAt: new Date(updatedAt).getTime(), @@ -298,6 +302,8 @@ class RunAdapter { userIdO2Stop: entityObject.userIdO2Stop, startTime: entityObject.startTime, endTime: entityObject.endTime, + qcTimeStart: entityObject.qcTimeStart, + qcTimeEnd: entityObject.qcTimeEnd, environmentId: entityObject.environmentId, runTypeId: entityObject.runTypeId, runQuality: entityObject.runQuality, diff --git a/lib/database/models/run.js b/lib/database/models/run.js index d1914693a4..04a12f76e4 100644 --- a/lib/database/models/run.js +++ b/lib/database/models/run.js @@ -65,6 +65,12 @@ module.exports = (sequelize) => { return (runEndString ? new Date(runEndString) : new Date()).getTime(); }, }, + qcTimeStart: { + type: Sequelize.DATE, + }, + qcTimeEnd: { + type: Sequelize.DATE, + }, runDuration: { type: Sequelize.VIRTUAL, // eslint-disable-next-line require-jsdoc diff --git a/lib/database/models/typedefs/SequelizeQcFlag.js b/lib/database/models/typedefs/SequelizeQcFlag.js index a40ad6df2e..0361b19c53 100644 --- a/lib/database/models/typedefs/SequelizeQcFlag.js +++ b/lib/database/models/typedefs/SequelizeQcFlag.js @@ -16,8 +16,8 @@ * * @property {number} id * @property {boolean} deleted - * @property {number} from - * @property {number} to + * @property {number|null} from + * @property {number|null} to * @property {string} comment * @property {string} origin * @property {number} createdById diff --git a/lib/database/models/typedefs/SequelizeRun.js b/lib/database/models/typedefs/SequelizeRun.js index 13b354ff3b..40ff5f2d26 100644 --- a/lib/database/models/typedefs/SequelizeRun.js +++ b/lib/database/models/typedefs/SequelizeRun.js @@ -25,6 +25,8 @@ * @property {number|null} userIdO2Stop relation to the user id that stopped the run * @property {number|null} startTime timestamp of the run's start, either trigger start if it exists or o2 start or null * @property {number|null} endTime timestamp of the run's end, either trigger end if it exists or o2 end or now (always null if start is null) + * @property {number|null} qcTimeStart coalesce of run first TF timestamp, trigger start and run o2 start + * @property {number|null} qcTimeEnd coalesce of run last TF timestamp, trigger stop and run o2 end * @property {string|null} environmentId * @property {string} runQuality * @property {number|null} nDetectors diff --git a/lib/database/repositories/QcFlagEffectivePeriodRepository.js b/lib/database/repositories/QcFlagEffectivePeriodRepository.js index d48e652483..b4a5ef8899 100644 --- a/lib/database/repositories/QcFlagEffectivePeriodRepository.js +++ b/lib/database/repositories/QcFlagEffectivePeriodRepository.js @@ -32,13 +32,14 @@ class QcFlagEffectivePeriodRepository extends Repository { * * @param {Period} period period which effective periods must overlap with * @param {number|Date} createdAtUpperLimit upper limit of QC flags creation timestamp which effective periods are to be found + * @param {object} monalisaProduction the scope of the flag * @param {number} [monalisaProduction.dataPassId] id of data pass, which the QC flag belongs to * @param {number} [monalisaProduction.simulationPassId] id of simulation pass, which the QC flags belongs to * @param {number} monalisaProduction.runNumber runNumber of run, which the QC flags belongs to * @param {number} monalisaProduction.detectorId id of DPL detector, which the QC flags belongs to * @return {Promise} effective periods promise */ - async findOverlappingPeriodsCreatedNotAfter(period, createdAtUpperLimit, { dataPassId, simulationPassId, runNumber, detectorId }) { + async findOverlappingPeriodsCreatedBeforeLimit(period, createdAtUpperLimit, { dataPassId, simulationPassId, runNumber, detectorId }) { const { to, from } = period; const flagIncludes = []; diff --git a/lib/database/repositories/QcFlagRepository.js b/lib/database/repositories/QcFlagRepository.js index dccde007fb..66367c8aec 100644 --- a/lib/database/repositories/QcFlagRepository.js +++ b/lib/database/repositories/QcFlagRepository.js @@ -16,23 +16,22 @@ const { models: { QcFlag } } = require('..'); const Repository = require('./Repository'); const GAQ_PERIODS_VIEW = ` + SELECT * FROM ( SELECT data_pass_id, run_number, - timestamp AS \`from\`, - NTH_VALUE(timestamp, 2) OVER ( - PARTITION BY data_pass_id, - run_number - ORDER BY ap.timestamp - ROWS BETWEEN CURRENT ROW AND 1 FOLLOWING - ) AS \`to\` + LAG(timestamp) OVER w AS \`from\`, + timestamp AS \`to\`, + LAG(ordering_timestamp) OVER w AS from_ordering_timestamp FROM ( ( SELECT gaqd.data_pass_id, gaqd.run_number, - COALESCE(UNIX_TIMESTAMP(qcfep.\`from\`), 0) AS timestamp + qcfep.\`from\` AS timestamp, + COALESCE(qcfep.\`from\`, r.qc_time_start, '0001-01-01 00:00:00.000') AS ordering_timestamp FROM quality_control_flag_effective_periods AS qcfep INNER JOIN quality_control_flags AS qcf ON qcf.id = qcfep.flag_id + INNER JOIN runs AS r ON qcf.run_number = r.run_number INNER JOIN data_pass_quality_control_flag AS dpqcf ON dpqcf.quality_control_flag_id = qcf.id -- Only flags of detectors which are defined in global_aggregated_quality_detectors -- should be taken into account for calculation of gaq_effective_periods @@ -45,9 +44,11 @@ const GAQ_PERIODS_VIEW = ` ( SELECT gaqd.data_pass_id, gaqd.run_number, - UNIX_TIMESTAMP(COALESCE(qcfep.\`to\`, NOW())) AS timestamp + qcfep.\`to\` AS timestamp, + COALESCE(qcfep.\`to\`, r.qc_time_end, NOW()) AS ordering_timestamp FROM quality_control_flag_effective_periods AS qcfep INNER JOIN quality_control_flags AS qcf ON qcf.id = qcfep.flag_id + INNER JOIN runs AS r ON qcf.run_number = r.run_number INNER JOIN data_pass_quality_control_flag AS dpqcf ON dpqcf.quality_control_flag_id = qcf.id -- Only flags of detectors which are defined in global_aggregated_quality_detectors -- should be taken into account for calculation of gaq_effective_periods @@ -56,8 +57,15 @@ const GAQ_PERIODS_VIEW = ` AND gaqd.run_number = qcf.run_number AND gaqd.detector_id = qcf.detector_id ) - ORDER BY timestamp - ) AS ap + ORDER BY ordering_timestamp + ) AS ap + WINDOW w AS ( + PARTITION BY data_pass_id, + run_number + ORDER BY ap.ordering_timestamp + ) + ) as gaq_periods_with_last_nullish_row + WHERE gaq_periods_with_last_nullish_row.from_ordering_timestamp IS NOT NULL `; /** @@ -104,8 +112,8 @@ class QcFlagRepository extends Repository { SELECT gaq_periods.data_pass_id AS dataPassId, gaq_periods.run_number AS runNumber, - IF(gaq_periods.\`from\` = 0, null, gaq_periods.\`from\` * 1000) AS \`from\`, - IF(gaq_periods.\`to\` = UNIX_TIMESTAMP(NOW()), null, gaq_periods.\`to\` * 1000) AS \`to\`, + gaq_periods.\`from\` AS \`from\`, + gaq_periods.\`to\` AS \`to\`, group_concat(qcf.id) AS contributingFlagIds FROM quality_control_flags AS qcf @@ -118,8 +126,8 @@ class QcFlagRepository extends Repository { AND gaqd.run_number = gaq_periods.run_number AND gaqd.detector_id = qcf.detector_id AND gaq_periods.run_number = qcf.run_number - AND (qcfep.\`from\` IS NULL OR UNIX_TIMESTAMP(qcfep.\`from\`) <= gaq_periods.\`from\`) - AND (qcfep.\`to\` IS NULL OR gaq_periods.\`to\` <= UNIX_TIMESTAMP(qcfep.\`to\`)) + AND (qcfep.\`from\` IS NULL OR qcfep.\`from\` <= gaq_periods.\`from\`) + AND (qcfep.\`to\` IS NULL OR gaq_periods.\`to\` <= qcfep.\`to\`) WHERE gaq_periods.data_pass_id = ${dataPassId} ${runNumber ? `AND gaq_periods.run_number = ${runNumber}` : ''} @@ -140,8 +148,8 @@ class QcFlagRepository extends Repository { }) => ({ dataPassId, runNumber, - from, - to, + from: from?.getTime(), + to: to?.getTime(), contributingFlagIds: contributingFlagIds.split(',').map((id) => parseInt(id, 10)), })); } @@ -160,8 +168,8 @@ class QcFlagRepository extends Repository { SELECT gaq_periods.data_pass_id AS dataPassId, gaq_periods.run_number AS runNumber, - IF(gaq_periods.\`from\` = 0, null, gaq_periods.\`from\`) AS \`from\`, - IF(gaq_periods.\`to\` = UNIX_TIMESTAMP(NOW()), null, gaq_periods.\`to\`) AS \`to\`, + gaq_periods.\`from\` AS \`from\`, + gaq_periods.\`to\` AS \`to\`, SUM(IF(qcft.monte_carlo_reproducible AND :mcReproducibleAsNotBad, false, qcft.bad)) >= 1 AS bad, SUM(qcft.bad) = SUM(qcft.monte_carlo_reproducible) AND SUM(qcft.monte_carlo_reproducible) AS mcReproducible, GROUP_CONCAT( DISTINCT qcfv.flag_id ) AS verifiedFlagsList, @@ -183,8 +191,8 @@ class QcFlagRepository extends Repository { AND gaqd.run_number = gaq_periods.run_number AND gaqd.detector_id = qcf.detector_id AND gaq_periods.run_number = qcf.run_number - AND (qcfep.\`from\` IS NULL OR UNIX_TIMESTAMP(qcfep.\`from\`) <= gaq_periods.\`from\`) - AND (qcfep.\`to\` IS NULL OR gaq_periods.\`to\` <= UNIX_TIMESTAMP(qcfep.\`to\`)) + AND (qcfep.\`from\` IS NULL OR qcfep.\`from\` <= gaq_periods.\`from\`) + AND (qcfep.\`to\` IS NULL OR gaq_periods.\`to\` <= qcfep.\`to\`) GROUP BY gaq_periods.data_pass_id, @@ -203,18 +211,16 @@ class QcFlagRepository extends Repository { GROUP_CONCAT(effectivePeriods.flagsList) AS flagsList, IF( - run.time_start IS NULL OR run.time_end IS NULL, + run.qc_time_start IS NULL OR run.qc_time_end IS NULL, IF( effectivePeriods.\`from\` IS NULL AND effectivePeriods.\`to\` IS NULL, 1, null ), SUM( - COALESCE(effectivePeriods.\`to\`, UNIX_TIMESTAMP(run.time_end)) - - COALESCE(effectivePeriods.\`from\`, UNIX_TIMESTAMP(run.time_start)) - ) / ( - UNIX_TIMESTAMP(run.time_end) - UNIX_TIMESTAMP(run.time_start) - ) + UNIX_TIMESTAMP(COALESCE(effectivePeriods.\`to\`,run.qc_time_end)) + - UNIX_TIMESTAMP(COALESCE(effectivePeriods.\`from\`, run.qc_time_start)) + ) / (UNIX_TIMESTAMP(run.qc_time_end) - UNIX_TIMESTAMP(run.qc_time_start)) ) AS effectiveRunCoverage FROM (${effectivePeriodsWithTypeSubQuery}) AS effectivePeriods diff --git a/lib/domain/entities/QcFlag.js b/lib/domain/entities/QcFlag.js index 752b40b215..24ece7ef61 100644 --- a/lib/domain/entities/QcFlag.js +++ b/lib/domain/entities/QcFlag.js @@ -16,8 +16,8 @@ * * @property {number} id * @property {boolean} deleted - * @property {number} from - * @property {number} to + * @property {number|null} from + * @property {number|null} to * @property {string} comment * @property {string} origin * @property {number} createdById diff --git a/lib/domain/entities/Run.js b/lib/domain/entities/Run.js index 4fb9788222..d1d263b9a5 100644 --- a/lib/domain/entities/Run.js +++ b/lib/domain/entities/Run.js @@ -25,6 +25,8 @@ * @property {number|null} userIdO2Stop relation to the user id that stopped the run * @property {Date|null} startTime timestamp of the run's start, either trigger start if it exists or o2 start or null * @property {Date|null} endTime timestamp of the run's end, either trigger end if it exists or o2 end or now (always null if start is null) + * @property {Date|null} qcTimeStart coalesce of run first TF timestamp, trigger start and run o2 start + * @property {Date|null} qcTimeEnd coalesce of run last TF timestamp, trigger stop and run o2 end * @property {string|null} environmentId * @property {number|null} runTypeId * @property {string} runQuality diff --git a/lib/server/services/qualityControlFlag/GaqService.js b/lib/server/services/qualityControlFlag/GaqService.js index 00bf00657b..349ed0388a 100644 --- a/lib/server/services/qualityControlFlag/GaqService.js +++ b/lib/server/services/qualityControlFlag/GaqService.js @@ -16,6 +16,27 @@ import { qcFlagAdapter } from '../../../database/adapters/index.js'; import { QcFlagRepository } from '../../../database/repositories/index.js'; import { QcFlagSummaryService } from './QcFlagSummaryService.js'; +/** + * @typedef GaqFlags + * + * @property {number} from + * @property {number} to + * @property {QcFlag[]} contributingFlags + */ + +/** + * @typedef RunGaqSummary + * @property {number} badEffectiveRunCoverage - fraction of run's data, which aggregated quality is bad + * @property {number} explicitlyNotBadEffectiveRunCoverage - fraction of run's data, which aggregated quality is explicitly good + * @property {number} missingVerificationsCount - number of not verified QC flags which are not discarded + * @property {boolean} mcReproducible - states whether some aggregation of QC flags is Limited Acceptance MC Reproducible + */ + +/** + * @typedef GaqSummary aggregated global quality summaries for given data pass + * @type {Object} runNumber to RunGaqSummary mapping + */ + const QC_SUMMARY_PROPERTIES = { badEffectiveRunCoverage: 'badEffectiveRunCoverage', explicitlyNotBadEffectiveRunCoverage: 'explicitlyNotBadEffectiveRunCoverage', diff --git a/lib/server/services/qualityControlFlag/QcFlagService.js b/lib/server/services/qualityControlFlag/QcFlagService.js index d562dc47ef..6491007a40 100644 --- a/lib/server/services/qualityControlFlag/QcFlagService.js +++ b/lib/server/services/qualityControlFlag/QcFlagService.js @@ -38,8 +38,8 @@ const { DetectorType } = require('../../../domain/enums/DetectorTypes.js'); /** * @typedef UserWithRoles - * @property {number} userId - * @property {number} externalUserId + * @property {number} [userId] + * @property {number} [externalUserId] * @property {string[]} roles */ @@ -63,45 +63,6 @@ const validateUserDetectorAccess = (userRoles, detectorName) => { } }; -/** - * @typedef RunQcSummary - * @type {Object} dplDetectorID to RunDetectorQcSummary mappings - */ - -/** - * @typedef QcSummary - * @type {Object>} runNumber to RunQcSummary mapping - */ - -/** - * @typedef GaqFlags - * - * @property {number} from - * @property {number} to - * @property {QcFlag[]} contributingFlags - */ - -/** - * @typedef RunGaqSummary - * @property {number} badEffectiveRunCoverage - fraction of run's data, which aggregated quality is bad - * @property {number} explicitlyNotBadEffectiveRunCoverage - fraction of run's data, which aggregated quality is explicitly good - * @property {number} missingVerificationsCount - number of not verified QC flags which are not discarded - * @property {boolean} mcReproducible - states whether some aggregation of QC flags is Limited Acceptance MC Reproducible - */ - -/** - * @typedef RunDetectorQcSummary - * @property {number} badEffectiveRunCoverage - fraction of run's data, marked explicitly with bad QC flag - * @property {number} explicitlyNotBadEffectiveRunCoverage - fraction of run's data, marked explicitly with good QC flag - * @property {number} missingVerificationsCount - number of not verified QC flags which are not discarded - * @property {boolean} mcReproducible - states whether some Limited Acceptance MC Reproducible flag was assigned - */ - -/** - * @typedef GaqSummary aggregated global quality summaries for given data pass - * @type {Object} runNumber to RunGaqSummary mapping - */ - /** * Quality control flags service */ @@ -175,6 +136,7 @@ class QcFlagService { return dataSource.transaction(async () => { const user = await getUserOrFail({ userId, externalUserId }); const detector = await getQcDetectorOrFail(detectorIdentifier); + const dataPass = dataPassIdentifier ? await getOneDataPassOrFail(dataPassIdentifier) : null; const simulationPass = simulationPassIdentifier ? await getOneSimulationPassOrFail(simulationPassIdentifier) : null; @@ -208,6 +170,7 @@ class QcFlagService { await newInstance.addSimulationPass(simulationPass); } + /** @var {SequelizeQcFlag} createdFlag */ const createdFlag = await QcFlagRepository.findOne({ where: { id: newInstance.id }, include: [ @@ -218,7 +181,7 @@ class QcFlagService { }); // Update effective periods - const effectivePeriodsToBeUpdated = await QcFlagEffectivePeriodRepository.findOverlappingPeriodsCreatedNotAfter( + const effectivePeriodsToBeUpdated = await QcFlagEffectivePeriodRepository.findOverlappingPeriodsCreatedBeforeLimit( { from, to }, createdFlag.createdAt, { dataPassId: dataPass?.id, simulationPassId: simulationPass?.id, runNumber, detectorId: detector.id }, @@ -288,7 +251,7 @@ class QcFlagService { for (const potentiallyOverlappingFlag of [...flagsCreatedBeforeRemovedFlag, ...flagsCreatedAfterRemovedFlag]) { const { id, from, to, createdAt, runNumber, detectorId } = potentiallyOverlappingFlag; - const overlappingEffectivePeriods = (await QcFlagEffectivePeriodRepository.findOverlappingPeriodsCreatedNotAfter( + const overlappingEffectivePeriods = (await QcFlagEffectivePeriodRepository.findOverlappingPeriodsCreatedBeforeLimit( { from, to }, createdAt, { dataPassId, simulationPassId, runNumber, detectorId }, @@ -548,7 +511,7 @@ class QcFlagService { ]); return { - count: count, + count, rows: rows.map(qcFlagAdapter.toEntity), }; } @@ -574,28 +537,20 @@ class QcFlagService { * @throws {BadParameterError} */ _prepareQcFlagPeriod(timestamps, targetRun) { - const { timeTrgStart, timeO2Start, firstTfTimestamp, timeTrgEnd, timeO2End, lastTfTimestamp } = targetRun; + const { qcTimeStart, qcTimeEnd } = targetRun; + const runStart = qcTimeStart?.getTime(); + const runEnd = qcTimeEnd?.getTime(); - let lowerBound = timeTrgStart?.getTime() ?? timeO2Start?.getTime() ?? null; - if (firstTfTimestamp) { - lowerBound = lowerBound ? Math.min(firstTfTimestamp.getTime(), lowerBound) : firstTfTimestamp.getTime(); - } - - let upperBound = timeTrgEnd?.getTime() ?? timeO2End?.getTime() ?? null; - if (lastTfTimestamp) { - upperBound = upperBound ? Math.max(lastTfTimestamp.getTime(), upperBound) : lastTfTimestamp.getTime(); - } - - const from = timestamps.from ?? lowerBound ?? null; - const to = timestamps.to ?? upperBound ?? null; + const { from, to } = timestamps; if (from && to && from >= to) { throw new BadParameterError('Parameter "to" timestamp must be greater than "from" timestamp'); } - const isFromOutOfRange = from && (lowerBound && from < lowerBound || upperBound && upperBound <= from); - const isToOutOfRange = to && (lowerBound && to <= lowerBound || upperBound && upperBound < to); + + const isFromOutOfRange = from && (runStart && from < runStart || runEnd && runEnd <= from); + const isToOutOfRange = to && (runStart && to <= runStart || runEnd && runEnd < to); if (isFromOutOfRange || isToOutOfRange) { - throw new BadParameterError(`Given QC flag period (${from}, ${to}) is out of run (${lowerBound}, ${upperBound}) period`); + throw new BadParameterError(`Given QC flag period (${from}, ${to}) is out of run (${runStart}, ${runEnd}) period`); } return { from, to }; @@ -604,33 +559,103 @@ class QcFlagService { /** * Remove a time segment from a list of QC flags effective periods * - * @param {Period} intersectingPeriod time segment that should be removed from given periods - * @param {SequelizeQcFlagEffectivePeriod} periods periods to be updated or discarded + * @param {Partial} eraseWindow time segment that should be removed from given periods + * @param {SequelizeQcFlagEffectivePeriod[]} periods effective periods from which time window should be erased * @return {Promise} resolve once all periods are updated */ - async _removeEffectivePeriodsAndPeriodIntersection({ from: newerPeriodFrom, to: newerPeriodTo }, periods) { - for (const effectivePeriod of periods) { - const { id: effectivePeriodId, from: effectiveFrom, to: effectiveTo } = effectivePeriod; - - const effectiveToIsLesserOrEqNewerPeriodTo = newerPeriodTo === null || effectiveTo !== null && effectiveTo <= newerPeriodTo; - - if (newerPeriodFrom <= effectiveFrom - && effectiveToIsLesserOrEqNewerPeriodTo) { // Old flag is fully covered by new one - await QcFlagEffectivePeriodRepository.removeOne({ where: { id: effectivePeriodId } }); - } else if (effectiveFrom < newerPeriodFrom && !effectiveToIsLesserOrEqNewerPeriodTo) { - // New flag's period is included in the old one's period. - await QcFlagEffectivePeriodRepository.update(effectivePeriod, { to: newerPeriodFrom }); + async _removeEffectivePeriodsAndPeriodIntersection(eraseWindow, periods) { + /* + * Example of what is named `before` and `after` an erase window: + * + * ------------------------------[ erase window ]----------------------------> time + * [ period BEFORE erase window ] [ period AFTER erase window ] + * + * Using this flag for example: + * ---------------------[ QC flag period ]-------------------- + * + * The corresponding sub-periods will be: + * ---------------------[ ]-------------------------------------------- => sub-period BEFORE erase window + * ----------------------------------------------[ ]-------------------- => sub-period AFTER erase window + * + * The following flag would have no sub-period after: + * ----------------[ QC flag period ]---------------------------------------- + * ----------------[ ]-------------------------------------------- => sub-period BEFORE erase window + * + * And the following flag would have no su-period before + * ---------------------------------------[ QC flag period ]----------------- + * ----------------------------------------------[ ]----------------- => sub-period AFTER erase window + */ + + /** + * Return the sub-period left before the erase window + * + * @param {Partial} period the period from which a time segment is to be removed + * @param {Partial} eraseWindow the erase window + * @return {Partial|null} the resulting sub-period, if it exists + */ + const getPeriodBeforeEraseWindow = (period, eraseWindow) => { + if ( + eraseWindow.from === null + || period.from !== null && period.from.getTime() >= eraseWindow.from // Period started after erase window from + ) { + return null; + } + + return { + from: period.from?.getTime(), + // If period.to is null, simply goes to eraseWindow.from + to: Math.min(period.to?.getTime() ?? eraseWindow.from, eraseWindow.from), + }; + }; + + /** + * Return the sub-period left after the erase window + * + * @param {Partial} period the period from which a time segment is to be removed + * @param {Partial} eraseWindow the erase window + * @return {Partial|null} the resulting sub-period, if it exists + */ + const getPeriodAfterEraseWindow = (period, eraseWindow) => { + if ( + eraseWindow.to === null + || period.to !== null && period.to.getTime() <= eraseWindow.to + ) { + return null; + } + + return { + from: Math.max(period.from?.getTime() ?? eraseWindow.to, eraseWindow.to), // If period.from is null, start from eraseWindow.to + to: period.to?.getTime(), + }; + }; + for (const period of periods) { + // Consider what effective period is left before the erase window + const periodBeforeEraseWindow = getPeriodBeforeEraseWindow(period, eraseWindow); + + // Consider what effective period is left after the erase window + const periodAfterEraseWindow = getPeriodAfterEraseWindow(period, eraseWindow); + + /* + * Four cases: + * - Empty sub-periods before and after => delete the whole effective period + * - Empty sub-period before and non-empty after => crop `from` + * - Empty sub-period after and non-empty before => crop `to` + * - two non-empty sub-periods => split the effective period in two, + * actually keeping the existing one as `before` and creating `after` + */ + if (periodBeforeEraseWindow === null && periodAfterEraseWindow === null) { // Old flag is fully covered by new one + await QcFlagEffectivePeriodRepository.removeOne({ where: { id: period.id } }); + } else if (periodBeforeEraseWindow === null) { + await QcFlagEffectivePeriodRepository.update(period, { from: periodAfterEraseWindow.from }); + } else if (periodAfterEraseWindow === null) { + await QcFlagEffectivePeriodRepository.update(period, { to: periodBeforeEraseWindow.to }); + } else { + await QcFlagEffectivePeriodRepository.update(period, { to: periodBeforeEraseWindow.to }); // Reuse the existing one await QcFlagEffectivePeriodRepository.insert({ - flagId: effectivePeriod.flagId, - from: newerPeriodTo, - to: effectiveTo, + flagId: period.flagId, + from: periodAfterEraseWindow.from, + to: periodAfterEraseWindow.to, }); - } else if (effectiveFrom < newerPeriodFrom) { - await QcFlagEffectivePeriodRepository.update(effectivePeriod, { to: newerPeriodFrom }); - } else if (!effectiveToIsLesserOrEqNewerPeriodTo) { - await QcFlagEffectivePeriodRepository.update(effectivePeriod, { from: newerPeriodTo }); - } else { - throw new Error('Incorrect state'); } } } @@ -681,7 +706,7 @@ class QcFlagService { const run = await RunRepository.findOne({ subQuery: false, - attributes: ['timeO2Start', 'timeTrgStart', 'firstTfTimestamp', 'lastTfTimestamp', 'timeTrgEnd', 'timeO2End'], + attributes: ['qcTimeStart', 'qcTimeEnd'], where: { runNumber }, include: runInclude, }); diff --git a/lib/server/services/qualityControlFlag/QcFlagSummaryService.js b/lib/server/services/qualityControlFlag/QcFlagSummaryService.js index 7c5bddb98b..6d2d847ef9 100644 --- a/lib/server/services/qualityControlFlag/QcFlagSummaryService.js +++ b/lib/server/services/qualityControlFlag/QcFlagSummaryService.js @@ -14,6 +14,25 @@ import { BadParameterError } from '../../errors/BadParameterError.js'; import { dataSource } from '../../../database/DataSource.js'; import { Op } from 'sequelize'; import { QcFlagRepository } from '../../../database/repositories/index.js'; +import { qcFlagEffectivePeriodAdapter } from '../../../database/adapters/index.js'; + +/** + * @typedef RunQcSummary + * @type {Object} dplDetectorID to RunDetectorQcSummary mappings + */ + +/** + * @typedef QcSummary + * @type {Object>} runNumber to RunQcSummary mapping + */ + +/** + * @typedef RunDetectorQcSummary + * @property {number} badEffectiveRunCoverage - fraction of run's data, marked explicitly with bad QC flag + * @property {number} explicitlyNotBadEffectiveRunCoverage - fraction of run's data, marked explicitly with good QC flag + * @property {number} missingVerificationsCount - number of not verified QC flags which are not discarded + * @property {boolean} mcReproducible - states whether some Limited Acceptance MC Reproducible flag was assigned + */ /** * @typedef RunDetectorQcSummary @@ -113,18 +132,16 @@ export class QcFlagSummaryService { [ sequelize.literal(` IF( - run.time_start IS NULL OR run.time_end IS NULL, + run.qc_time_start IS NULL OR run.qc_time_end IS NULL, IF( effectivePeriods.\`from\` IS NULL AND effectivePeriods.\`to\` IS NULL, 1, null ), SUM( - UNIX_TIMESTAMP(COALESCE(effectivePeriods.\`to\`, run.time_end)) - - UNIX_TIMESTAMP(COALESCE(effectivePeriods.\`from\`, run.time_start)) - ) / ( - UNIX_TIMESTAMP(run.time_end) - UNIX_TIMESTAMP(run.time_start) - ) + UNIX_TIMESTAMP(COALESCE(effectivePeriods.\`to\`,run.qc_time_end)) + - UNIX_TIMESTAMP(COALESCE(effectivePeriods.\`from\`, run.qc_time_start)) + ) / (UNIX_TIMESTAMP(run.qc_time_end) - UNIX_TIMESTAMP(run.qc_time_start)) ) `), 'effectiveRunCoverage', @@ -145,15 +162,22 @@ export class QcFlagSummaryService { `)); const runDetectorSummaryList = (await QcFlagRepository.findAll(queryBuilder)) - .map((summaryDb) => - ({ + .map((summaryDb) => { + const effectiveRunCoverageString = summaryDb.get('effectiveRunCoverage'); + const effectiveRunCoverage = (effectiveRunCoverageString ?? null) !== null + ? parseFloat(effectiveRunCoverageString) + : null; + console.log(effectiveRunCoverageString); + + return { runNumber: summaryDb.runNumber, detectorId: summaryDb.detectorId, - effectiveRunCoverage: parseFloat(summaryDb.get('effectiveRunCoverage'), 10) || null, + effectiveRunCoverage, bad: Boolean(summaryDb.get('bad')), flagIds: (summaryDb.get('flagIds')?.split(',') ?? []).map((id) => parseInt(id, 10)), mcReproducible: Boolean(summaryDb.get('mcReproducible')), - })); + }; + }); const allFlagsIds = new Set(runDetectorSummaryList.flatMap(({ flagIds }) => flagIds)); const notVerifiedFlagsIds = new Set((await QcFlagRepository.findAll({ diff --git a/lib/server/services/qualityControlFlag/QcSummaryUnit.js b/lib/server/services/qualityControlFlag/QcSummaryUnit.js new file mode 100644 index 0000000000..3d008a659b --- /dev/null +++ b/lib/server/services/qualityControlFlag/QcSummaryUnit.js @@ -0,0 +1,18 @@ +/** + * @license + * Copyright CERN and copyright holders of ALICE O2. This software is + * distributed under the terms of the GNU General Public License v3 (GPL + * Version 3), copied verbatim in the file "COPYING". + * + * See http://alice-o2.web.cern.ch/license for full licensing information. + * + * In applying this license CERN does not waive the privileges and immunities + * granted to it by virtue of its status as an Intergovernmental Organization + * or submit itself to any jurisdiction. + */ + +/** + * QC summary unit + */ +export class QcSummaryUnit { +} diff --git a/test/lib/server/services/qualityControlFlag/QcFlagService.test.js b/test/lib/server/services/qualityControlFlag/QcFlagService.test.js index 59d5739129..f9c8d0e6d8 100644 --- a/test/lib/server/services/qualityControlFlag/QcFlagService.test.js +++ b/test/lib/server/services/qualityControlFlag/QcFlagService.test.js @@ -415,7 +415,7 @@ module.exports = () => { ); }); - it('should successfully create quality control flag with externalUserId', async () => { + it('should successfully create quality control and update ', async () => { const qcFlags = [ { from: new Date('2019-08-09 01:29:50').getTime(), @@ -639,11 +639,9 @@ module.exports = () => { }, ] = await qcFlagService.create([qcFlag], scope, relations); - const { startTime, endTime } = await RunRepository.findOne({ where: { runNumber } }); - expect({ from, to, comment, flagTypeId, runNumber, detectorId, externalUserId }).to.be.eql({ - from: startTime, - to: endTime, + from: null, + to: null, comment: qcFlag.comment, flagTypeId: qcFlag.flagTypeId, runNumber: scope.runNumber, @@ -692,18 +690,18 @@ module.exports = () => { const run = await RunRepository.findOne({ where: { runNumber } }); // Create run without timestamps await run.addDataPass(dataPassId); await run.addDetector(detectorId); - let from; - let to; - let qcFlag; - let effectivePeriods; - let qcSummary; const createdFlagIds = []; + const secondFlagTo = new Date('2024-07-01 12:00:00').getTime(); + const thirdFlagFrom = new Date('2024-07-01 16:00:00').getTime(); + const fourthFlagFrom = new Date('2024-07-01 13:00:00').getTime(); + const fourthFlagTo = new Date('2024-07-01 15:00:00').getTime(); + // 1 { - from = null; - to = null; - [qcFlag] = await qcFlagService.create( + const from = null; + const to = null; + const [qcFlag] = await qcFlagService.create( [{ flagTypeId, from, to }], { runNumber, @@ -712,42 +710,19 @@ module.exports = () => { }, relations, ); - effectivePeriods = await getEffectivePeriodsOfQcFlag(qcFlag.id); createdFlagIds.push(qcFlag.id); - expect(effectivePeriods.map(({ from, to }) => ({ from, to }))).to.be.eql([{ from, to }]); - - qcSummary = await qcFlagSummaryService.getQcFlagsSummary({ dataPassId }); - expect(qcSummary).to.be.eql({ - 106: { - 1: { - badEffectiveRunCoverage: 1, - mcReproducible: false, - missingVerificationsCount: 1, - explicitlyNotBadEffectiveRunCoverage: 0, - }, - 16: { - badEffectiveRunCoverage: 0, - explicitlyNotBadEffectiveRunCoverage: 1, - mcReproducible: false, - missingVerificationsCount: 1, - }, - }, - [runNumber]: { - [detectorId]: { - badEffectiveRunCoverage: 1, - mcReproducible: true, - missingVerificationsCount: 1, - explicitlyNotBadEffectiveRunCoverage: 0, - }, - }, - }); + + const effectivePeriods = await getEffectivePeriodsOfQcFlag(qcFlag.id); + expect(effectivePeriods).to.lengthOf(1); + expect(effectivePeriods[0].from).to.equal(null); + expect(effectivePeriods[0].to).to.equal(null); } // 2 { - from = null; - to = new Date('2024-07-01 12:00:00').getTime(); - [qcFlag] = await qcFlagService.create( + const from = null; + const to = secondFlagTo; + const [qcFlag] = await qcFlagService.create( [{ flagTypeId, from, to }], { runNumber, @@ -756,46 +731,25 @@ module.exports = () => { }, relations, ); - effectivePeriods = await getEffectivePeriodsOfQcFlag(qcFlag.id); createdFlagIds.push(qcFlag.id); - expect(effectivePeriods.map(({ from, to }) => ({ from, to }))).to.have.all.deep.members([{ from, to }]); + + const newFlagEffectivePeriods = await getEffectivePeriodsOfQcFlag(qcFlag.id); + expect(newFlagEffectivePeriods).to.lengthOf(1); + expect(newFlagEffectivePeriods[0].from).to.equal(null); + expect(newFlagEffectivePeriods[0].to).to.equal(to); // Previous: first flag - effectivePeriods = await getEffectivePeriodsOfQcFlag(createdFlagIds[0]); - expect(effectivePeriods.map(({ from, to }) => ({ from, to }))).to.have.all.deep.members([{ from: to, to: null }]); - - qcSummary = await qcFlagSummaryService.getQcFlagsSummary({ dataPassId }); - expect(qcSummary).to.be.eql({ - 106: { - 1: { - badEffectiveRunCoverage: 1, - mcReproducible: false, - missingVerificationsCount: 1, - explicitlyNotBadEffectiveRunCoverage: 0, - }, - 16: { - badEffectiveRunCoverage: 0, - explicitlyNotBadEffectiveRunCoverage: 1, - mcReproducible: false, - missingVerificationsCount: 1, - }, - }, - [runNumber]: { - [detectorId]: { - badEffectiveRunCoverage: null, - explicitlyNotBadEffectiveRunCoverage: 0, - mcReproducible: true, - missingVerificationsCount: 2, - }, - }, - }); + const firstFlagEffectivePeriods = await getEffectivePeriodsOfQcFlag(createdFlagIds[0]); + expect(firstFlagEffectivePeriods).to.lengthOf(1); + expect(firstFlagEffectivePeriods[0].from).to.equal(secondFlagTo); + expect(firstFlagEffectivePeriods[0].to).to.equal(null); } // 3 { - from = new Date('2024-07-01 16:00:00').getTime(); - to = null; - [qcFlag] = await qcFlagService.create( + const from = thirdFlagFrom; + const to = null; + const [qcFlag] = await qcFlagService.create( [{ flagTypeId, from, to }], { runNumber, @@ -804,26 +758,31 @@ module.exports = () => { }, relations, ); - effectivePeriods = await getEffectivePeriodsOfQcFlag(qcFlag.id); createdFlagIds.push(qcFlag.id); - expect(effectivePeriods.map(({ from, to }) => ({ from, to }))).to.be.eql([{ from, to }]); + + const newFlagEffectivePeriods = await getEffectivePeriodsOfQcFlag(qcFlag.id); + expect(newFlagEffectivePeriods).to.lengthOf(1); + expect(newFlagEffectivePeriods[0].from).to.equal(from); + expect(newFlagEffectivePeriods[0].to).to.equal(null); // Previous: first flag - effectivePeriods = await getEffectivePeriodsOfQcFlag(createdFlagIds[0]); - expect(effectivePeriods.map(({ from, to }) => ({ from, to }))).to - .have.all.deep.members([{ from: new Date('2024-07-01 12:00:00').getTime(), to: new Date('2024-07-01 16:00:00').getTime() }]); + const firstFlagEffectivePeriods = await getEffectivePeriodsOfQcFlag(createdFlagIds[0]); + expect(firstFlagEffectivePeriods).to.lengthOf(1); + expect(firstFlagEffectivePeriods[0].from).to.equal(secondFlagTo); + expect(firstFlagEffectivePeriods[0].to).to.equal(thirdFlagFrom); // Previous: second flag - effectivePeriods = await getEffectivePeriodsOfQcFlag(createdFlagIds[1]); - expect(effectivePeriods.map(({ from, to }) => ({ from, to }))).to - .have.all.deep.members([{ from: null, to: new Date('2024-07-01 12:00:00').getTime() }]); + const secondFlagEffectivePeriods = await getEffectivePeriodsOfQcFlag(createdFlagIds[1]); + expect(secondFlagEffectivePeriods).to.lengthOf(1); + expect(secondFlagEffectivePeriods[0].from).to.equal(null); + expect(secondFlagEffectivePeriods[0].to).to.equal(secondFlagTo); } // 4 { - from = new Date('2024-07-01 13:00:00').getTime(); - to = new Date('2024-07-01 15:00:00').getTime(); - [qcFlag] = await qcFlagService.create( + const from = fourthFlagFrom; + const to = fourthFlagTo; + const [qcFlag] = await qcFlagService.create( [{ flagTypeId, from, to }], { runNumber, @@ -832,28 +791,6 @@ module.exports = () => { }, relations, ); - effectivePeriods = await getEffectivePeriodsOfQcFlag(qcFlag.id); - createdFlagIds.push(qcFlag.id); - expect(effectivePeriods.map(({ from, to }) => ({ from, to }))).to.be.eql([{ from, to }]); - - // Previous: first flag - effectivePeriods = await getEffectivePeriodsOfQcFlag(createdFlagIds[0]); - expect(effectivePeriods.map(({ from, to }) => ({ from, to }))).to.have.all.deep.members([ - { from: new Date('2024-07-01 12:00:00').getTime(), to: new Date('2024-07-01 13:00:00').getTime() }, - { from: new Date('2024-07-01 15:00:00').getTime(), to: new Date('2024-07-01 16:00:00').getTime() }, - ]); - - // Previous: second flag - effectivePeriods = await getEffectivePeriodsOfQcFlag(createdFlagIds[1]); - expect(effectivePeriods.map(({ from, to }) => ({ from, to }))).to.have.all.deep.members([ - { from: null, to: new Date('2024-07-01 12:00:00').getTime() }, - ]); - - // Previous: third flag - effectivePeriods = await getEffectivePeriodsOfQcFlag(createdFlagIds[2]); - expect(effectivePeriods.map(({ from, to }) => ({ from, to }))).to.have.all.deep.members([ - { from: new Date('2024-07-01 16:00:00').getTime(), to: null }, - ]); } }); }); @@ -1137,11 +1074,9 @@ module.exports = () => { const [{ id, from, to, comment, flagTypeId, runNumber, dplDetectorId: detectorId, createdBy: { externalId: externalUserId } }] = await qcFlagService.create([qcFlagCreationParameters], scope, relations); - const { startTime, endTime } = await RunRepository.findOne({ where: { runNumber } }); - expect({ from, to, comment, flagTypeId, runNumber, detectorId, externalUserId }).to.be.eql({ - from: startTime, - to: endTime, + from: null, + to: null, comment: qcFlagCreationParameters.comment, flagTypeId: qcFlagCreationParameters.flagTypeId, runNumber: scope.runNumber, From 1a10e318491a0531cb82ec15521db524ffa6b358 Mon Sep 17 00:00:00 2001 From: martinboulais <31805063+martinboulais@users.noreply.github.com> Date: Fri, 7 Mar 2025 16:15:16 +0100 Subject: [PATCH 04/18] WIP on tests --- lib/server/services/qualityControlFlag/QcFlagSummaryService.js | 2 -- lib/usecases/run/GetAllRunsUseCase.js | 2 ++ test/lib/usecases/index.js | 2 ++ 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/server/services/qualityControlFlag/QcFlagSummaryService.js b/lib/server/services/qualityControlFlag/QcFlagSummaryService.js index 6d2d847ef9..a734a9c161 100644 --- a/lib/server/services/qualityControlFlag/QcFlagSummaryService.js +++ b/lib/server/services/qualityControlFlag/QcFlagSummaryService.js @@ -14,7 +14,6 @@ import { BadParameterError } from '../../errors/BadParameterError.js'; import { dataSource } from '../../../database/DataSource.js'; import { Op } from 'sequelize'; import { QcFlagRepository } from '../../../database/repositories/index.js'; -import { qcFlagEffectivePeriodAdapter } from '../../../database/adapters/index.js'; /** * @typedef RunQcSummary @@ -167,7 +166,6 @@ export class QcFlagSummaryService { const effectiveRunCoverage = (effectiveRunCoverageString ?? null) !== null ? parseFloat(effectiveRunCoverageString) : null; - console.log(effectiveRunCoverageString); return { runNumber: summaryDb.runNumber, diff --git a/lib/usecases/run/GetAllRunsUseCase.js b/lib/usecases/run/GetAllRunsUseCase.js index 4fbaf0151a..2cd7406d49 100644 --- a/lib/usecases/run/GetAllRunsUseCase.js +++ b/lib/usecases/run/GetAllRunsUseCase.js @@ -344,8 +344,10 @@ class GetAllRunsUseCase { const { mcReproducibleAsNotBad = false } = gaq; const [dataPassId] = dataPassIds; const gaqSummary = await gaqService.getSummary(dataPassId, { mcReproducibleAsNotBad }); + console.log(gaqSummary, mcReproducibleAsNotBad); for (const [operator, value] of Object.entries(gaq.notBadFraction)) { + console.log(operator, value); const runNumbers = Object.entries(gaqSummary) .filter(([_, { badEffectiveRunCoverage }]) => { switch (operator) { diff --git a/test/lib/usecases/index.js b/test/lib/usecases/index.js index e059378670..b5fc953604 100644 --- a/test/lib/usecases/index.js +++ b/test/lib/usecases/index.js @@ -20,8 +20,10 @@ const RunTypeSuite = require('./runType/index.js'); const FlpSuite = require('./flp/index.js'); const ServerSuite = require('./server/index.js'); const TagSuite = require('./tag/index.js'); +const { resetDatabaseContent } = require('../../utilities/resetDatabaseContent.js'); module.exports = () => { + before(resetDatabaseContent); describe('Status use-case', StatusSuite); describe('Log use-case', LogSuite); describe('LhcFill use-case', LhcFillSuite); From d85307ef0ea740e36eb104089cac95f4e28ab260 Mon Sep 17 00:00:00 2001 From: Martin Boulais <31805063+martinboulais@users.noreply.github.com> Date: Tue, 11 Mar 2025 15:17:43 +0100 Subject: [PATCH 05/18] Finalize service test and fix linter --- lib/usecases/run/GetAllRunsUseCase.js | 2 - .../qualityControlFlag/QcFlagService.test.js | 60 ++++++++++++++++++- 2 files changed, 59 insertions(+), 3 deletions(-) diff --git a/lib/usecases/run/GetAllRunsUseCase.js b/lib/usecases/run/GetAllRunsUseCase.js index 2cd7406d49..4fbaf0151a 100644 --- a/lib/usecases/run/GetAllRunsUseCase.js +++ b/lib/usecases/run/GetAllRunsUseCase.js @@ -344,10 +344,8 @@ class GetAllRunsUseCase { const { mcReproducibleAsNotBad = false } = gaq; const [dataPassId] = dataPassIds; const gaqSummary = await gaqService.getSummary(dataPassId, { mcReproducibleAsNotBad }); - console.log(gaqSummary, mcReproducibleAsNotBad); for (const [operator, value] of Object.entries(gaq.notBadFraction)) { - console.log(operator, value); const runNumbers = Object.entries(gaqSummary) .filter(([_, { badEffectiveRunCoverage }]) => { switch (operator) { diff --git a/test/lib/server/services/qualityControlFlag/QcFlagService.test.js b/test/lib/server/services/qualityControlFlag/QcFlagService.test.js index f9c8d0e6d8..f2e6cf2496 100644 --- a/test/lib/server/services/qualityControlFlag/QcFlagService.test.js +++ b/test/lib/server/services/qualityControlFlag/QcFlagService.test.js @@ -697,8 +697,21 @@ module.exports = () => { const fourthFlagFrom = new Date('2024-07-01 13:00:00').getTime(); const fourthFlagTo = new Date('2024-07-01 15:00:00').getTime(); + /* + * --- 12 - 13 - 14 - 15 - 16 - 17 --> hours + * ( ) First flag + * ( ]----------------------------- Second flag + * ------------------------[ ) Third flag + * ---------[ ]-------------- Fourth flag + */ + // 1 { + /* + * --- 12 - 13 - 14 - 15 - 16 - 17 --> hours + * ( ) First flag effective period + */ + const from = null; const to = null; const [qcFlag] = await qcFlagService.create( @@ -720,6 +733,11 @@ module.exports = () => { // 2 { + /* + * --- 12 - 13 - 14 - 15 - 16 - 17 --> hours + * -----[ ) First flag effective period + * ( ]----------------------------- Second flag effective period + */ const from = null; const to = secondFlagTo; const [qcFlag] = await qcFlagService.create( @@ -741,12 +759,19 @@ module.exports = () => { // Previous: first flag const firstFlagEffectivePeriods = await getEffectivePeriodsOfQcFlag(createdFlagIds[0]); expect(firstFlagEffectivePeriods).to.lengthOf(1); - expect(firstFlagEffectivePeriods[0].from).to.equal(secondFlagTo); + expect(firstFlagEffectivePeriods[0].from).to.equal(to); expect(firstFlagEffectivePeriods[0].to).to.equal(null); } // 3 { + /* + * --- 12 - 13 - 14 - 15 - 16 - 17 --> hours + * -----[ ]---------- First flag effective period + * ( ]----------------------------- Second flag effective period + * ------------------------[ ) Third flag effective period + */ + const from = thirdFlagFrom; const to = null; const [qcFlag] = await qcFlagService.create( @@ -780,6 +805,14 @@ module.exports = () => { // 4 { + /* + * --- 12 - 13 - 14 - 15 - 16 - 17 --> hours + * -----[ ]----------[ ]---------- First flag effective periods + * ( ]----------------------------- Second flag effective period + * ------------------------[ ) Third flag effective period + * ---------[ ]-------------- Fourth flag effective period + */ + const from = fourthFlagFrom; const to = fourthFlagTo; const [qcFlag] = await qcFlagService.create( @@ -791,6 +824,31 @@ module.exports = () => { }, relations, ); + + const newFlagEffectivePeriods = await getEffectivePeriodsOfQcFlag(qcFlag.id); + expect(newFlagEffectivePeriods).to.lengthOf(1); + expect(newFlagEffectivePeriods[0].from).to.equal(from); + expect(newFlagEffectivePeriods[0].to).to.equal(to); + + // Previous: first flag + const firstFlagEffectivePeriods = await getEffectivePeriodsOfQcFlag(createdFlagIds[0]); + expect(firstFlagEffectivePeriods).to.lengthOf(2); + expect(firstFlagEffectivePeriods[0].from).to.equal(secondFlagTo); + expect(firstFlagEffectivePeriods[0].to).to.equal(from); + expect(firstFlagEffectivePeriods[1].from).to.equal(to); + expect(firstFlagEffectivePeriods[1].to).to.equal(thirdFlagFrom); + + // Previous: second flag + const secondFlagEffectivePeriods = await getEffectivePeriodsOfQcFlag(createdFlagIds[1]); + expect(secondFlagEffectivePeriods).to.lengthOf(1); + expect(secondFlagEffectivePeriods[0].from).to.equal(null); + expect(secondFlagEffectivePeriods[0].to).to.equal(secondFlagTo); + + // Previous: third flag + const thirdFlagEffectivePeriods = await getEffectivePeriodsOfQcFlag(createdFlagIds[2]); + expect(thirdFlagEffectivePeriods).to.lengthOf(1); + expect(thirdFlagEffectivePeriods[0].from).to.equal(thirdFlagFrom); + expect(thirdFlagEffectivePeriods[0].to).to.equal(null); } }); }); From 4dc6916a0e4609dd2a30e70c32e0ea7ca2605a1c Mon Sep 17 00:00:00 2001 From: Martin Boulais <31805063+martinboulais@users.noreply.github.com> Date: Tue, 11 Mar 2025 16:26:13 +0100 Subject: [PATCH 06/18] Fix more tests --- test/public/qcFlags/gaqOverview.test.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/public/qcFlags/gaqOverview.test.js b/test/public/qcFlags/gaqOverview.test.js index 2991d39b79..d907993054 100644 --- a/test/public/qcFlags/gaqOverview.test.js +++ b/test/public/qcFlags/gaqOverview.test.js @@ -112,7 +112,7 @@ module.exports = () => { expect(await getTableContent(page)).to.have.all.deep.ordered.members([ [ 'good', - '08/08/2019\n13:00:00', + 'Since run start', '08/08/2019\n22:43:20', '', 'Good', @@ -155,7 +155,7 @@ module.exports = () => { [ 'good', '09/08/2019\n09:50:00', - '09/08/2019\n14:00:00', + 'Until run end', '', 'Good', ], From ff6cd20059043da6a693e5304cefd7dd6143aa8a Mon Sep 17 00:00:00 2001 From: Martin Boulais <31805063+martinboulais@users.noreply.github.com> Date: Thu, 13 Mar 2025 09:24:37 +0100 Subject: [PATCH 07/18] Minor refactoring and documentation rewrite --- lib/server/controllers/qcFlag.controller.js | 2 +- .../qualityControlFlag/QcFlagService.js | 8 ++++---- .../qualityControlFlag/QcFlagSummaryService.js | 2 +- .../qualityControlFlag/QcSummaryUnit.js | 18 ------------------ .../qualityControlFlag/QcFlagService.test.js | 16 ++++++++-------- 5 files changed, 14 insertions(+), 32 deletions(-) delete mode 100644 lib/server/services/qualityControlFlag/QcSummaryUnit.js diff --git a/lib/server/controllers/qcFlag.controller.js b/lib/server/controllers/qcFlag.controller.js index 5ec00816f5..dd12563950 100644 --- a/lib/server/controllers/qcFlag.controller.js +++ b/lib/server/controllers/qcFlag.controller.js @@ -326,7 +326,7 @@ const getQcFlagsSummaryHandler = async (req, res) => { mcReproducibleAsNotBad = false, } = validatedDTO.query; - const data = await qcFlagSummaryService.getQcFlagsSummary({ dataPassId, simulationPassId, lhcPeriodId }, { mcReproducibleAsNotBad }); + const data = await qcFlagSummaryService.getSummary({ dataPassId, simulationPassId, lhcPeriodId }, { mcReproducibleAsNotBad }); res.json({ data }); } catch (error) { updateExpressResponseFromNativeError(res, error); diff --git a/lib/server/services/qualityControlFlag/QcFlagService.js b/lib/server/services/qualityControlFlag/QcFlagService.js index 6491007a40..8e9c5108f1 100644 --- a/lib/server/services/qualityControlFlag/QcFlagService.js +++ b/lib/server/services/qualityControlFlag/QcFlagService.js @@ -570,18 +570,18 @@ class QcFlagService { * ------------------------------[ erase window ]----------------------------> time * [ period BEFORE erase window ] [ period AFTER erase window ] * - * Using this flag for example: + * When erasing time segment from the following QC flag (using the erase window from above): * ---------------------[ QC flag period ]-------------------- * - * The corresponding sub-periods will be: + * Resultant effective sub-periods of the QC flag will be: * ---------------------[ ]-------------------------------------------- => sub-period BEFORE erase window * ----------------------------------------------[ ]-------------------- => sub-period AFTER erase window * - * The following flag would have no sub-period after: + * Analogously, for the the following flag would have no sub-period after: * ----------------[ QC flag period ]---------------------------------------- * ----------------[ ]-------------------------------------------- => sub-period BEFORE erase window * - * And the following flag would have no su-period before + * And the following flag would have no sub-period before * ---------------------------------------[ QC flag period ]----------------- * ----------------------------------------------[ ]----------------- => sub-period AFTER erase window */ diff --git a/lib/server/services/qualityControlFlag/QcFlagSummaryService.js b/lib/server/services/qualityControlFlag/QcFlagSummaryService.js index a734a9c161..c2690e7f55 100644 --- a/lib/server/services/qualityControlFlag/QcFlagSummaryService.js +++ b/lib/server/services/qualityControlFlag/QcFlagSummaryService.js @@ -96,7 +96,7 @@ export class QcFlagSummaryService { * good one * @return {Promise} summary */ - async getQcFlagsSummary({ dataPassId, simulationPassId, lhcPeriodId }, { mcReproducibleAsNotBad = false } = {}) { + async getSummary({ dataPassId, simulationPassId, lhcPeriodId }, { mcReproducibleAsNotBad = false } = {}) { if (Boolean(dataPassId) + Boolean(simulationPassId) + Boolean(lhcPeriodId) > 1) { throw new BadParameterError('`dataPassId`, `simulationPassId` and `lhcPeriodId` are exclusive options'); } diff --git a/lib/server/services/qualityControlFlag/QcSummaryUnit.js b/lib/server/services/qualityControlFlag/QcSummaryUnit.js deleted file mode 100644 index 3d008a659b..0000000000 --- a/lib/server/services/qualityControlFlag/QcSummaryUnit.js +++ /dev/null @@ -1,18 +0,0 @@ -/** - * @license - * Copyright CERN and copyright holders of ALICE O2. This software is - * distributed under the terms of the GNU General Public License v3 (GPL - * Version 3), copied verbatim in the file "COPYING". - * - * See http://alice-o2.web.cern.ch/license for full licensing information. - * - * In applying this license CERN does not waive the privileges and immunities - * granted to it by virtue of its status as an Intergovernmental Organization - * or submit itself to any jurisdiction. - */ - -/** - * QC summary unit - */ -export class QcSummaryUnit { -} diff --git a/test/lib/server/services/qualityControlFlag/QcFlagService.test.js b/test/lib/server/services/qualityControlFlag/QcFlagService.test.js index f2e6cf2496..e168197e8d 100644 --- a/test/lib/server/services/qualityControlFlag/QcFlagService.test.js +++ b/test/lib/server/services/qualityControlFlag/QcFlagService.test.js @@ -159,7 +159,7 @@ module.exports = () => { describe('Get QC flags summary', () => { it('should successfully get non-empty QC flag summary for data pass', async () => { - expect(await qcFlagSummaryService.getQcFlagsSummary({ dataPassId: 1 })).to.be.eql({ + expect(await qcFlagSummaryService.getSummary({ dataPassId: 1 })).to.be.eql({ 106: { 1: { missingVerificationsCount: 3, @@ -178,7 +178,7 @@ module.exports = () => { }); it('should successfully get non-empty QC flag summary with MC.Reproducible interpreted as not-bad for data pass', async () => { - expect(await qcFlagSummaryService.getQcFlagsSummary({ dataPassId: 1 }, { mcReproducibleAsNotBad: true })).to.be.eql({ + expect(await qcFlagSummaryService.getSummary({ dataPassId: 1 }, { mcReproducibleAsNotBad: true })).to.be.eql({ 106: { 1: { missingVerificationsCount: 3, @@ -222,7 +222,7 @@ module.exports = () => { .filter(({ flag: { flagType: { bad } } }) => !bad) .reduce((coverage, { from, to }) => coverage + (to - from), 0); - expect(await qcFlagSummaryService.getQcFlagsSummary({ dataPassId })).to.be.eql({ + expect(await qcFlagSummaryService.getSummary({ dataPassId })).to.be.eql({ 1: { 1: { missingVerificationsCount: 0, @@ -239,7 +239,7 @@ module.exports = () => { // Verify flag and fetch summary one more time const relations = { user: { roles: ['admin'], externalUserId: 456 } }; await qcFlagService.verifyFlag({ flagId: 4 }, relations); - expect(await qcFlagSummaryService.getQcFlagsSummary({ dataPassId })).to.be.eql({ + expect(await qcFlagSummaryService.getSummary({ dataPassId })).to.be.eql({ 1: { 1: { missingVerificationsCount: 0, @@ -252,11 +252,11 @@ module.exports = () => { }); it('should successfully get empty QC flag summary for data pass', async () => { - expect(await qcFlagSummaryService.getQcFlagsSummary({ dataPassId: 3 })).to.be.eql({}); + expect(await qcFlagSummaryService.getSummary({ dataPassId: 3 })).to.be.eql({}); }); it('should successfully get non-empty QC flag summary for simulation pass', async () => { - expect(await qcFlagSummaryService.getQcFlagsSummary({ simulationPassId: 1 })).to.be.eql({ + expect(await qcFlagSummaryService.getSummary({ simulationPassId: 1 })).to.be.eql({ 106: { 1: { missingVerificationsCount: 1, @@ -269,11 +269,11 @@ module.exports = () => { }); it('should successfully get empty QC flag summary for simulation pass', async () => { - expect(await qcFlagSummaryService.getQcFlagsSummary({ simulationPassId: 2 })).to.be.eql({}); + expect(await qcFlagSummaryService.getSummary({ simulationPassId: 2 })).to.be.eql({}); }); it('should successfully get QC summary of synchronous QC flags for one LHC Period', async () => { - expect(await qcFlagSummaryService.getQcFlagsSummary({ lhcPeriodId: 1 })).to.be.eql({ + expect(await qcFlagSummaryService.getSummary({ lhcPeriodId: 1 })).to.be.eql({ 56: { // FT0 7: { From 8ed905250a2440084509e8e897fc2ebd934659bb Mon Sep 17 00:00:00 2001 From: Martin Boulais <31805063+martinboulais@users.noreply.github.com> Date: Thu, 13 Mar 2025 14:43:21 +0100 Subject: [PATCH 08/18] Improve naming --- .../services/qualityControlFlag/QcFlagService.js | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/lib/server/services/qualityControlFlag/QcFlagService.js b/lib/server/services/qualityControlFlag/QcFlagService.js index 8e9c5108f1..fda0c33f80 100644 --- a/lib/server/services/qualityControlFlag/QcFlagService.js +++ b/lib/server/services/qualityControlFlag/QcFlagService.js @@ -537,9 +537,8 @@ class QcFlagService { * @throws {BadParameterError} */ _prepareQcFlagPeriod(timestamps, targetRun) { - const { qcTimeStart, qcTimeEnd } = targetRun; - const runStart = qcTimeStart?.getTime(); - const runEnd = qcTimeEnd?.getTime(); + const qcTimeStart = targetRun.qcTimeStart?.getTime(); + const qcTimeEnd = targetRun.qcTimeEnd?.getTime(); const { from, to } = timestamps; @@ -547,10 +546,10 @@ class QcFlagService { throw new BadParameterError('Parameter "to" timestamp must be greater than "from" timestamp'); } - const isFromOutOfRange = from && (runStart && from < runStart || runEnd && runEnd <= from); - const isToOutOfRange = to && (runStart && to <= runStart || runEnd && runEnd < to); + const isFromOutOfRange = from && (qcTimeStart && from < qcTimeStart || qcTimeEnd && qcTimeEnd <= from); + const isToOutOfRange = to && (qcTimeStart && to <= qcTimeStart || qcTimeEnd && qcTimeEnd < to); if (isFromOutOfRange || isToOutOfRange) { - throw new BadParameterError(`Given QC flag period (${from}, ${to}) is out of run (${runStart}, ${runEnd}) period`); + throw new BadParameterError(`Given QC flag period (${from}, ${to}) is out of run (${qcTimeStart}, ${qcTimeEnd}) period`); } return { from, to }; From 382b3fbdf77de8dc364fd8df61b27081e1d67580 Mon Sep 17 00:00:00 2001 From: Martin Boulais <31805063+martinboulais@users.noreply.github.com> Date: Tue, 18 Mar 2025 09:46:40 +0100 Subject: [PATCH 09/18] Add tests --- lib/database/seeders/20200713103855-runs.js | 2 + .../seeders/20240112102011-data-passes.js | 2 + .../seeders/20240404100811-qc-flags.js | 80 ++++++++++++++++--- .../QcFlagsForDataPassOverviewPage.js | 2 +- .../qcFlagsChartComponent.js | 11 ++- .../qualityControlFlag/QcFlagService.test.js | 2 +- .../public/qcFlags/detailsForDataPass.test.js | 12 +++ .../qcFlags/forSimulationPassOverview.test.js | 4 +- .../qcFlags/synchronousOverview.test.js | 4 +- test/public/runs/dataPassesUtilities.js | 21 +++++ .../runs/runsPerDataPass.overview.test.js | 51 ++++-------- 11 files changed, 133 insertions(+), 58 deletions(-) create mode 100644 test/public/runs/dataPassesUtilities.js diff --git a/lib/database/seeders/20200713103855-runs.js b/lib/database/seeders/20200713103855-runs.js index 9d904287c4..e3042661ce 100644 --- a/lib/database/seeders/20200713103855-runs.js +++ b/lib/database/seeders/20200713103855-runs.js @@ -2506,6 +2506,8 @@ module.exports = { { id: 100, run_number: 100, + time_o2_end: '2019-08-08 13:00:00', + time_trg_end: '2019-08-08 14:23:45', run_type_id: 12, run_quality: 'test', n_detectors: 3, diff --git a/lib/database/seeders/20240112102011-data-passes.js b/lib/database/seeders/20240112102011-data-passes.js index 9275aded03..a0cafbf432 100644 --- a/lib/database/seeders/20240112102011-data-passes.js +++ b/lib/database/seeders/20240112102011-data-passes.js @@ -203,6 +203,7 @@ module.exports = { { data_pass_id: 1, run_number: 106, ready_for_skimming: true }, { data_pass_id: 1, run_number: 107, ready_for_skimming: false }, { data_pass_id: 1, run_number: 108, ready_for_skimming: false }, + { data_pass_id: 2, run_number: 1 }, { data_pass_id: 2, run_number: 2 }, { data_pass_id: 2, run_number: 55 }, @@ -215,6 +216,7 @@ module.exports = { { data_pass_id: 4, run_number: 49 }, { data_pass_id: 4, run_number: 54 }, { data_pass_id: 4, run_number: 56 }, + { data_pass_id: 4, run_number: 100 }, { data_pass_id: 4, run_number: 105 }, { data_pass_id: 5, run_number: 49 }, diff --git a/lib/database/seeders/20240404100811-qc-flags.js b/lib/database/seeders/20240404100811-qc-flags.js index 69ef14a2b6..e86b24258e 100644 --- a/lib/database/seeders/20240404100811-qc-flags.js +++ b/lib/database/seeders/20240404100811-qc-flags.js @@ -71,8 +71,8 @@ module.exports = { { id: 7, deleted: false, - from: '2019-08-08 13:00:00', - to: '2019-08-09 14:00:00', + from: null, + to: null, comment: 'Some qc comment 7', origin: 'qc_async/ZDC/AverageClusterSize', @@ -122,7 +122,7 @@ module.exports = { id: 6, deleted: false, from: '2019-08-09 08:50:00', - to: '2019-08-09 14:00:00', + to: null, comment: 'Some qc comment 4', // Associations @@ -134,13 +134,45 @@ module.exports = { updated_at: '2024-02-13 11:58:20', }, + { + id: 8, + deleted: false, + from: null, + to: null, + comment: 'Some qc comment 5', + + // Associations + created_by_id: 2, + flag_type_id: 13, // Bad + run_number: 100, + detector_id: 1, // CPV + created_at: '2024-02-13 11:58:20', + updated_at: '2024-02-13 11:58:20', + }, + + { + id: 9, + deleted: false, + from: null, + to: null, + comment: 'Some qc comment 6', + + // Associations + created_by_id: 2, + flag_type_id: 3, // Bad + run_number: 105, + detector_id: 1, // CPV + created_at: '2024-02-13 11:58:20', + updated_at: '2024-02-13 11:58:20', + }, + /** Synchronous */ // Run : 56, FT0 { id: 100, deleted: false, - from: '2019-08-08 20:00:00', + from: null, to: '2019-08-08 20:50:00', comment: 'first part good', @@ -156,7 +188,7 @@ module.exports = { id: 101, deleted: false, from: '2019-08-08 20:50:00', - to: '2019-08-08 21:00:00', + to: null, comment: 'second part bad', run_number: 56, @@ -172,8 +204,8 @@ module.exports = { { id: 102, deleted: false, - from: '2019-08-08 20:00:00', - to: '2019-08-08 21:00:00', + from: null, + to: null, comment: 'all good', flag_type_id: 3, // Good @@ -220,8 +252,21 @@ module.exports = { { id: 7, flag_id: 7, - from: '2019-08-08 13:00:00', - to: '2019-08-09 14:00:00', + from: null, + to: null, + }, + + { + id: 8, + flag_id: 8, + from: null, + to: null, + }, + { + id: 9, + flag_id: 9, + from: null, + to: null, }, /** Synchronous */ @@ -229,22 +274,22 @@ module.exports = { { id: 100, flag_id: 100, - from: '2019-08-08 20:00:00', + from: null, to: '2019-08-08 20:50:00', }, { id: 101, flag_id: 101, from: '2019-08-08 20:50:00', - to: '2019-08-08 21:00:00', + to: null, }, // Run : 56, ITS { id: 102, flag_id: 102, - from: '2019-08-08 20:00:00', - to: '2019-08-08 21:00:00', + from: null, + to: null, }, ], { transaction }), @@ -270,6 +315,15 @@ module.exports = { data_pass_id: 2, // LHC22b_apass2 quality_control_flag_id: 4, }, + + { + data_pass_id: 4, // LHC22a_apass2 + quality_control_flag_id: 8, + }, + { + data_pass_id: 4, // LHC22a_apass2 + quality_control_flag_id: 9, + }, ], { transaction }), queryInterface.bulkInsert('simulation_pass_quality_control_flag', [ diff --git a/lib/public/views/QcFlags/ForDataPass/QcFlagsForDataPassOverviewPage.js b/lib/public/views/QcFlags/ForDataPass/QcFlagsForDataPassOverviewPage.js index 274484d1a3..87a0af178a 100644 --- a/lib/public/views/QcFlags/ForDataPass/QcFlagsForDataPassOverviewPage.js +++ b/lib/public/views/QcFlags/ForDataPass/QcFlagsForDataPassOverviewPage.js @@ -89,7 +89,7 @@ export const QcFlagsForDataPassOverviewPage = ({ }), mergeRemoteData([remoteQcFlags, remoteRun]) .match({ - Success: ([flags, run]) => qcFlagsChartComponent(flags, run), + Success: ([flags, run]) => h('#qc-flags-chart-component', qcFlagsChartComponent(flags, run)), Failure: () => null, Loading: () => spinner({ size: 2, absolute: false }), NotAsked: () => errorAlert([{ title: 'No data', detail: 'No QC flags or run data was asked for' }]), diff --git a/lib/public/views/QcFlags/qcFlagsVisualization/qcFlagsChartComponent.js b/lib/public/views/QcFlags/qcFlagsVisualization/qcFlagsChartComponent.js index 04421157b3..314629ee32 100644 --- a/lib/public/views/QcFlags/qcFlagsVisualization/qcFlagsChartComponent.js +++ b/lib/public/views/QcFlags/qcFlagsVisualization/qcFlagsChartComponent.js @@ -15,18 +15,23 @@ import { barChartComponent } from '../../../components/common/chart/barChart/bar import { BarPattern } from '../../../components/common/chart/rendering/dataset/renderDatasetAsBars.js'; import { formatTimestamp } from '../../../utilities/formatting/formatTimestamp.js'; import { ChartColors } from '../../Statistics/chartColors.js'; +import { h } from '/js/src/index.js'; /** - * Create QC flags visualiation as horizontall bar chart + * Create QC flags visualization as horizontal bar chart * * @param {QualityControlFlag[]} flags flags order by createdAt in descending manner * @param {Run} run the run associated with given flags * @return {Component} the QC flags visualization */ export const qcFlagsChartComponent = (flags, run) => { + if ((run.qcTimeStart ?? null) === null || (run.qcTimeEnd ?? null) === null) { + return h('h4.pv3', 'Missing run start or end, no QC flags chart'); + } + /** - * Get bars' data for effective and ineffectie periods of given QC flag - * Bars corresponging with effective periods have flagType.color + * Get bars' data for effective and ineffective periods of given QC flag + * Bars corresponding with effective periods have flagType.color * For ineffective periods bars are additionally marked with stripes pattern * @param {QcFlag} flag a QC flag * @return {Bar[]} bars data diff --git a/test/lib/server/services/qualityControlFlag/QcFlagService.test.js b/test/lib/server/services/qualityControlFlag/QcFlagService.test.js index e168197e8d..1b0bf3781a 100644 --- a/test/lib/server/services/qualityControlFlag/QcFlagService.test.js +++ b/test/lib/server/services/qualityControlFlag/QcFlagService.test.js @@ -1362,7 +1362,7 @@ module.exports = () => { }, }, }, - })).to.lengthOf(8); // 9 from seeders, then 1 deleted => 8 + })).to.lengthOf(10); // 11 from seeders, then 1 deleted => 10 expect((await QcFlagRepository.findAll({ include: { diff --git a/test/public/qcFlags/detailsForDataPass.test.js b/test/public/qcFlags/detailsForDataPass.test.js index aa0571f1df..887fd02e06 100644 --- a/test/public/qcFlags/detailsForDataPass.test.js +++ b/test/public/qcFlags/detailsForDataPass.test.js @@ -27,6 +27,7 @@ const { waitForTableLength, } = require('../defaults.js'); const { resetDatabaseContent } = require('../../utilities/resetDatabaseContent.js'); +const { navigateToRunsPerDataPass } = require('../runs/dataPassesUtilities.js'); const { expect } = chai; @@ -163,4 +164,15 @@ module.exports = () => { await expectInnerText(page, '#qc-flag-details-verified', 'Verified:\nYes'); await page.waitForSelector('#delete:disabled'); }); + + it('should successfully display a message instead of QC flags chart when run is missing boundaries', async () => { + await navigateToRunsPerDataPass(page, 1, 4, 5); + await waitForNavigation(page, () => pressElement(page, '#row105-CPV-text a')); + await expectInnerText(page, '#qc-flags-chart-component', 'Missing run start or end, no QC flags chart'); + + await page.goBack(); + + await waitForNavigation(page, () => pressElement(page, '#row100-CPV-text a')); + await expectInnerText(page, '#qc-flags-chart-component', 'Missing run start or end, no QC flags chart'); + }); }; diff --git a/test/public/qcFlags/forSimulationPassOverview.test.js b/test/public/qcFlags/forSimulationPassOverview.test.js index 1160bd229a..9012ce7df6 100644 --- a/test/public/qcFlags/forSimulationPassOverview.test.js +++ b/test/public/qcFlags/forSimulationPassOverview.test.js @@ -100,8 +100,8 @@ module.exports = () => { const tableDataValidators = { flagType: (flagType) => flagType && flagType !== '-', createdBy: (userName) => userName && userName !== '-', - from: (timestamp) => timestamp === 'Whole run coverage' || validateDate(timestamp), - to: (timestamp) => timestamp === 'Whole run coverage' || validateDate(timestamp), + from: (timestamp) => timestamp === 'Whole run coverage' || timestamp === 'From run start' || validateDate(timestamp), + to: (timestamp) => timestamp === 'Whole run coverage' || timestamp === 'Until run end' || validateDate(timestamp), createdAt: validateDate, updatedAt: validateDate, }; diff --git a/test/public/qcFlags/synchronousOverview.test.js b/test/public/qcFlags/synchronousOverview.test.js index 5b366c2a8a..e72c4eca91 100644 --- a/test/public/qcFlags/synchronousOverview.test.js +++ b/test/public/qcFlags/synchronousOverview.test.js @@ -63,8 +63,8 @@ module.exports = () => { const tableDataValidators = { flagType: (flagType) => flagType && flagType !== '-', createdBy: (userName) => userName && userName !== '-', - from: (timestamp) => timestamp === 'Whole run coverage' || validateDate(timestamp), - to: (timestamp) => timestamp === 'Whole run coverage' || validateDate(timestamp), + from: (timestamp) => timestamp === 'Whole run coverage' || timestamp === 'Since run start' || validateDate(timestamp), + to: (timestamp) => timestamp === 'Whole run coverage' || timestamp === 'Until run end' || validateDate(timestamp), createdAt: validateDate, updatedAt: validateDate, }; diff --git a/test/public/runs/dataPassesUtilities.js b/test/public/runs/dataPassesUtilities.js new file mode 100644 index 0000000000..2b97b2f0d4 --- /dev/null +++ b/test/public/runs/dataPassesUtilities.js @@ -0,0 +1,21 @@ +const { waitForNavigation, pressElement, getInnerText, expectUrlParams, waitForTableLength } = require('../defaults.js'); + +/** + * Navigate to Runs per Data Pass page + * + * @param {Puppeteer.Page} page page + * @param {number} lhcPeriodId id of lhc period on LHC Period overview page + * @param {number} dataPassId id of data pass on Data Passes per LHC Period page + * @param {number} expectedRowsCount expected number of rows on runs per data pass page + * @return {Promise} promise + */ +exports.navigateToRunsPerDataPass = async (page, lhcPeriodId, dataPassId, expectedRowsCount) => { + await waitForNavigation(page, () => pressElement(page, 'a#lhc-period-overview', true)); + const pdpBeamType = await getInnerText(await page.waitForSelector(`#row${lhcPeriodId}-beamTypes`)); + await waitForNavigation(page, () => pressElement(page, `#row${lhcPeriodId}-associatedDataPasses a`, true)); + expectUrlParams(page, { page: 'data-passes-per-lhc-period-overview', lhcPeriodId }); + await page.waitForSelector('th#description'); + await waitForNavigation(page, () => pressElement(page, `#row${dataPassId}-associatedRuns a`, true)); + expectUrlParams(page, { page: 'runs-per-data-pass', dataPassId, pdpBeamType }); + await waitForTableLength(page, expectedRowsCount); +}; diff --git a/test/public/runs/runsPerDataPass.overview.test.js b/test/public/runs/runsPerDataPass.overview.test.js index 428d703993..98a0b6dd3b 100644 --- a/test/public/runs/runsPerDataPass.overview.test.js +++ b/test/public/runs/runsPerDataPass.overview.test.js @@ -40,6 +40,7 @@ const { const { resetDatabaseContent } = require('../../utilities/resetDatabaseContent.js'); const DataPassRepository = require('../../../lib/database/repositories/DataPassRepository.js'); const { BkpRoles } = require('../../../lib/domain/enums/BkpRoles.js'); +const { navigateToRunsPerDataPass } = require('./dataPassesUtilities.js'); const { expect } = chai; @@ -61,28 +62,6 @@ const DETECTORS = [ 'ZDC', ]; -/** - * Navigate to Runs per Data Pass page - * - * @param {Puppeteer.page} page page - * @param {number} params.lhcPeriodId id of lhc period on LHC Period overview page - * @param {number} params.dataPassId id of data pass on Data Passes per LHC Period page - * @param {number} [options.epectedRowsCount] expected number of rows on runs per data pass page - * @return {Promise} promise - */ -const navigateToRunsPerDataPass = async (page, { lhcPeriodId, dataPassId }, { epectedRowsCount } = {}) => { - await waitForNavigation(page, () => pressElement(page, 'a#lhc-period-overview', true)); - const pdpBeamType = await getInnerText(await page.waitForSelector(`#row${lhcPeriodId}-beamTypes`)); - await waitForNavigation(page, () => pressElement(page, `#row${lhcPeriodId}-associatedDataPasses a`, true)); - expectUrlParams(page, { page: 'data-passes-per-lhc-period-overview', lhcPeriodId }); - await page.waitForSelector('th#description'); - await waitForNavigation(page, () => pressElement(page, `#row${dataPassId}-associatedRuns a`, true)); - expectUrlParams(page, { page: 'runs-per-data-pass', dataPassId, pdpBeamType }); - if (epectedRowsCount) { - await waitForTableLength(page, epectedRowsCount); - } -}; - module.exports = () => { let page; let browser; @@ -134,7 +113,7 @@ module.exports = () => { ])), }; - await navigateToRunsPerDataPass(page, { lhcPeriodId: 1, dataPassId: 3 }, { epectedRowsCount: 4 }); + await navigateToRunsPerDataPass(page, 1, 3, 4); // Expectations of header texts being of a certain datatype let tableDataValidators = { ...commonColumnsValidators, @@ -146,7 +125,7 @@ module.exports = () => { await validateTableData(page, new Map(Object.entries(tableDataValidators))); - await navigateToRunsPerDataPass(page, { lhcPeriodId: 2, dataPassId: 1 }, { epectedRowsCount: 3 }); + await navigateToRunsPerDataPass(page, 2, 1, 3); // Expectations of header texts being of a certain datatype tableDataValidators = { muInelasticInteractionRate: (value) => value === '-' || !isNaN(Number(value.replace(/,/g, ''))), @@ -205,7 +184,7 @@ module.exports = () => { }); it('successfully switch to raw timestamp display', async () => { - await navigateToRunsPerDataPass(page, { lhcPeriodId: 1, dataPassId: 3 }, { epectedRowsCount: 4 }); + await navigateToRunsPerDataPass(page, 1, 3, 4); await expectInnerText(page, '#row56 td:nth-child(3)', '08/08/2019\n20:00:00'); await expectInnerText(page, '#row56 td:nth-child(4)', '08/08/2019\n21:00:00'); @@ -282,7 +261,7 @@ module.exports = () => { }); it('should successfully export runs', async () => { - await navigateToRunsPerDataPass(page, { lhcPeriodId: 1, dataPassId: 3 }, { epectedRowsCount: 4 }); + await navigateToRunsPerDataPass(page, 1, 3, 4); const targetFileName = 'runs.json'; @@ -316,7 +295,7 @@ module.exports = () => { // Filters it('should successfully apply runNumber filter', async () => { - await navigateToRunsPerDataPass(page, { lhcPeriodId: 2, dataPassId: 1 }, { epectedRowsCount: 3 }); + await navigateToRunsPerDataPass(page, 2, 1, 3); await pressElement(page, '#openFilterToggle'); @@ -330,7 +309,7 @@ module.exports = () => { }); it('should successfully apply detectors filter', async () => { - await navigateToRunsPerDataPass(page, { lhcPeriodId: 2, dataPassId: 2 }, { epectedRowsCount: 3 }); + await navigateToRunsPerDataPass(page, 2, 2, 3); await pressElement(page, '#openFilterToggle'); @@ -343,7 +322,7 @@ module.exports = () => { }); it('should successfully apply tags filter', async () => { - await navigateToRunsPerDataPass(page, { lhcPeriodId: 2, dataPassId: 1 }, { epectedRowsCount: 3 }); + await navigateToRunsPerDataPass(page, 2, 1, 3); await pressElement(page, '#openFilterToggle'); await pressElement(page, '.tags-filter .dropdown-trigger'); @@ -358,7 +337,7 @@ module.exports = () => { }); it('should successfully apply duration filter', async () => { - await navigateToRunsPerDataPass(page, { lhcPeriodId: 2, dataPassId: 2 }, { epectedRowsCount: 3 }); + await navigateToRunsPerDataPass(page, 2, 2, 3); await pressElement(page, '#openFilterToggle'); await page.select('.runDuration-filter select', '>='); @@ -378,7 +357,7 @@ module.exports = () => { }); it('should successfully apply alice currents filters', async () => { - await navigateToRunsPerDataPass(page, { lhcPeriodId: 1, dataPassId: 3 }, { epectedRowsCount: 4 }); + await navigateToRunsPerDataPass(page, 1, 3, 4); await pressElement(page, '#openFilterToggle'); const popoverSelector = await getPopoverSelector(await page.waitForSelector('.aliceL3AndDipoleCurrent-filter .popover-trigger')); @@ -413,7 +392,7 @@ module.exports = () => { } it('should successfully apply gaqNotBadFraction filters', async () => { - await navigateToRunsPerDataPass(page, { lhcPeriodId: 2, dataPassId: 1 }, { epectedRowsCount: 3 }); + await navigateToRunsPerDataPass(page, 2, 1, 3); await pressElement(page, '#openFilterToggle', true); @@ -431,7 +410,7 @@ module.exports = () => { }); it('should successfully apply muInelasticInteractionRate filters', async () => { - await navigateToRunsPerDataPass(page, { lhcPeriodId: 2, dataPassId: 1 }, { epectedRowsCount: 3 }); + await navigateToRunsPerDataPass(page, 2, 1, 3); await pressElement(page, '#openFilterToggle'); await page.waitForSelector('#muInelasticInteractionRate-operand'); @@ -446,7 +425,7 @@ module.exports = () => { it('should successfully mark as skimmable', async () => { await expectInnerText(page, '#skimmableControl .badge', 'Skimmable'); await DataPassRepository.updateAll({ skimmingStage: null }, { where: { id: 1 } }); - await navigateToRunsPerDataPass(page, { lhcPeriodId: 2, dataPassId: 1 }, { epectedRowsCount: 3 }); + await navigateToRunsPerDataPass(page, 2, 1, 3); await expectInnerText(page, '#skimmableControl button', 'Mark as skimmable'); setConfirmationDialogToBeAccepted(page); await pressElement(page, '#skimmableControl button', true); @@ -455,7 +434,7 @@ module.exports = () => { }); it('should display bad runs marked out', async () => { - await navigateToRunsPerDataPass(page, { lhcPeriodId: 2, dataPassId: 2 }, { epectedRowsCount: 3 }); + await navigateToRunsPerDataPass(page, 2, 2, 3); await page.waitForSelector('tr#row2.danger'); await page.waitForSelector('tr#row2 .column-CPV .popover-trigger svg'); @@ -465,7 +444,7 @@ module.exports = () => { }); it('should successfully change ready_for_skimming status', async () => { - await navigateToRunsPerDataPass(page, { lhcPeriodId: 2, dataPassId: 1 }, { epectedRowsCount: 3 }); + await navigateToRunsPerDataPass(page, 2, 1, 3); await expectColumnValues(page, 'readyForSkimming', ['not ready', 'not ready', 'ready']); await pressElement(page, '#row108-readyForSkimming input', true); await expectInnerText(page, '#row108-readyForSkimming', 'ready'); From 5343665fb785051da0a98602bfec41b42aea0648 Mon Sep 17 00:00:00 2001 From: Martin Boulais <31805063+martinboulais@users.noreply.github.com> Date: Tue, 18 Mar 2025 10:21:53 +0100 Subject: [PATCH 10/18] Try to fix balloon failing test --- lib/public/components/common/popover/overflowBalloon.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/public/components/common/popover/overflowBalloon.js b/lib/public/components/common/popover/overflowBalloon.js index 451068d965..ba8578d6a2 100644 --- a/lib/public/components/common/popover/overflowBalloon.js +++ b/lib/public/components/common/popover/overflowBalloon.js @@ -52,14 +52,14 @@ export const overflowBalloon = (content, options = null) => { if (previousTriggerNode) { const node = stretch ? previousTriggerNode.parentNode : previousTriggerNode; - node.removeEventListener('mouseenter', this.showPopover); + node.removeEventListener('mouseover', this.showPopover); node.removeEventListener('mouseleave', this.hidePopover); } if (newTriggerNode) { const node = stretch ? newTriggerNode.parentNode : newTriggerNode; - node.addEventListener('mouseenter', this.showPopover); + node.addEventListener('mouseover', this.showPopover); node.addEventListener('mouseleave', this.hidePopover); } }, From 8b69799668bf1de8c520338454bea6a827391443 Mon Sep 17 00:00:00 2001 From: Martin Boulais <31805063+martinboulais@users.noreply.github.com> Date: Tue, 18 Mar 2025 10:41:02 +0100 Subject: [PATCH 11/18] Fix linter --- test/public/runs/runsPerDataPass.overview.test.js | 1 - 1 file changed, 1 deletion(-) diff --git a/test/public/runs/runsPerDataPass.overview.test.js b/test/public/runs/runsPerDataPass.overview.test.js index 98a0b6dd3b..1f512f3880 100644 --- a/test/public/runs/runsPerDataPass.overview.test.js +++ b/test/public/runs/runsPerDataPass.overview.test.js @@ -31,7 +31,6 @@ const { getPopoverContent, getPopoverSelector, getInnerText, - expectUrlParams, getPopoverInnerText, testTableSortingByColumn, setConfirmationDialogToBeAccepted, From c6273709fccca5f18cc03d46fed794b5c6ff1e0e Mon Sep 17 00:00:00 2001 From: Martin Boulais <31805063+martinboulais@users.noreply.github.com> Date: Wed, 19 Mar 2025 08:09:10 +0100 Subject: [PATCH 12/18] Another attempt at fixing balloon tests --- test/public/defaults.js | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/test/public/defaults.js b/test/public/defaults.js index 2c8c29a38b..7e5d0c4e89 100644 --- a/test/public/defaults.js +++ b/test/public/defaults.js @@ -561,10 +561,17 @@ module.exports.getPopoverInnerText = getPopoverInnerText; * @returns {Promise} resolve once balloon is validated */ module.exports.checkColumnBalloon = async (page, rowIndex, columnIndex) => { - const popoverTrigger = await page.waitForSelector(`tbody tr:nth-of-type(${rowIndex}) td:nth-of-type(${columnIndex}) .popover-trigger`); + const popoverTriggerSelector = `tbody tr:nth-of-type(${rowIndex}) td:nth-of-type(${columnIndex}) .popover-trigger`; + const popoverTrigger = await page.waitForSelector(popoverTriggerSelector); + const triggerContent = await popoverTrigger.evaluate((element) => element.querySelector('.w-wrapped').innerText); - await page.hover(`tbody tr:nth-of-type(${rowIndex}) td:nth-of-type(${columnIndex}) .popover-trigger`); + // Puppeteer hover function sometimes do not trigger the mouseover, manually trigger it + await page.evaluate((popoverTriggerSelector) => { + const element = document.querySelector(popoverTriggerSelector); + element.dispatchEvent(new Event('mouseover', { bubbles: true })); + }, popoverTriggerSelector); + const popoverSelector = await this.getPopoverSelector(popoverTrigger); await this.expectInnerTextTo( From 1db90ea1308358442b6c8446a6ac1985ec9fd10a Mon Sep 17 00:00:00 2001 From: Martin Boulais <31805063+martinboulais@users.noreply.github.com> Date: Fri, 21 Mar 2025 10:55:48 +0100 Subject: [PATCH 13/18] Fix imports --- .../services/qualityControlFlag/GaqService.js | 16 +++++++++------- .../qualityControlFlag/QcFlagSummaryService.js | 15 +++++++++------ 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/lib/server/services/qualityControlFlag/GaqService.js b/lib/server/services/qualityControlFlag/GaqService.js index 349ed0388a..ad0ef86b8c 100644 --- a/lib/server/services/qualityControlFlag/GaqService.js +++ b/lib/server/services/qualityControlFlag/GaqService.js @@ -10,11 +10,6 @@ * granted to it by virtue of its status as an Intergovernmental Organization * or submit itself to any jurisdiction. */ -import { getOneDataPassOrFail } from '../dataPasses/getOneDataPassOrFail.js'; -import { Op } from 'sequelize'; -import { qcFlagAdapter } from '../../../database/adapters/index.js'; -import { QcFlagRepository } from '../../../database/repositories/index.js'; -import { QcFlagSummaryService } from './QcFlagSummaryService.js'; /** * @typedef GaqFlags @@ -32,6 +27,11 @@ import { QcFlagSummaryService } from './QcFlagSummaryService.js'; * @property {boolean} mcReproducible - states whether some aggregation of QC flags is Limited Acceptance MC Reproducible */ +const { getOneDataPassOrFail } = require('../dataPasses/getOneDataPassOrFail.js'); +const { QcFlagRepository } = require('../../../database/repositories/index.js'); +const { QcFlagSummaryService } = require('./QcFlagSummaryService.js'); +const { qcFlagAdapter } = require('../../../database/adapters/index.js'); + /** * @typedef GaqSummary aggregated global quality summaries for given data pass * @type {Object} runNumber to RunGaqSummary mapping @@ -47,7 +47,7 @@ const QC_SUMMARY_PROPERTIES = { /** * Globally aggregated quality (QC flags aggregated for a predefined list of detectors per runs) service */ -export class GaqService { +class GaqService { /** * Get GAQ summary * @@ -131,4 +131,6 @@ export class GaqService { } } -export const gaqService = new GaqService(); +exports.GaqService = GaqService; + +exports.gaqService = new GaqService(); diff --git a/lib/server/services/qualityControlFlag/QcFlagSummaryService.js b/lib/server/services/qualityControlFlag/QcFlagSummaryService.js index c2690e7f55..caa4c96043 100644 --- a/lib/server/services/qualityControlFlag/QcFlagSummaryService.js +++ b/lib/server/services/qualityControlFlag/QcFlagSummaryService.js @@ -10,10 +10,6 @@ * granted to it by virtue of its status as an Intergovernmental Organization * or submit itself to any jurisdiction. */ -import { BadParameterError } from '../../errors/BadParameterError.js'; -import { dataSource } from '../../../database/DataSource.js'; -import { Op } from 'sequelize'; -import { QcFlagRepository } from '../../../database/repositories/index.js'; /** * @typedef RunQcSummary @@ -33,6 +29,11 @@ import { QcFlagRepository } from '../../../database/repositories/index.js'; * @property {boolean} mcReproducible - states whether some Limited Acceptance MC Reproducible flag was assigned */ +const { BadParameterError } = require('../../errors/BadParameterError.js'); +const { dataSource } = require('../../../database/DataSource.js'); +const { QcFlagRepository } = require('../../../database/repositories/index.js'); +const { Op } = require('sequelize'); + /** * @typedef RunDetectorQcSummary * @property {number} badEffectiveRunCoverage - fraction of run's data, marked explicitly with bad QC flag @@ -51,7 +52,7 @@ const QC_SUMMARY_PROPERTIES = { /** * QC flag summary service */ -export class QcFlagSummaryService { +class QcFlagSummaryService { /** * Update RunDetectorQcSummary or RunGaqSummary with new information * @@ -217,4 +218,6 @@ export class QcFlagSummaryService { } } -export const qcFlagSummaryService = new QcFlagSummaryService(); +exports.QcFlagSummaryService = QcFlagSummaryService; + +exports.qcFlagSummaryService = new QcFlagSummaryService(); From d95c5a40c1ccde5848747a087c8094b7fdab0fe6 Mon Sep 17 00:00:00 2001 From: Martin Boulais <31805063+martinboulais@users.noreply.github.com> Date: Fri, 21 Mar 2025 22:34:11 +0100 Subject: [PATCH 14/18] Fix missing import --- lib/server/services/qualityControlFlag/GaqService.js | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/server/services/qualityControlFlag/GaqService.js b/lib/server/services/qualityControlFlag/GaqService.js index ad0ef86b8c..b9d4551a4a 100644 --- a/lib/server/services/qualityControlFlag/GaqService.js +++ b/lib/server/services/qualityControlFlag/GaqService.js @@ -31,6 +31,7 @@ const { getOneDataPassOrFail } = require('../dataPasses/getOneDataPassOrFail.js' const { QcFlagRepository } = require('../../../database/repositories/index.js'); const { QcFlagSummaryService } = require('./QcFlagSummaryService.js'); const { qcFlagAdapter } = require('../../../database/adapters/index.js'); +const { Op } = require('sequelize'); /** * @typedef GaqSummary aggregated global quality summaries for given data pass From 3377e1257740840acff78256c02350c272f500bf Mon Sep 17 00:00:00 2001 From: Martin Boulais <31805063+martinboulais@users.noreply.github.com> Date: Tue, 25 Mar 2025 16:25:16 +0100 Subject: [PATCH 15/18] Restrict effective run coverage between 0 and 1 --- lib/database/repositories/QcFlagRepository.js | 22 ++++++++++++------- .../QcFlagSummaryService.js | 2 +- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/lib/database/repositories/QcFlagRepository.js b/lib/database/repositories/QcFlagRepository.js index 66367c8aec..a7e1d68af4 100644 --- a/lib/database/repositories/QcFlagRepository.js +++ b/lib/database/repositories/QcFlagRepository.js @@ -242,14 +242,20 @@ class QcFlagRepository extends Repository { mcReproducible, flagsList, verifiedFlagsList, - }) => ({ - runNumber, - bad, - effectiveRunCoverage: parseFloat(effectiveRunCoverage) || null, - mcReproducible: Boolean(mcReproducible), - flagsIds: [...new Set(flagsList.split(','))], - verifiedFlagsIds: verifiedFlagsList ? [...new Set(verifiedFlagsList.split(','))] : [], - })); + }) => { + if ((effectiveRunCoverage ?? null) != null) { + effectiveRunCoverage = Math.min(1, Math.max(0, parseFloat(effectiveRunCoverage))); + } + + return { + runNumber, + bad, + effectiveRunCoverage, + mcReproducible: Boolean(mcReproducible), + flagsIds: [...new Set(flagsList.split(','))], + verifiedFlagsIds: verifiedFlagsList ? [...new Set(verifiedFlagsList.split(','))] : [], + }; + }); } /** diff --git a/lib/server/services/qualityControlFlag/QcFlagSummaryService.js b/lib/server/services/qualityControlFlag/QcFlagSummaryService.js index caa4c96043..f68b84eb59 100644 --- a/lib/server/services/qualityControlFlag/QcFlagSummaryService.js +++ b/lib/server/services/qualityControlFlag/QcFlagSummaryService.js @@ -165,7 +165,7 @@ class QcFlagSummaryService { .map((summaryDb) => { const effectiveRunCoverageString = summaryDb.get('effectiveRunCoverage'); const effectiveRunCoverage = (effectiveRunCoverageString ?? null) !== null - ? parseFloat(effectiveRunCoverageString) + ? Math.min(1, Math.max(0, parseFloat(effectiveRunCoverageString))) : null; return { From 15a67fd89cfff9fdcb00f49d0cd76708f9f39e17 Mon Sep 17 00:00:00 2001 From: Martin Boulais <31805063+martinboulais@users.noreply.github.com> Date: Tue, 25 Mar 2025 17:10:26 +0100 Subject: [PATCH 16/18] Add missing coalesce --- lib/database/repositories/QcFlagRepository.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/database/repositories/QcFlagRepository.js b/lib/database/repositories/QcFlagRepository.js index a7e1d68af4..d5d4d3d1be 100644 --- a/lib/database/repositories/QcFlagRepository.js +++ b/lib/database/repositories/QcFlagRepository.js @@ -27,7 +27,7 @@ const GAQ_PERIODS_VIEW = ` ( SELECT gaqd.data_pass_id, gaqd.run_number, - qcfep.\`from\` AS timestamp, + COALESCE(qcfep.\`from\`, r.qc_time_start) AS timestamp, COALESCE(qcfep.\`from\`, r.qc_time_start, '0001-01-01 00:00:00.000') AS ordering_timestamp FROM quality_control_flag_effective_periods AS qcfep INNER JOIN quality_control_flags AS qcf ON qcf.id = qcfep.flag_id @@ -44,7 +44,7 @@ const GAQ_PERIODS_VIEW = ` ( SELECT gaqd.data_pass_id, gaqd.run_number, - qcfep.\`to\` AS timestamp, + COALESCE(qcfep.\`to\`, r.qc_time_end) AS timestamp, COALESCE(qcfep.\`to\`, r.qc_time_end, NOW()) AS ordering_timestamp FROM quality_control_flag_effective_periods AS qcfep INNER JOIN quality_control_flags AS qcf ON qcf.id = qcfep.flag_id From 571d1da368e1a45c4008105ff1c2c034f1c2c9c6 Mon Sep 17 00:00:00 2001 From: Martin Boulais <31805063+martinboulais@users.noreply.github.com> Date: Wed, 26 Mar 2025 13:50:02 +0100 Subject: [PATCH 17/18] Fix tests again --- test/public/qcFlags/gaqOverview.test.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/public/qcFlags/gaqOverview.test.js b/test/public/qcFlags/gaqOverview.test.js index d907993054..2991d39b79 100644 --- a/test/public/qcFlags/gaqOverview.test.js +++ b/test/public/qcFlags/gaqOverview.test.js @@ -112,7 +112,7 @@ module.exports = () => { expect(await getTableContent(page)).to.have.all.deep.ordered.members([ [ 'good', - 'Since run start', + '08/08/2019\n13:00:00', '08/08/2019\n22:43:20', '', 'Good', @@ -155,7 +155,7 @@ module.exports = () => { [ 'good', '09/08/2019\n09:50:00', - 'Until run end', + '09/08/2019\n14:00:00', '', 'Good', ], From 39cd033a2bb2136c4588c0aac5449e1bd3848dce Mon Sep 17 00:00:00 2001 From: Martin Boulais <31805063+martinboulais@users.noreply.github.com> Date: Wed, 26 Mar 2025 15:14:42 +0100 Subject: [PATCH 18/18] Add migration to set null boundaries --- ...t-flag-bound-null-when-equal-to-run-end.js | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 lib/database/migrations/v1/20250326133702-set-flag-bound-null-when-equal-to-run-end.js diff --git a/lib/database/migrations/v1/20250326133702-set-flag-bound-null-when-equal-to-run-end.js b/lib/database/migrations/v1/20250326133702-set-flag-bound-null-when-equal-to-run-end.js new file mode 100644 index 0000000000..15e787ac0e --- /dev/null +++ b/lib/database/migrations/v1/20250326133702-set-flag-bound-null-when-equal-to-run-end.js @@ -0,0 +1,56 @@ +'use strict'; + +/** @type {import('sequelize-cli').Migration} */ +module.exports = { + up: async (queryInterface) => queryInterface.sequelize.transaction(async () => { + await queryInterface.sequelize.query(`UPDATE quality_control_flags qcf INNER JOIN runs r ON qcf.run_number = r.run_number + SET qcf.\`from\` = NULL + WHERE r.qc_time_start IS NOT NULL + AND qcf.\`from\` = r.qc_time_start`); + + await queryInterface.sequelize.query(`UPDATE quality_control_flag_effective_periods qcfep + INNER JOIN quality_control_flags qcf ON qcfep.flag_id = qcf.id + INNER JOIN runs r ON qcf.run_number = r.run_number + SET qcfep.\`from\` = NULL + WHERE r.qc_time_start IS NOT NULL + AND qcfep.\`from\` = r.qc_time_start`); + + await queryInterface.sequelize.query(`UPDATE quality_control_flags qcf INNER JOIN runs r ON qcf.run_number = r.run_number + SET qcf.\`to\` = NULL + WHERE r.qc_time_end IS NOT NULL + AND qcf.\`to\` = r.qc_time_end`); + + await queryInterface.sequelize.query(`UPDATE quality_control_flag_effective_periods qcfep + INNER JOIN quality_control_flags qcf ON qcfep.flag_id = qcf.id + INNER JOIN runs r ON qcf.run_number = r.run_number + SET qcfep.\`to\` = NULL + WHERE r.qc_time_end IS NOT NULL + AND qcfep.\`to\` = r.qc_time_end`); + }), + + down: async (queryInterface) => queryInterface.sequelize.transaction(async () => { + await queryInterface.sequelize.query(`UPDATE quality_control_flags qcf INNER JOIN runs r ON qcf.run_number = r.run_number + SET qcf.\`from\` = r.qc_time_start + WHERE r.qc_time_start IS NOT NULL + AND qcf.\`from\` IS NULL`); + + await queryInterface.sequelize.query(`UPDATE quality_control_flag_effective_periods qcfep + INNER JOIN quality_control_flags qcf ON qcfep.flag_id = qcf.id + INNER JOIN runs r ON qcf.run_number = r.run_number + SET qcfep.\`from\` = r.qc_time_start + WHERE r.qc_time_start IS NOT NULL + AND qcfep.\`from\` IS NULL`); + + await queryInterface.sequelize.query(`UPDATE quality_control_flags qcf INNER JOIN runs r ON qcf.run_number = r.run_number + SET qcf.\`to\` = r.qc_time_end + WHERE r.qc_time_end IS NOT NULL + AND qcf.\`to\` IS NULL`); + + await queryInterface.sequelize.query(`UPDATE quality_control_flag_effective_periods qcfep + INNER JOIN quality_control_flags qcf ON qcfep.flag_id = qcf.id + INNER JOIN runs r ON qcf.run_number = r.run_number + SET qcfep.\`to\` = r.qc_time_end + WHERE r.qc_time_end IS NOT NULL + AND qcfep.\`to\` IS NULL`); + }), +};