diff --git a/lib/database/repositories/QcFlagRepository.js b/lib/database/repositories/QcFlagRepository.js index dccde007fb..3eeaddfbe2 100644 --- a/lib/database/repositories/QcFlagRepository.js +++ b/lib/database/repositories/QcFlagRepository.js @@ -16,21 +16,19 @@ const { models: { QcFlag } } = require('..'); const Repository = require('./Repository'); const GAQ_PERIODS_VIEW = ` + SELECT * FROM ( SELECT data_pass_id, run_number, - timestamp AS \`from\`, - NTH_VALUE(timestamp, 2) OVER ( - PARTITION BY data_pass_id, - run_number - ORDER BY ap.timestamp - ROWS BETWEEN CURRENT ROW AND 1 FOLLOWING - ) AS \`to\` + LAG(timestamp) OVER w AS \`from\`, + timestamp AS \`to\`, + LAG(ordering_timestamp) OVER w AS from_ordering_timestamp FROM ( ( SELECT gaqd.data_pass_id, gaqd.run_number, - COALESCE(UNIX_TIMESTAMP(qcfep.\`from\`), 0) AS timestamp + qcfep.\`from\` AS timestamp, + COALESCE(qcfep.\`from\`, '0001-01-01 00:00:00.000') AS ordering_timestamp FROM quality_control_flag_effective_periods AS qcfep INNER JOIN quality_control_flags AS qcf ON qcf.id = qcfep.flag_id INNER JOIN data_pass_quality_control_flag AS dpqcf ON dpqcf.quality_control_flag_id = qcf.id @@ -45,7 +43,8 @@ const GAQ_PERIODS_VIEW = ` ( SELECT gaqd.data_pass_id, gaqd.run_number, - UNIX_TIMESTAMP(COALESCE(qcfep.\`to\`, NOW())) AS timestamp + qcfep.\`to\` AS timestamp, + COALESCE(qcfep.\`from\`, NOW()) AS ordering_timestamp FROM quality_control_flag_effective_periods AS qcfep INNER JOIN quality_control_flags AS qcf ON qcf.id = qcfep.flag_id INNER JOIN data_pass_quality_control_flag AS dpqcf ON dpqcf.quality_control_flag_id = qcf.id @@ -56,8 +55,15 @@ const GAQ_PERIODS_VIEW = ` AND gaqd.run_number = qcf.run_number AND gaqd.detector_id = qcf.detector_id ) - ORDER BY timestamp - ) AS ap + ORDER BY ordering_timestamp + ) AS ap + WINDOW w AS ( + PARTITION BY data_pass_id, + run_number + ORDER BY ap.timestamp + ) + ) as gaq_periods_with_last_nullish_row + WHERE gaq_periods_with_last_nullish_row.from_ordering_timestamp IS NOT NULL `; /** @@ -104,8 +110,8 @@ class QcFlagRepository extends Repository { SELECT gaq_periods.data_pass_id AS dataPassId, gaq_periods.run_number AS runNumber, - IF(gaq_periods.\`from\` = 0, null, gaq_periods.\`from\` * 1000) AS \`from\`, - IF(gaq_periods.\`to\` = UNIX_TIMESTAMP(NOW()), null, gaq_periods.\`to\` * 1000) AS \`to\`, + gaq_periods.\`from\` AS \`from\`, + gaq_periods.\`to\` AS \`to\`, group_concat(qcf.id) AS contributingFlagIds FROM quality_control_flags AS qcf @@ -118,8 +124,8 @@ class QcFlagRepository extends Repository { AND gaqd.run_number = gaq_periods.run_number AND gaqd.detector_id = qcf.detector_id AND gaq_periods.run_number = qcf.run_number - AND (qcfep.\`from\` IS NULL OR UNIX_TIMESTAMP(qcfep.\`from\`) <= gaq_periods.\`from\`) - AND (qcfep.\`to\` IS NULL OR gaq_periods.\`to\` <= UNIX_TIMESTAMP(qcfep.\`to\`)) + AND (qcfep.\`from\` IS NULL OR qcfep.\`from\` <= gaq_periods.\`from\`) + AND (qcfep.\`to\` IS NULL OR gaq_periods.\`to\` <= qcfep.\`to\`) WHERE gaq_periods.data_pass_id = ${dataPassId} ${runNumber ? `AND gaq_periods.run_number = ${runNumber}` : ''} @@ -160,8 +166,8 @@ class QcFlagRepository extends Repository { SELECT gaq_periods.data_pass_id AS dataPassId, gaq_periods.run_number AS runNumber, - IF(gaq_periods.\`from\` = 0, null, gaq_periods.\`from\`) AS \`from\`, - IF(gaq_periods.\`to\` = UNIX_TIMESTAMP(NOW()), null, gaq_periods.\`to\`) AS \`to\`, + gaq_periods.\`from\` AS \`from\`, + gaq_periods.\`to\` AS \`to\`, SUM(IF(qcft.monte_carlo_reproducible AND :mcReproducibleAsNotBad, false, qcft.bad)) >= 1 AS bad, SUM(qcft.bad) = SUM(qcft.monte_carlo_reproducible) AND SUM(qcft.monte_carlo_reproducible) AS mcReproducible, GROUP_CONCAT( DISTINCT qcfv.flag_id ) AS verifiedFlagsList, @@ -183,8 +189,8 @@ class QcFlagRepository extends Repository { AND gaqd.run_number = gaq_periods.run_number AND gaqd.detector_id = qcf.detector_id AND gaq_periods.run_number = qcf.run_number - AND (qcfep.\`from\` IS NULL OR UNIX_TIMESTAMP(qcfep.\`from\`) <= gaq_periods.\`from\`) - AND (qcfep.\`to\` IS NULL OR gaq_periods.\`to\` <= UNIX_TIMESTAMP(qcfep.\`to\`)) + AND (qcfep.\`from\` IS NULL OR qcfep.\`from\` <= gaq_periods.\`from\`) + AND (qcfep.\`to\` IS NULL OR gaq_periods.\`to\` <= qcfep.\`to\`) GROUP BY gaq_periods.data_pass_id, @@ -203,18 +209,19 @@ class QcFlagRepository extends Repository { GROUP_CONCAT(effectivePeriods.flagsList) AS flagsList, IF( - run.time_start IS NULL OR run.time_end IS NULL, + run.qc_time_start IS NULL OR run.qc_time_end IS NULL, IF( effectivePeriods.\`from\` IS NULL AND effectivePeriods.\`to\` IS NULL, 1, null ), SUM( - COALESCE(effectivePeriods.\`to\`, UNIX_TIMESTAMP(run.time_end)) - - COALESCE(effectivePeriods.\`from\`, UNIX_TIMESTAMP(run.time_start)) - ) / ( - UNIX_TIMESTAMP(run.time_end) - UNIX_TIMESTAMP(run.time_start) - ) + TIMESTAMPDIFF( + MICROSECOND, + COALESCE(effectivePeriods.\`to\`,run.qc_time_end), + COALESCE(effectivePeriods.\`from\`, run.qc_time_start) + ) + ) / (TIMESTAMPDIFF(MICROSECOND, run.qc_time_end, run.qc_time_start)) ) AS effectiveRunCoverage FROM (${effectivePeriodsWithTypeSubQuery}) AS effectivePeriods diff --git a/lib/server/services/qualityControlFlag/QcFlagService.js b/lib/server/services/qualityControlFlag/QcFlagService.js index 99cd93c86c..6a2437eeed 100644 --- a/lib/server/services/qualityControlFlag/QcFlagService.js +++ b/lib/server/services/qualityControlFlag/QcFlagService.js @@ -267,18 +267,17 @@ class QcFlagService { [ sequelize.literal(` IF( - run.time_start IS NULL OR run.time_end IS NULL, + run.qc_time_start IS NULL OR run.qc_time_end IS NULL, IF( effectivePeriods.\`from\` IS NULL AND effectivePeriods.\`to\` IS NULL, 1, null ), - SUM( - UNIX_TIMESTAMP(COALESCE(effectivePeriods.\`to\`, run.time_end)) - - UNIX_TIMESTAMP(COALESCE(effectivePeriods.\`from\`, run.time_start)) - ) / ( - UNIX_TIMESTAMP(run.time_end) - UNIX_TIMESTAMP(run.time_start) - ) + SUM(TIMESTAMPDIFF( + MICROSECOND, + COALESCE(effectivePeriods.\`to\`, run.qc_time_end), + COALESCE(effectivePeriods.\`from\`, run.qc_time_start) + )) / (TIMESTAMPDIFF(MICROSECOND, run.qc_time_end, run.qc_time_start)) ) `), 'effectiveRunCoverage', @@ -299,15 +298,21 @@ class QcFlagService { `)); const runDetectorSummaryList = (await QcFlagRepository.findAll(queryBuilder)) - .map((summaryDb) => - ({ + .map((summaryDb) => { + const effectiveRunCoverageString = summaryDb.get('effectiveRunCoverage'); + const effectiveRunCoverage = (effectiveRunCoverageString ?? null) !== null + ? parseFloat(effectiveRunCoverageString) + : null; + + return { runNumber: summaryDb.runNumber, detectorId: summaryDb.detectorId, - effectiveRunCoverage: parseFloat(summaryDb.get('effectiveRunCoverage'), 10) || null, + effectiveRunCoverage, bad: Boolean(summaryDb.get('bad')), flagIds: (summaryDb.get('flagIds')?.split(',') ?? []).map((id) => parseInt(id, 10)), mcReproducible: Boolean(summaryDb.get('mcReproducible')), - })); + }; + }); const allFlagsIds = new Set(runDetectorSummaryList.flatMap(({ flagIds }) => flagIds)); const notVerifiedFlagsIds = new Set((await QcFlagRepository.findAll({