Skip to content

[O2B-1423] Uniformize QC flags to use qc_time_start/qc_time_end #1867

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
martinboulais wants to merge 2 commits into from
Closed

[O2B-1423] Uniformize QC flags to use qc_time_start/qc_time_end #1867

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
56d9da4
[O2B-1423] Uniformize QC flags to use qc_time_start/qc_time_end
martinboulais Jan 29, 2025
c19af61
WIP simplification
martinboulais Jan 29, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
57 changes: 32 additions & 25 deletions lib/database/repositories/QcFlagRepository.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,21 +16,19 @@ const { models: { QcFlag } } = require('..');
const Repository = require('./Repository');

const GAQ_PERIODS_VIEW = `
SELECT * FROM (
SELECT
data_pass_id,
run_number,
timestamp AS \`from\`,
NTH_VALUE(timestamp, 2) OVER (
PARTITION BY data_pass_id,
run_number
ORDER BY ap.timestamp
ROWS BETWEEN CURRENT ROW AND 1 FOLLOWING
) AS \`to\`
LAG(timestamp) OVER w AS \`from\`,
timestamp AS \`to\`,
LAG(ordering_timestamp) OVER w AS from_ordering_timestamp
FROM (
(
SELECT gaqd.data_pass_id,
gaqd.run_number,
COALESCE(UNIX_TIMESTAMP(qcfep.\`from\`), 0) AS timestamp
qcfep.\`from\` AS timestamp,
COALESCE(qcfep.\`from\`, '0001-01-01 00:00:00.000') AS ordering_timestamp
FROM quality_control_flag_effective_periods AS qcfep
INNER JOIN quality_control_flags AS qcf ON qcf.id = qcfep.flag_id
INNER JOIN data_pass_quality_control_flag AS dpqcf ON dpqcf.quality_control_flag_id = qcf.id
Expand All @@ -45,7 +43,8 @@ const GAQ_PERIODS_VIEW = `
(
SELECT gaqd.data_pass_id,
gaqd.run_number,
UNIX_TIMESTAMP(COALESCE(qcfep.\`to\`, NOW())) AS timestamp
qcfep.\`to\` AS timestamp,
COALESCE(qcfep.\`from\`, NOW()) AS ordering_timestamp
FROM quality_control_flag_effective_periods AS qcfep
INNER JOIN quality_control_flags AS qcf ON qcf.id = qcfep.flag_id
INNER JOIN data_pass_quality_control_flag AS dpqcf ON dpqcf.quality_control_flag_id = qcf.id
Expand All @@ -56,8 +55,15 @@ const GAQ_PERIODS_VIEW = `
AND gaqd.run_number = qcf.run_number
AND gaqd.detector_id = qcf.detector_id
)
ORDER BY timestamp
) AS ap
ORDER BY ordering_timestamp
) AS ap
WINDOW w AS (
PARTITION BY data_pass_id,
run_number
ORDER BY ap.timestamp
)
) as gaq_periods_with_last_nullish_row
WHERE gaq_periods_with_last_nullish_row.from_ordering_timestamp IS NOT NULL
`;

/**
Expand Down Expand Up @@ -104,8 +110,8 @@ class QcFlagRepository extends Repository {
SELECT
gaq_periods.data_pass_id AS dataPassId,
gaq_periods.run_number AS runNumber,
IF(gaq_periods.\`from\` = 0, null, gaq_periods.\`from\` * 1000) AS \`from\`,
IF(gaq_periods.\`to\` = UNIX_TIMESTAMP(NOW()), null, gaq_periods.\`to\` * 1000) AS \`to\`,
gaq_periods.\`from\` AS \`from\`,
gaq_periods.\`to\` AS \`to\`,
group_concat(qcf.id) AS contributingFlagIds

FROM quality_control_flags AS qcf
Expand All @@ -118,8 +124,8 @@ class QcFlagRepository extends Repository {
AND gaqd.run_number = gaq_periods.run_number
AND gaqd.detector_id = qcf.detector_id
AND gaq_periods.run_number = qcf.run_number
AND (qcfep.\`from\` IS NULL OR UNIX_TIMESTAMP(qcfep.\`from\`) <= gaq_periods.\`from\`)
AND (qcfep.\`to\` IS NULL OR gaq_periods.\`to\` <= UNIX_TIMESTAMP(qcfep.\`to\`))
AND (qcfep.\`from\` IS NULL OR qcfep.\`from\` <= gaq_periods.\`from\`)
AND (qcfep.\`to\` IS NULL OR gaq_periods.\`to\` <= qcfep.\`to\`)

WHERE gaq_periods.data_pass_id = ${dataPassId}
${runNumber ? `AND gaq_periods.run_number = ${runNumber}` : ''}
Expand Down Expand Up @@ -160,8 +166,8 @@ class QcFlagRepository extends Repository {
SELECT
gaq_periods.data_pass_id AS dataPassId,
gaq_periods.run_number AS runNumber,
IF(gaq_periods.\`from\` = 0, null, gaq_periods.\`from\`) AS \`from\`,
IF(gaq_periods.\`to\` = UNIX_TIMESTAMP(NOW()), null, gaq_periods.\`to\`) AS \`to\`,
gaq_periods.\`from\` AS \`from\`,
gaq_periods.\`to\` AS \`to\`,
SUM(IF(qcft.monte_carlo_reproducible AND :mcReproducibleAsNotBad, false, qcft.bad)) >= 1 AS bad,
SUM(qcft.bad) = SUM(qcft.monte_carlo_reproducible) AND SUM(qcft.monte_carlo_reproducible) AS mcReproducible,
GROUP_CONCAT( DISTINCT qcfv.flag_id ) AS verifiedFlagsList,
Expand All @@ -183,8 +189,8 @@ class QcFlagRepository extends Repository {
AND gaqd.run_number = gaq_periods.run_number
AND gaqd.detector_id = qcf.detector_id
AND gaq_periods.run_number = qcf.run_number
AND (qcfep.\`from\` IS NULL OR UNIX_TIMESTAMP(qcfep.\`from\`) <= gaq_periods.\`from\`)
AND (qcfep.\`to\` IS NULL OR gaq_periods.\`to\` <= UNIX_TIMESTAMP(qcfep.\`to\`))
AND (qcfep.\`from\` IS NULL OR qcfep.\`from\` <= gaq_periods.\`from\`)
AND (qcfep.\`to\` IS NULL OR gaq_periods.\`to\` <= qcfep.\`to\`)

GROUP BY
gaq_periods.data_pass_id,
Expand All @@ -203,18 +209,19 @@ class QcFlagRepository extends Repository {
GROUP_CONCAT(effectivePeriods.flagsList) AS flagsList,

IF(
run.time_start IS NULL OR run.time_end IS NULL,
run.qc_time_start IS NULL OR run.qc_time_end IS NULL,
IF(
effectivePeriods.\`from\` IS NULL AND effectivePeriods.\`to\` IS NULL,
1,
null
),
SUM(
COALESCE(effectivePeriods.\`to\`, UNIX_TIMESTAMP(run.time_end))
- COALESCE(effectivePeriods.\`from\`, UNIX_TIMESTAMP(run.time_start))
) / (
UNIX_TIMESTAMP(run.time_end) - UNIX_TIMESTAMP(run.time_start)
)
TIMESTAMPDIFF(
MICROSECOND,
COALESCE(effectivePeriods.\`to\`,run.qc_time_end),
COALESCE(effectivePeriods.\`from\`, run.qc_time_start)
)
) / (TIMESTAMPDIFF(MICROSECOND, run.qc_time_end, run.qc_time_start))
) AS effectiveRunCoverage

FROM (${effectivePeriodsWithTypeSubQuery}) AS effectivePeriods
Expand Down
27 changes: 16 additions & 11 deletions lib/server/services/qualityControlFlag/QcFlagService.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -267,18 +267,17 @@
[
sequelize.literal(`
IF(
run.time_start IS NULL OR run.time_end IS NULL,
run.qc_time_start IS NULL OR run.qc_time_end IS NULL,
IF(
effectivePeriods.\`from\` IS NULL AND effectivePeriods.\`to\` IS NULL,
1,
null
),
SUM(
UNIX_TIMESTAMP(COALESCE(effectivePeriods.\`to\`, run.time_end))
- UNIX_TIMESTAMP(COALESCE(effectivePeriods.\`from\`, run.time_start))
) / (
UNIX_TIMESTAMP(run.time_end) - UNIX_TIMESTAMP(run.time_start)
)
SUM(TIMESTAMPDIFF(
MICROSECOND,
COALESCE(effectivePeriods.\`to\`, run.qc_time_end),
COALESCE(effectivePeriods.\`from\`, run.qc_time_start)
)) / (TIMESTAMPDIFF(MICROSECOND, run.qc_time_end, run.qc_time_start))
)
`),
'effectiveRunCoverage',
Expand All @@ -299,15 +298,21 @@
`));

const runDetectorSummaryList = (await QcFlagRepository.findAll(queryBuilder))
.map((summaryDb) =>
({
.map((summaryDb) => {
const effectiveRunCoverageString = summaryDb.get('effectiveRunCoverage');
const effectiveRunCoverage = (effectiveRunCoverageString ?? null) !== null
? parseFloat(effectiveRunCoverageString)
: null;

Check warning on line 305 in lib/server/services/qualityControlFlag/QcFlagService.js

View check run for this annotation

Codecov / codecov/patch

lib/server/services/qualityControlFlag/QcFlagService.js#L305 

Added line #L305 was not covered by tests

return {
runNumber: summaryDb.runNumber,
detectorId: summaryDb.detectorId,
effectiveRunCoverage: parseFloat(summaryDb.get('effectiveRunCoverage'), 10) || null,
effectiveRunCoverage,
bad: Boolean(summaryDb.get('bad')),
flagIds: (summaryDb.get('flagIds')?.split(',') ?? []).map((id) => parseInt(id, 10)),
mcReproducible: Boolean(summaryDb.get('mcReproducible')),
}));
};
});

const allFlagsIds = new Set(runDetectorSummaryList.flatMap(({ flagIds }) => flagIds));
const notVerifiedFlagsIds = new Set((await QcFlagRepository.findAll({
Expand Down
Loading