From 4b9517f595abc81fa02fcd65b21efb79fca14e65 Mon Sep 17 00:00:00 2001 From: Martin Boulais <31805063+martinboulais@users.noreply.github.com> Date: Mon, 17 Feb 2025 10:59:56 +0100 Subject: [PATCH 1/5] [O2B-1412] Configure kafka to retry indefinitely Slightly improve logging --- lib/application.js | 3 +++ lib/server/kafka/AliEcsSynchronizer.js | 6 +++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/application.js b/lib/application.js index ee3ed8d0bd..a293602bf3 100644 --- a/lib/application.js +++ b/lib/application.js @@ -51,6 +51,9 @@ class BookkeepingApplication { clientId: 'bookkeeping', brokers: KafkaConfig.brokers, logLevel: logLevel.NOTHING, + retry: { + retries: Infinity, + }, }); this.aliEcsSynchronizer = new AliEcsSynchronizer({ diff --git a/lib/server/kafka/AliEcsSynchronizer.js b/lib/server/kafka/AliEcsSynchronizer.js index 8ec4f73472..d45886e7cb 100644 --- a/lib/server/kafka/AliEcsSynchronizer.js +++ b/lib/server/kafka/AliEcsSynchronizer.js @@ -123,14 +123,14 @@ class AliEcsSynchronizer { start() { this._logger.infoMessage('Starting to consume AliECS messages'); this.ecsEnvironmentsConsumer.start() - .catch((error) => this._logger.errorMessage(`Error when starting ECS environment consumer: ${error.message}\n${error.trace}`)); + .catch((error) => this._logger.errorMessage(`Error when starting ECS environment consumer: ${error.message}\n${JSON.stringify(error)}`)); this.ecsRunConsumer.start() - .catch((error) => this._logger.errorMessage(`Error when starting ECS runs consumer: ${error.message}\n${error.trace}`)); + .catch((error) => this._logger.errorMessage(`Error when starting ECS runs consumer: ${error.message}\n${JSON.stringify(error)}`)); this.ecsTrgIntegratedServiceConsumer.start() .catch((error) => this._logger - .errorMessage(`Error when starting ECS trigger integrated service consumer: ${error.message}\n${error.trace}`)); + .errorMessage(`Error when starting ECS trigger integrated service consumer: ${error.message}\n${JSON.stringify(error)}`)); } } From a6c8bffda1bd783b40917e1c2ca5c6f7f9dbf6dc Mon Sep 17 00:00:00 2001 From: Martin Boulais <31805063+martinboulais@users.noreply.github.com> Date: Mon, 17 Feb 2025 11:08:28 +0100 Subject: [PATCH 2/5] Fix linter --- .eslintrc | 2 +- lib/server/kafka/AliEcsSynchronizer.js | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.eslintrc b/.eslintrc index 5540787ec2..b310176f23 100644 --- a/.eslintrc +++ b/.eslintrc @@ -87,7 +87,7 @@ ], "function-paren-newline": [ "error", - "multiline" + "multiline-arguments" ], "indent": [ "error", diff --git a/lib/server/kafka/AliEcsSynchronizer.js b/lib/server/kafka/AliEcsSynchronizer.js index d45886e7cb..d5b34197b9 100644 --- a/lib/server/kafka/AliEcsSynchronizer.js +++ b/lib/server/kafka/AliEcsSynchronizer.js @@ -122,8 +122,9 @@ class AliEcsSynchronizer { */ start() { this._logger.infoMessage('Starting to consume AliECS messages'); - this.ecsEnvironmentsConsumer.start() - .catch((error) => this._logger.errorMessage(`Error when starting ECS environment consumer: ${error.message}\n${JSON.stringify(error)}`)); + this.ecsEnvironmentsConsumer.start().catch((error) => this._logger.errorMessage( + `Error when starting ECS environment consumer: ${error.message}\n${JSON.stringify(error)}`, + )); this.ecsRunConsumer.start() .catch((error) => this._logger.errorMessage(`Error when starting ECS runs consumer: ${error.message}\n${JSON.stringify(error)}`)); From 37fb5fef3faa3aa7743a7a839a640df142b60d68 Mon Sep 17 00:00:00 2001 From: Martin Boulais <31805063+martinboulais@users.noreply.github.com> Date: Mon, 17 Feb 2025 12:01:04 +0100 Subject: [PATCH 3/5] Use kafka log creator to send logs to infologger --- lib/application.js | 6 ++- lib/server/kafka/InfologgerKafkaLogCreator.js | 52 +++++++++++++++++++ 2 files changed, 56 insertions(+), 2 deletions(-) create mode 100644 lib/server/kafka/InfologgerKafkaLogCreator.js diff --git a/lib/application.js b/lib/application.js index a293602bf3..397bcb7871 100644 --- a/lib/application.js +++ b/lib/application.js @@ -10,6 +10,7 @@ * granted to it by virtue of its status as an Intergovernmental Organization * or submit itself to any jurisdiction. */ + const database = require('./database'); const { webUiServer } = require('./server'); const { gRPCServer } = require('./server/index.js'); @@ -21,7 +22,7 @@ const { isInTestMode } = require('./utilities/env-utils.js'); const { ScheduledProcessesManager } = require('./server/services/ScheduledProcessesManager.js'); const { MonAlisaSynchronizer } = require('./server/externalServicesSynchronization/monalisa/MonAlisaSynchronizer'); const { LogManager } = require('@aliceo2/web-ui'); -const { Kafka, logLevel } = require('kafkajs'); +const { Kafka } = require('kafkajs'); const { KafkaConfig } = require('./config/index.js'); const { AliEcsSynchronizer } = require('./server/kafka/AliEcsSynchronizer.js'); const { environmentService } = require('./server/services/environment/EnvironmentService.js'); @@ -30,6 +31,7 @@ const { CcdbSynchronizer } = require('./server/externalServicesSynchronization/c const { promises: fs } = require('fs'); const { MonAlisaClient } = require('./server/externalServicesSynchronization/monalisa/MonAlisaClient.js'); const https = require('https'); +const { InfologgerKafkaLogCreator } = require('./server/kafka/InfologgerKafkaLogCreator.js'); /** * Bookkeeping Application @@ -50,7 +52,7 @@ class BookkeepingApplication { const kafkaClient = new Kafka({ clientId: 'bookkeeping', brokers: KafkaConfig.brokers, - logLevel: logLevel.NOTHING, + logCreator: InfologgerKafkaLogCreator, retry: { retries: Infinity, }, diff --git a/lib/server/kafka/InfologgerKafkaLogCreator.js b/lib/server/kafka/InfologgerKafkaLogCreator.js new file mode 100644 index 0000000000..7cfcbb39cd --- /dev/null +++ b/lib/server/kafka/InfologgerKafkaLogCreator.js @@ -0,0 +1,52 @@ +/** + * @license + * Copyright CERN and copyright holders of ALICE O2. This software is + * distributed under the terms of the GNU General Public License v3 (GPL + * Version 3), copied verbatim in the file "COPYING". + * + * See http://alice-o2.web.cern.ch/license for full licensing information. + * + * In applying this license CERN does not waive the privileges and immunities + * granted to it by virtue of its status as an Intergovernmental Organization + * or submit itself to any jurisdiction. + */ + +const { LogManager } = require('@aliceo2/web-ui'); +const { logLevel } = require('kafkajs'); + +/** + * Kafka log creator to send messages to infologger + * + * @return {function} log creator ready to be used for infologger + * @constructor + */ +exports.InfologgerKafkaLogCreator = () => { + const logger = LogManager.getLogger('KAFKA'); + return ({ level, log }) => { + const { broker, message, stack } = log; + let logMessage = ''; + if (broker) { + logMessage = `Broker ${broker} - `; + } + logMessage += message; + if (stack) { + logMessage += `\n${stack}`; + } + + switch (level) { + case logLevel.ERROR: + case logLevel.NOTHING: + logger.errorMessage(logMessage); + break; + case logLevel.WARN: + logger.warnMessage(logMessage); + break; + case logLevel.INFO: + logger.infoMessage(logMessage); + break; + default: + logger.debugMessage(logMessage); + break; + } + }; +}; From be9b628dc4bc0b9f9f146e6efbaad0462eddd1da Mon Sep 17 00:00:00 2001 From: martinboulais <31805063+martinboulais@users.noreply.github.com> Date: Fri, 21 Feb 2025 15:34:49 +0100 Subject: [PATCH 4/5] Try to fix randomly failing test --- test/public/defaults.js | 8 ++++++-- test/public/runs/runsPerDataPass.overview.test.js | 6 ++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/test/public/defaults.js b/test/public/defaults.js index 6cd2ebbea6..24827a8993 100644 --- a/test/public/defaults.js +++ b/test/public/defaults.js @@ -393,14 +393,18 @@ module.exports.getInnerText = getInnerText; * @param {Object} page Puppeteer page object. * @param {string} selector Css selector. * @param {string} innerText Text to search for. + * @param {object} [options] eventual options + * @param {number} [options.timeout] Optional timeout + * @param {puppeteer.FrameWaitForFunctionOptions} [options.polling] Optional polling methods, see * @return {Promise} resolves once the text has been checked */ -module.exports.expectInnerText = async (page, selector, innerText) => { +module.exports.expectInnerText = async (page, selector, innerText, options = {}) => { const elementHandle = await page.waitForSelector(selector); + try { await page.waitForFunction( (selector, innerText) => document.querySelector(selector).innerText === innerText, - {}, + options, selector, innerText, ); diff --git a/test/public/runs/runsPerDataPass.overview.test.js b/test/public/runs/runsPerDataPass.overview.test.js index 9ad7894bd6..575266ee34 100644 --- a/test/public/runs/runsPerDataPass.overview.test.js +++ b/test/public/runs/runsPerDataPass.overview.test.js @@ -482,12 +482,14 @@ module.exports = () => { // eslint-disable-next-line no-undef sessionService.get().access.push(role); }, BkpRoles.DPG_ASYNC_QC_ADMIN); - await setConfirmationDialogToBeAccepted(page); const popoverSelector = await getPopoverSelector(await page.waitForSelector('#actions-dropdown-button .popover-trigger')); // Press again actions dropdown to re-trigger render await pressElement(page, '#actions-dropdown-button .popover-trigger', true); + await setConfirmationDialogToBeAccepted(page); await pressElement(page, `${popoverSelector} button:nth-child(3)`, true); + await pressElement(page, '#actions-dropdown-button .popover-trigger', true); await waitForTableLength(page, 3); - await expectInnerText(page, '#row106-CPV-text', 'QC'); // Expect QC flag button to be there + // Processing of data might take a bit of time + await expectInnerText(page, '#row106-CPV-text', 'QC', {timeout: 10_000, polling: 'mutation'}); // Expect QC flag button to be there }); }; From 2ecc25733ee0493de3a14daf72468405a0bd2c92 Mon Sep 17 00:00:00 2001 From: martinboulais <31805063+martinboulais@users.noreply.github.com> Date: Fri, 21 Feb 2025 15:40:03 +0100 Subject: [PATCH 5/5] Fix linter --- test/public/runs/runsPerDataPass.overview.test.js | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/test/public/runs/runsPerDataPass.overview.test.js b/test/public/runs/runsPerDataPass.overview.test.js index 575266ee34..1607785fd7 100644 --- a/test/public/runs/runsPerDataPass.overview.test.js +++ b/test/public/runs/runsPerDataPass.overview.test.js @@ -489,7 +489,12 @@ module.exports = () => { await pressElement(page, `${popoverSelector} button:nth-child(3)`, true); await pressElement(page, '#actions-dropdown-button .popover-trigger', true); await waitForTableLength(page, 3); - // Processing of data might take a bit of time - await expectInnerText(page, '#row106-CPV-text', 'QC', {timeout: 10_000, polling: 'mutation'}); // Expect QC flag button to be there + // Processing of data might take a bit of time, but then expect QC flag button to be there + await expectInnerText( + page, + '#row106-CPV-text', + 'QC', + { timeout: 10000, polling: 'mutation' }, + ); }); };