From 72acda01bb1fe963e45e1b920e20bcf46d68f8e3 Mon Sep 17 00:00:00 2001 From: Adrien MARCHAL Date: Wed, 23 Jul 2025 14:17:21 +0200 Subject: [PATCH 1/6] !!!WARNING!!! - REMOVED OLD V1 METHODS --- .../rest/responses/CampaignController.java | 2 +- .../responses/InterrogationController.java | 2 + .../rest/responses/ModeController.java | 13 -- .../responses/QuestionnaireController.java | 6 - .../rest/responses/ResponseController.java | 44 +++---- .../domain/ports/api/SurveyUnitApiPort.java | 12 +- .../ports/spi/SurveyUnitPersistencePort.java | 5 +- .../service/surveyunit/SurveyUnitService.java | 118 +++++------------- .../adapter/SurveyUnitMongoAdapter.java | 28 +---- .../SurveyUnitMongoDBRepository.java | 9 +- .../rest/responses/ModeControllerTest.java | 4 +- .../QuestionnaireControllerTest.java | 2 +- .../domain/service/SurveyUnitServiceTest.java | 8 +- .../stubs/SurveyUnitPersistencePortStub.java | 42 +++---- 14 files changed, 91 insertions(+), 204 deletions(-) diff --git a/src/main/java/fr/insee/genesis/controller/rest/responses/CampaignController.java b/src/main/java/fr/insee/genesis/controller/rest/responses/CampaignController.java index 9daaa129..77b77266 100644 --- a/src/main/java/fr/insee/genesis/controller/rest/responses/CampaignController.java +++ b/src/main/java/fr/insee/genesis/controller/rest/responses/CampaignController.java @@ -35,7 +35,7 @@ public ResponseEntity> getCampaigns() { @Operation(summary = "List campaigns in database with their questionnaires") @GetMapping(path = "/with-questionnaires") public ResponseEntity> getCampaignsWithQuestionnaires() { - List questionnairesByCampaigns = surveyUnitService.findCampaignsWithQuestionnaires(); + List questionnairesByCampaigns = surveyUnitService.findCampaignsWithQuestionnairesV2(); return ResponseEntity.ok(questionnairesByCampaigns); } diff --git a/src/main/java/fr/insee/genesis/controller/rest/responses/InterrogationController.java b/src/main/java/fr/insee/genesis/controller/rest/responses/InterrogationController.java index c3eda664..1ef6d67a 100644 --- a/src/main/java/fr/insee/genesis/controller/rest/responses/InterrogationController.java +++ b/src/main/java/fr/insee/genesis/controller/rest/responses/InterrogationController.java @@ -28,6 +28,8 @@ public InterrogationController(SurveyUnitApiPort surveyUnitService) { } + //NOTE : methode used (It has to be kept with V2 functions!) + //!!!WARNING!!! : A CALL WITH THIS ENDPOINT ON A BIG COLLECTION (> 300k) MAY KILL THE GENESIS-API APP.!!! @Operation(summary = "Retrieve all interrogations for a given questionnaire") @GetMapping(path = "/by-questionnaire") public ResponseEntity> getAllInterrogationIdsByQuestionnaire(@RequestParam("questionnaireId") String questionnaireId) { diff --git a/src/main/java/fr/insee/genesis/controller/rest/responses/ModeController.java b/src/main/java/fr/insee/genesis/controller/rest/responses/ModeController.java index 8c2afd30..56609806 100644 --- a/src/main/java/fr/insee/genesis/controller/rest/responses/ModeController.java +++ b/src/main/java/fr/insee/genesis/controller/rest/responses/ModeController.java @@ -27,19 +27,6 @@ public ModeController(SurveyUnitApiPort surveyUnitService) { } - @Operation(summary = "List sources/modes used for a given questionnaire") - @GetMapping(path = "/by-questionnaire") - public ResponseEntity> getModesByQuestionnaire(@RequestParam("questionnaireId") String questionnaireId) { - List modes = surveyUnitService.findModesByQuestionnaireId(questionnaireId); - return ResponseEntity.ok(modes); - } - - @Operation(summary = "List sources/modes used for a given campaign") - @GetMapping(path = "/by-campaign") - public ResponseEntity> getModesByCampaign(@RequestParam("campaignId") String campaignId) { - List modes = surveyUnitService.findModesByCampaignId(campaignId); - return ResponseEntity.ok(modes); - } //========= OPTIMISATIONS PERFS (START) ========== @Operation(summary = "List sources/modes used for a given questionnaire") diff --git a/src/main/java/fr/insee/genesis/controller/rest/responses/QuestionnaireController.java b/src/main/java/fr/insee/genesis/controller/rest/responses/QuestionnaireController.java index 68baa76a..6b631e83 100644 --- a/src/main/java/fr/insee/genesis/controller/rest/responses/QuestionnaireController.java +++ b/src/main/java/fr/insee/genesis/controller/rest/responses/QuestionnaireController.java @@ -46,12 +46,6 @@ public ResponseEntity> getQuestionnairesWithCamp return ResponseEntity.ok(questionnaireWithCampaignList); } - @Operation(summary = "List questionnaires used for a given campaign") - @GetMapping(path = "/by-campaign") - public ResponseEntity> getQuestionnairesByCampaign(@RequestParam("campaignId") String campaignId) { - Set questionnaires = surveyUnitService.findQuestionnaireIdsByCampaignId(campaignId); - return ResponseEntity.ok(questionnaires); - } //========= OPTIMISATIONS PERFS (START) ========== /** diff --git a/src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java b/src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java index 673f721f..d524dda0 100644 --- a/src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java +++ b/src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java @@ -1,7 +1,5 @@ package fr.insee.genesis.controller.rest.responses; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; import fr.insee.bpm.exceptions.MetadataParserException; import fr.insee.bpm.metadata.model.VariablesMap; import fr.insee.bpm.metadata.reader.ddi.DDIReader; @@ -245,6 +243,9 @@ public ResponseEntity findResponsesByInterrogationAndQuestionnaireLatest return ResponseEntity.ok(responseQualityToolDto); } + /** + * !!!WARNING!!! : A CALL WITH THIS ENDPOINT ON A BIG COLLECTION (> 300k) MAY KILL THE GENESIS-API APP.!!! + */ @Hidden @Operation(summary = "Retrieve all responses (for all interrogations) of one questionnaire") @GetMapping(path = "/by-questionnaire") @@ -301,48 +302,33 @@ public ResponseEntity getLatestByInterrogationOneObject(@R } + + + //========= OPTIMISATIONS PERFS (START) ========== + /** + * @author Adrien Marchal + */ @Operation(summary = "Retrieve all responses for a questionnaire and a list of UE", description = "Return the latest state for each variable for the given ids and a given questionnaire.
" + "For a given id, the endpoint returns a document by collection mode (if there is more than one).") @PostMapping(path = "/simplified/by-list-interrogation-and-questionnaire/latest") @PreAuthorize("hasRole('USER_KRAFTWERK')") public ResponseEntity> getLatestForInterrogationList(@RequestParam("questionnaireId") String questionnaireId, - @RequestBody List interrogationIds) { - List results = new ArrayList<>(); - List modes = surveyUnitService.findModesByQuestionnaireId(questionnaireId); - interrogationIds.forEach(interrogationId -> { - List responses = surveyUnitService.findLatestByIdAndByQuestionnaireId(interrogationId.getInterrogationId(), questionnaireId); - modes.forEach(mode -> { - List outputVariables = new ArrayList<>(); - List outputExternalVariables = new ArrayList<>(); - responses.stream().filter(rep -> rep.getMode().equals(mode)).forEach(response -> { - outputVariables.addAll(response.getCollectedVariables()); - outputExternalVariables.addAll(response.getExternalVariables()); - }); - if (!outputVariables.isEmpty() || !outputExternalVariables.isEmpty()) { - results.add(SurveyUnitSimplified.builder() - .questionnaireId(responses.getFirst().getQuestionnaireId()) - .campaignId(responses.getFirst().getCampaignId()) - .interrogationId(responses.getFirst().getInterrogationId()) - .mode(mode) - .variablesUpdate(outputVariables) - .externalVariables(outputExternalVariables) - .build()); - } - }); - }); - return ResponseEntity.ok(results); + @RequestBody List interrogationIds) { + List enumModes = surveyUnitService.findModesByQuestionnaireIdV2(questionnaireId); + // => convertion of "List" -> "List" for query using lamda + List modes = enumModes.stream().map(Mode::getModeName).toList(); + return getLatestForInterrogationListV2(questionnaireId, modes, interrogationIds); } - //========= OPTIMISATIONS PERFS (START) ========== /** * @author Adrien Marchal */ @Operation(summary = "Retrieve all responses for a questionnaire and a list of UE", description = "Return the latest state for each variable for the given ids and a given questionnaire.
" + "For a given id, the endpoint returns a document by collection mode (if there is more than one).") - @PostMapping(path = "/simplified/by-list-interrogation-and-questionnaire/latestV2") + @PostMapping(path = "/simplified/by-list-interrogation-and-questionnaire-and-modes/latest") @PreAuthorize("hasRole('USER_KRAFTWERK')") public ResponseEntity> getLatestForInterrogationListV2(@RequestParam("questionnaireId") String questionnaireId, @RequestParam List modes, diff --git a/src/main/java/fr/insee/genesis/domain/ports/api/SurveyUnitApiPort.java b/src/main/java/fr/insee/genesis/domain/ports/api/SurveyUnitApiPort.java index 74838b2c..26f246a8 100644 --- a/src/main/java/fr/insee/genesis/domain/ports/api/SurveyUnitApiPort.java +++ b/src/main/java/fr/insee/genesis/domain/ports/api/SurveyUnitApiPort.java @@ -25,6 +25,7 @@ public interface SurveyUnitApiPort { Stream findByQuestionnaireId(String questionnaireId); + //NEW VERSION OF THE METHOD List findLatestByIdAndByQuestionnaireId(String interrogationId, String questionnaireId); //========= OPTIMISATIONS PERFS (START) ========== @@ -35,6 +36,9 @@ public interface SurveyUnitApiPort { List findInterrogationIdsAndModesByQuestionnaireId(String questionnaireId); + /** + * !!!WARNING!!! : A CALL WITH THIS ENDPOINT ON A BIG COLLECTION (> 300k) MAY KILL THE GENESIS-API APP.!!! + */ List findDistinctInterrogationIdsByQuestionnaireId(String questionnaireId); //========= OPTIMISATIONS PERFS (START) ========== @@ -46,9 +50,6 @@ List findDistinctPageableInterrogationIdsByQuestionnaireId(Stri List findModesByQuestionnaireIdV2(String questionnaireId); //========= OPTIMISATIONS PERFS (END) ========== - List findModesByQuestionnaireId(String questionnaireId); - - List findModesByCampaignId(String campaignId); //========= OPTIMISATIONS PERFS (START) ========== List findModesByCampaignIdV2(String campaignId); @@ -58,7 +59,6 @@ List findDistinctPageableInterrogationIdsByQuestionnaireId(Stri long countResponses(); - Set findQuestionnaireIdsByCampaignId(String campaignId); //========= OPTIMISATIONS PERFS (START) ========== /** @@ -73,7 +73,9 @@ List findDistinctPageableInterrogationIdsByQuestionnaireId(Stri Set findDistinctQuestionnaireIds(); - List findCampaignsWithQuestionnaires(); + //========= OPTIMISATIONS PERFS (START) ========== + List findCampaignsWithQuestionnairesV2(); + //========= OPTIMISATIONS PERFS (END) ========== List findQuestionnairesWithCampaigns(); diff --git a/src/main/java/fr/insee/genesis/domain/ports/spi/SurveyUnitPersistencePort.java b/src/main/java/fr/insee/genesis/domain/ports/spi/SurveyUnitPersistencePort.java index 7f69b753..9a8ff8fb 100644 --- a/src/main/java/fr/insee/genesis/domain/ports/spi/SurveyUnitPersistencePort.java +++ b/src/main/java/fr/insee/genesis/domain/ports/spi/SurveyUnitPersistencePort.java @@ -25,6 +25,9 @@ public interface SurveyUnitPersistencePort { Stream findByQuestionnaireId(String questionnaireId); + /** + * !!!WARNING!!! : A CALL WITH THIS ENDPOINT ON A BIG COLLECTION (> 300k) MAY KILL THE GENESIS-API APP.!!! + */ List findInterrogationIdsByQuestionnaireId(String questionnaireId); //======== OPTIMISATIONS PERFS (START) ======== @@ -37,13 +40,11 @@ public interface SurveyUnitPersistencePort { List findModesByQuestionnaireIdV2(String questionnaireId); //======= OPTIMISATIONS PERFS (END) ========= - List findInterrogationIdsByCampaignId(String campaignId); Long deleteByQuestionnaireId(String questionnaireId); long count(); - Set findQuestionnaireIdsByCampaignId(String campaignId); //========= OPTIMISATIONS PERFS (START) ========== /** diff --git a/src/main/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitService.java b/src/main/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitService.java index 186070e9..b2e3e990 100644 --- a/src/main/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitService.java +++ b/src/main/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitService.java @@ -63,6 +63,9 @@ public Stream findByQuestionnaireId(String questionnaireId) { return surveyUnitPersistencePort.findByQuestionnaireId(questionnaireId); } + + + //========= OPTIMISATIONS PERFS (START) ========== /** * In this method we want to get the latest update for each variable of a survey unit * But we need to separate the updates by mode @@ -73,69 +76,31 @@ public Stream findByQuestionnaireId(String questionnaireId) { */ @Override public List findLatestByIdAndByQuestionnaireId(String interrogationId, String questionnaireId) { - List latestUpdatesbyVariables = new ArrayList<>(); - List surveyUnitModels = surveyUnitPersistencePort.findByIds(interrogationId, questionnaireId); - List modes = getDistinctsModes(surveyUnitModels); - modes.forEach(mode ->{ - List suByMode = surveyUnitModels.stream() - .filter(surveyUnitModel -> surveyUnitModel.getMode().equals(mode)) - .sorted((o1, o2) -> o2.getRecordDate().compareTo(o1.getRecordDate())) //Sorting update by date (latest updates first by date of upload in database) - .toList(); - - //We had all the variables of the oldest update - latestUpdatesbyVariables.add(suByMode.getFirst()); - //We keep the name of already added variables to skip them in older updates - List addedVariables = new ArrayList<>(); - SurveyUnitModel latestUpdate = suByMode.getFirst(); - - if(latestUpdate.getCollectedVariables() == null){ - latestUpdate.setCollectedVariables(new ArrayList<>()); - } - if(latestUpdate.getExternalVariables() == null){ - latestUpdate.setExternalVariables(new ArrayList<>()); + List enumModes = findModesByQuestionnaireIdV2(questionnaireId); + // => convertion of "List" -> "List" for query using lamda + List modes = enumModes.stream().map(Mode::getModeName).toList(); + List interrogationIds = List.of(new InterrogationId(interrogationId)); + + List responses = new ArrayList<>(); + for(String mode : modes) { + List> surveyUnitModels = findLatestByIdAndByQuestionnaireIdAndModeOrdered(questionnaireId, mode, interrogationIds); + for(List singleSurveyUnitModel : surveyUnitModels) { + responses.addAll(singleSurveyUnitModel); } + } - latestUpdate.getCollectedVariables().forEach(colVar -> addedVariables.add(new VarIdScopeTuple(colVar.varId(), - colVar.scope(), colVar.iteration()))); - latestUpdate.getExternalVariables().forEach(extVar -> addedVariables.add(new VarIdScopeTuple(extVar.varId(), extVar.scope(), extVar.iteration()))); - - suByMode.forEach(surveyUnitModel -> { - List collectedVariablesToKeep = new ArrayList<>(); - List externalVariablesToKeep = new ArrayList<>(); - // We iterate over the variables of the update and add them to the list if they are not already added - if (surveyUnitModel.getCollectedVariables() != null) { - surveyUnitModel.getCollectedVariables().stream() - .filter(colVar -> !addedVariables.contains(new VarIdScopeTuple(colVar.varId(), colVar.scope() - , colVar.iteration()))) - .forEach(colVar -> { - collectedVariablesToKeep.add(colVar); - addedVariables.add(new VarIdScopeTuple(colVar.varId(), colVar.scope(), colVar.iteration())); - }); - } - if (surveyUnitModel.getExternalVariables() != null){ - surveyUnitModel.getExternalVariables().stream() - .filter(extVar -> !addedVariables.contains(new VarIdScopeTuple(extVar.varId(), extVar.scope(), - extVar.iteration()))) - .forEach(extVar -> { - externalVariablesToKeep.add(extVar); - addedVariables.add(new VarIdScopeTuple(extVar.varId(), extVar.scope(), extVar.iteration())); - }); - } - - // If there are new variables, we add the update to the list of latest updates - if (!collectedVariablesToKeep.isEmpty() || !externalVariablesToKeep.isEmpty()){ - surveyUnitModel.setCollectedVariables(collectedVariablesToKeep); - surveyUnitModel.setExternalVariables(externalVariablesToKeep); - latestUpdatesbyVariables.add(surveyUnitModel); - } - }); - }); - return latestUpdatesbyVariables; + return responses; } - //========= OPTIMISATIONS PERFS (START) ========== /** + * In this method we want to get the latest update for each variable of a survey unit + * But we need to separate the updates by mode + * So we will calculate the latest state for a given collection mode + * @param questionnaireId : Questionnaire id + * @param mode : collect mode + * @param interrogationIds : !!!A LIST OF!!! Survey unit ids + * @return the latest update for each variable of a survey unit * @author Adrien Marchal */ @Override @@ -146,12 +111,6 @@ public List> findLatestByIdAndByQuestionnaireIdAndModeOrde //1) QUERY // => convertion of "List" -> "List" for query using lamda - /* - List queryInParam = new ArrayList(); - for(InterrogationId interrId : interrogationIds) { - queryInParam.add(interrId.getInterrogationId()); - } - */ List queryInParam = interrogationIds.stream().map(InterrogationId::getInterrogationId).collect(Collectors.toList()); //Get !!!all versions!!! of a set of "interrogationIds" @@ -248,6 +207,9 @@ public SurveyUnitDto findLatestValuesByStateByIdAndByQuestionnaireId(String inte return surveyUnitDto; } + /** + * !!!WARNING!!! : A CALL WITH THIS ENDPOINT ON A BIG COLLECTION (> 300k) MAY KILL THE GENESIS-API APP.!!! + */ @Override public List findDistinctInterrogationIdsByQuestionnaireId(String questionnaireId) { List surveyUnitModels = surveyUnitPersistencePort.findInterrogationIdsByQuestionnaireId(questionnaireId); @@ -256,6 +218,7 @@ public List findDistinctInterrogationIdsByQuestionnaireId(Strin return suIds.stream().distinct().toList(); } + //============ OPTIMISATIONS PERFS (START) ============ /** @@ -298,27 +261,15 @@ public long countInterrogationIdsByQuestionnaireId(String questionnaireId) { //=========== OPTIMISATIONS PERFS (END) ============= + /** + * !!!WARNING!!! : A CALL WITH THIS ENDPOINT ON A BIG COLLECTION (> 300k) MAY KILL THE GENESIS-API APP.!!! + */ @Override public List findInterrogationIdsAndModesByQuestionnaireId(String questionnaireId) { List surveyUnitModels = surveyUnitPersistencePort.findInterrogationIdsByQuestionnaireId(questionnaireId); return surveyUnitModels.stream().distinct().toList(); } - @Override - public List findModesByQuestionnaireId(String questionnaireId) { - List surveyUnitModels = surveyUnitPersistencePort.findInterrogationIdsByQuestionnaireId(questionnaireId); - List sources = new ArrayList<>(); - surveyUnitModels.forEach(surveyUnitModel -> sources.add(surveyUnitModel.getMode())); - return sources.stream().distinct().toList(); - } - - @Override - public List findModesByCampaignId(String campaignId) { - List surveyUnitModels = surveyUnitPersistencePort.findInterrogationIdsByCampaignId(campaignId); - List sources = new ArrayList<>(); - surveyUnitModels.forEach(surveyUnitModel -> sources.add(surveyUnitModel.getMode())); - return sources.stream().distinct().toList(); - } //========= OPTIMISATIONS PERFS (START) ========== @Override @@ -348,10 +299,6 @@ public long countResponses() { return surveyUnitPersistencePort.count(); } - @Override - public Set findQuestionnaireIdsByCampaignId(String campaignId) { - return surveyUnitPersistencePort.findQuestionnaireIdsByCampaignId(campaignId); - } //========= OPTIMISATIONS PERFS (START) ========== /** @@ -368,15 +315,18 @@ public Set findDistinctCampaignIds() { return surveyUnitPersistencePort.findDistinctCampaignIds(); } + + //========= OPTIMISATIONS PERFS (START) ========== @Override - public List findCampaignsWithQuestionnaires() { + public List findCampaignsWithQuestionnairesV2() { List campaignsWithQuestionnaireList = new ArrayList<>(); for(String campaignId : findDistinctCampaignIds()){ - Set questionnaires = findQuestionnaireIdsByCampaignId(campaignId); + Set questionnaires = findQuestionnaireIdsByCampaignIdV2(campaignId); campaignsWithQuestionnaireList.add(new CampaignWithQuestionnaire(campaignId,questionnaires)); } return campaignsWithQuestionnaireList; } + //========= OPTIMISATIONS PERFS (END) ========== @Override public long countResponsesByCampaignId(String campaignId){ diff --git a/src/main/java/fr/insee/genesis/infrastructure/adapter/SurveyUnitMongoAdapter.java b/src/main/java/fr/insee/genesis/infrastructure/adapter/SurveyUnitMongoAdapter.java index 5eec2c2c..1fa80d8d 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/adapter/SurveyUnitMongoAdapter.java +++ b/src/main/java/fr/insee/genesis/infrastructure/adapter/SurveyUnitMongoAdapter.java @@ -93,25 +93,6 @@ public long count() { return mongoRepository.count(); } - @Override - public Set findQuestionnaireIdsByCampaignId(String campaignId){ - Set mongoResponse = - mongoRepository.findQuestionnaireIdsByCampaignId(campaignId); - - //Extract questionnaireIds from JSON response - Set questionnaireIds = new HashSet<>(); - for(String line : mongoResponse){ - ObjectMapper objectMapper = new ObjectMapper().findAndRegisterModules(); - try{ - JsonNode jsonNode = objectMapper.readTree(line); - questionnaireIds.add(jsonNode.get("questionnaireId").asText()); - }catch (JsonProcessingException e){ - log.error(e.getMessage()); - } - } - - return questionnaireIds; - } //========= OPTIMISATIONS PERFS (START) ========== /** @@ -148,6 +129,9 @@ public Set findDistinctCampaignIds() { return campaignIds; } + /** + * !!!WARNING!!! : A CALL WITH THIS ENDPOINT ON A BIG COLLECTION (> 300k) MAY KILL THE GENESIS-API APP.!!! + */ @Override public List findInterrogationIdsByQuestionnaireId(String questionnaireId) { List surveyUnits = mongoRepository.findInterrogationIdsByQuestionnaireId(questionnaireId); @@ -180,12 +164,6 @@ public List findModesByQuestionnaireIdV2(String questionnaireId } //========== OPTIMISATIONS PERFS (END) ============ - @Override - public List findInterrogationIdsByCampaignId(String campaignId) { - List surveyUnits = mongoRepository.findInterrogationIdsByCampaignId(campaignId); - return surveyUnits.isEmpty() ? Collections.emptyList() : SurveyUnitDocumentMapper.INSTANCE.listDocumentToListModel(surveyUnits); - - } public long countByCampaignId(String campaignId){ return mongoRepository.countByCampaignId(campaignId); diff --git a/src/main/java/fr/insee/genesis/infrastructure/repository/SurveyUnitMongoDBRepository.java b/src/main/java/fr/insee/genesis/infrastructure/repository/SurveyUnitMongoDBRepository.java index dc915db9..c1ac3086 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/repository/SurveyUnitMongoDBRepository.java +++ b/src/main/java/fr/insee/genesis/infrastructure/repository/SurveyUnitMongoDBRepository.java @@ -26,6 +26,9 @@ public interface SurveyUnitMongoDBRepository extends MongoRepository findBySetOfIdsAndQuestionnaireIdAndMode(String questionnaireId, String mode, List interrogationIdSet); //========= OPTIMISATIONS PERFS (END) ========== + /** + * !!!WARNING!!! : A CALL WITH THIS ENDPOINT ON A BIG COLLECTION (> 300k) MAY KILL THE GENESIS-API APP.!!! + */ @Query(value = "{ 'questionnaireId' : ?0 }", fields = "{ 'interrogationId' : 1, 'mode' : 1 }") List findInterrogationIdsByQuestionnaireId(String questionnaireId); @@ -60,11 +63,9 @@ public interface SurveyUnitMongoDBRepository extends MongoRepository findModesByQuestionnaireIdV2(String campaignId); + List findModesByQuestionnaireIdV2(String questionnaireId); //========= OPTIMISATIONS PERFS (END) ========== - @Query(value = "{ 'campaignId' : ?0 }", fields = "{ 'interrogationId' : 1, 'mode' : 1 }") - List findInterrogationIdsByCampaignId(String campaignId); Long deleteByQuestionnaireId(String questionnaireId); @@ -73,8 +74,6 @@ public interface SurveyUnitMongoDBRepository extends MongoRepository findQuestionnaireIdsByCampaignId(String campaignId); //========= OPTIMISATIONS PERFS (START) ========== /** diff --git a/src/test/java/fr/insee/genesis/controller/rest/responses/ModeControllerTest.java b/src/test/java/fr/insee/genesis/controller/rest/responses/ModeControllerTest.java index a1585784..855c58a3 100644 --- a/src/test/java/fr/insee/genesis/controller/rest/responses/ModeControllerTest.java +++ b/src/test/java/fr/insee/genesis/controller/rest/responses/ModeControllerTest.java @@ -38,7 +38,7 @@ void reset() throws IOException { //When + Then @Test void getModesByQuestionnaireTest() { - ResponseEntity> response = modeControllerStatic.getModesByQuestionnaire(DEFAULT_QUESTIONNAIRE_ID); + ResponseEntity> response = modeControllerStatic.getModesByQuestionnaireV2(DEFAULT_QUESTIONNAIRE_ID); Assertions.assertThat(response.getStatusCode().is2xxSuccessful()).isTrue(); Assertions.assertThat(response.getBody()).isNotNull().isNotEmpty().hasSize(1); @@ -47,7 +47,7 @@ void getModesByQuestionnaireTest() { @Test void getModesByCampaignTest() { - ResponseEntity> response = modeControllerStatic.getModesByCampaign("TESTCAMPAIGNID"); + ResponseEntity> response = modeControllerStatic.getModesByCampaignV2("TESTCAMPAIGNID"); Assertions.assertThat(response.getStatusCode().is2xxSuccessful()).isTrue(); Assertions.assertThat(response.getBody()).isNotNull().isNotEmpty().hasSize(1); diff --git a/src/test/java/fr/insee/genesis/controller/rest/responses/QuestionnaireControllerTest.java b/src/test/java/fr/insee/genesis/controller/rest/responses/QuestionnaireControllerTest.java index de1767a0..995644c2 100644 --- a/src/test/java/fr/insee/genesis/controller/rest/responses/QuestionnaireControllerTest.java +++ b/src/test/java/fr/insee/genesis/controller/rest/responses/QuestionnaireControllerTest.java @@ -53,7 +53,7 @@ void getQuestionnairesTest() { void getQuestionnairesByCampaignTest() { Utils.addAdditionalSurveyUnitModelToMongoStub("TESTQUESTIONNAIRE2", surveyUnitPersistencePortStub); - ResponseEntity> response = questionnaireControllerStatic.getQuestionnairesByCampaign("TESTCAMPAIGNID"); + ResponseEntity> response = questionnaireControllerStatic.getQuestionnairesByCampaignV2("TESTCAMPAIGNID"); Assertions.assertThat(response.getStatusCode().is2xxSuccessful()).isTrue(); Assertions.assertThat(response.getBody()).isNotNull().isNotEmpty().containsOnly( diff --git a/src/test/java/fr/insee/genesis/domain/service/SurveyUnitServiceTest.java b/src/test/java/fr/insee/genesis/domain/service/SurveyUnitServiceTest.java index b07378c6..3d6e220c 100644 --- a/src/test/java/fr/insee/genesis/domain/service/SurveyUnitServiceTest.java +++ b/src/test/java/fr/insee/genesis/domain/service/SurveyUnitServiceTest.java @@ -236,12 +236,6 @@ void findDistinctInterrogationIdsByQuestionnaireIdTest(){ ).isNotEmpty().hasSize(1); } - @Test - void findInterrogationIdsByQuestionnaireIdTest(){ - Assertions.assertThat(surveyUnitServiceStatic.findModesByQuestionnaireId(DEFAULT_QUESTIONNAIRE_ID)).filteredOn( - mode -> mode.equals(Mode.WEB) - ).isNotEmpty(); - } //========= OPTIMISATIONS PERFS (START) ========== @Test @@ -256,7 +250,7 @@ void findModesByQuestionnaireIdV2Test(){ void getQuestionnairesByCampaignTest() { addAdditionnalSurveyUnitModelToMongoStub("TESTQUESTIONNAIRE2"); - Assertions.assertThat(surveyUnitServiceStatic.findQuestionnaireIdsByCampaignId("TESTCAMPAIGNID")).isNotEmpty().hasSize(2); + Assertions.assertThat(surveyUnitServiceStatic.findQuestionnaireIdsByCampaignIdV2("TESTCAMPAIGNID")).isNotEmpty().hasSize(2); } diff --git a/src/test/java/fr/insee/genesis/stubs/SurveyUnitPersistencePortStub.java b/src/test/java/fr/insee/genesis/stubs/SurveyUnitPersistencePortStub.java index 0df561fe..b17db5b2 100644 --- a/src/test/java/fr/insee/genesis/stubs/SurveyUnitPersistencePortStub.java +++ b/src/test/java/fr/insee/genesis/stubs/SurveyUnitPersistencePortStub.java @@ -35,8 +35,13 @@ public List findByIds(String interrogationId, String questionna * @author Adrien Marchal */ public List findBySetOfIdsAndQuestionnaireIdAndMode(String questionnaireId, String mode, List interrogationIdSet) { - //TODO : TO BE IMPLEMENTED - return new ArrayList(); + List surveyUnitModelList = new ArrayList<>(); + for(SurveyUnitModel SurveyUnitModel : mongoStub){ + if(interrogationIdSet.contains(SurveyUnitModel.getInterrogationId()) && SurveyUnitModel.getQuestionnaireId().equals(questionnaireId)) + surveyUnitModelList.add(SurveyUnitModel); + } + + return surveyUnitModelList; } //========= OPTIMISATIONS PERFS (START) ========== @@ -76,6 +81,9 @@ public Stream findByQuestionnaireId(String questionnaireId) { return surveyUnitModelList.stream(); } + /** + * !!!WARNING!!! : A CALL WITH THIS ENDPOINT ON A BIG COLLECTION (> 300k) MAY KILL THE GENESIS-API APP.!!! + */ @Override public List findInterrogationIdsByQuestionnaireId(String questionnaireId) { List surveyUnitModelList = new ArrayList<>(); @@ -103,8 +111,11 @@ public long countInterrogationIdsByQuestionnaireId(String questionnaireId) { */ public List findPageableInterrogationIdsByQuestionnaireId(String questionnaireId, Long skip, Long limit) { List surveyUnitModelList = new ArrayList<>(); - if(skip < mongoStub.size()) { - + for(SurveyUnitModel SurveyUnitModel : mongoStub){ + if(SurveyUnitModel.getQuestionnaireId().equals(questionnaireId)) + surveyUnitModelList.add( + new SurveyUnitModel(SurveyUnitModel.getInterrogationId(), SurveyUnitModel.getMode()) + ); } return surveyUnitModelList; } @@ -138,18 +149,6 @@ public List findModesByQuestionnaireIdV2(String questionnaireId //======= OPTIMISATIONS PERFS (END) ========= - @Override - public List findInterrogationIdsByCampaignId(String campaignId) { - List surveyUnitModelList = new ArrayList<>(); - for(SurveyUnitModel SurveyUnitModel : mongoStub){ - if(SurveyUnitModel.getCampaignId().equals(campaignId)) - surveyUnitModelList.add( - new SurveyUnitModel(SurveyUnitModel.getInterrogationId(), SurveyUnitModel.getMode()) - ); - } - - return surveyUnitModelList; - } @Override public Long deleteByQuestionnaireId(String questionnaireId) { @@ -161,8 +160,10 @@ public long count() { return mongoStub.size(); } + + //========= OPTIMISATIONS PERFS (START) ========== @Override - public Set findQuestionnaireIdsByCampaignId(String campaignId) { + public Set findQuestionnaireIdsByCampaignIdV2(String campaignId) { Set questionnaireIdSet = new HashSet<>(); for(SurveyUnitModel SurveyUnitModel : mongoStub){ if(SurveyUnitModel.getCampaignId().equals(campaignId)) @@ -171,13 +172,6 @@ public Set findQuestionnaireIdsByCampaignId(String campaignId) { return questionnaireIdSet; } - - //========= OPTIMISATIONS PERFS (START) ========== - @Override - public Set findQuestionnaireIdsByCampaignIdV2(String campaignId) { - //This stub is explicitally the same as method "findQuestionnaireIdsByCampaignId()" - return findQuestionnaireIdsByCampaignId(campaignId); - } //========= OPTIMISATIONS PERFS (END) ========== @Override From f3478aebf39477b9aa7708648e6d02f99dfcd3ea Mon Sep 17 00:00:00 2001 From: Adrien MARCHAL Date: Wed, 23 Jul 2025 14:56:22 +0200 Subject: [PATCH 2/6] !!!WARNING!!! - REMOVE ALL V2 REFS --- .../rest/responses/CampaignController.java | 2 +- .../responses/InterrogationController.java | 7 ++-- .../rest/responses/ModeController.java | 15 ++++----- .../responses/QuestionnaireController.java | 9 +++-- .../rest/responses/ResponseController.java | 10 ++---- .../domain/ports/api/SurveyUnitApiPort.java | 18 +++------- .../ports/spi/SurveyUnitPersistencePort.java | 12 ++----- .../service/surveyunit/SurveyUnitService.java | 33 +++++++------------ .../adapter/SurveyUnitMongoAdapter.java | 21 +++++------- .../SurveyUnitMongoDBRepository.java | 16 ++++----- .../rest/responses/ModeControllerTest.java | 24 ++------------ .../QuestionnaireControllerTest.java | 2 +- .../domain/service/SurveyUnitServiceTest.java | 9 +++-- .../stubs/SurveyUnitPersistencePortStub.java | 15 +++------ 14 files changed, 64 insertions(+), 129 deletions(-) diff --git a/src/main/java/fr/insee/genesis/controller/rest/responses/CampaignController.java b/src/main/java/fr/insee/genesis/controller/rest/responses/CampaignController.java index 77b77266..9daaa129 100644 --- a/src/main/java/fr/insee/genesis/controller/rest/responses/CampaignController.java +++ b/src/main/java/fr/insee/genesis/controller/rest/responses/CampaignController.java @@ -35,7 +35,7 @@ public ResponseEntity> getCampaigns() { @Operation(summary = "List campaigns in database with their questionnaires") @GetMapping(path = "/with-questionnaires") public ResponseEntity> getCampaignsWithQuestionnaires() { - List questionnairesByCampaigns = surveyUnitService.findCampaignsWithQuestionnairesV2(); + List questionnairesByCampaigns = surveyUnitService.findCampaignsWithQuestionnaires(); return ResponseEntity.ok(questionnairesByCampaigns); } diff --git a/src/main/java/fr/insee/genesis/controller/rest/responses/InterrogationController.java b/src/main/java/fr/insee/genesis/controller/rest/responses/InterrogationController.java index 1ef6d67a..2830c33c 100644 --- a/src/main/java/fr/insee/genesis/controller/rest/responses/InterrogationController.java +++ b/src/main/java/fr/insee/genesis/controller/rest/responses/InterrogationController.java @@ -28,8 +28,9 @@ public InterrogationController(SurveyUnitApiPort surveyUnitService) { } - //NOTE : methode used (It has to be kept with V2 functions!) - //!!!WARNING!!! : A CALL WITH THIS ENDPOINT ON A BIG COLLECTION (> 300k) MAY KILL THE GENESIS-API APP.!!! + /** + * !!!WARNING!!! : A CALL WITH THIS ENDPOINT ON A BIG COLLECTION (> 300k) MAY KILL THE GENESIS-API APP.!!! + */ @Operation(summary = "Retrieve all interrogations for a given questionnaire") @GetMapping(path = "/by-questionnaire") public ResponseEntity> getAllInterrogationIdsByQuestionnaire(@RequestParam("questionnaireId") String questionnaireId) { @@ -38,7 +39,6 @@ public ResponseEntity> getAllInterrogationIdsByQuestionnai } - //========= OPTIMISATIONS PERFS (START) ========== /** * @author Adrien Marchal */ @@ -65,7 +65,6 @@ public ResponseEntity> getPaginatedInterrogationIdsByQuest List responses = surveyUnitService.findDistinctPageableInterrogationIdsByQuestionnaireId(questionnaireId, totalSize, blockSize, page); return ResponseEntity.ok(responses); } - //======== OPTIMISATIONS PERFS (END) =========== } diff --git a/src/main/java/fr/insee/genesis/controller/rest/responses/ModeController.java b/src/main/java/fr/insee/genesis/controller/rest/responses/ModeController.java index 56609806..f093943c 100644 --- a/src/main/java/fr/insee/genesis/controller/rest/responses/ModeController.java +++ b/src/main/java/fr/insee/genesis/controller/rest/responses/ModeController.java @@ -27,21 +27,18 @@ public ModeController(SurveyUnitApiPort surveyUnitService) { } - - //========= OPTIMISATIONS PERFS (START) ========== @Operation(summary = "List sources/modes used for a given questionnaire") - @GetMapping(path = "/by-questionnaireV2") - public ResponseEntity> getModesByQuestionnaireV2(@RequestParam("questionnaireId") String questionnaireId) { - List modes = surveyUnitService.findModesByQuestionnaireIdV2(questionnaireId); + @GetMapping(path = "/by-questionnaire") + public ResponseEntity> getModesByQuestionnaire(@RequestParam("questionnaireId") String questionnaireId) { + List modes = surveyUnitService.findModesByQuestionnaireId(questionnaireId); return ResponseEntity.ok(modes); } @Operation(summary = "List sources/modes used for a given campaign") - @GetMapping(path = "/by-campaignV2") - public ResponseEntity> getModesByCampaignV2(@RequestParam("campaignId") String campaignId) { - List modes = surveyUnitService.findModesByCampaignIdV2(campaignId); + @GetMapping(path = "/by-campaign") + public ResponseEntity> getModesByCampaign(@RequestParam("campaignId") String campaignId) { + List modes = surveyUnitService.findModesByCampaignId(campaignId); return ResponseEntity.ok(modes); } - //========= OPTIMISATIONS PERFS (END) ========== } diff --git a/src/main/java/fr/insee/genesis/controller/rest/responses/QuestionnaireController.java b/src/main/java/fr/insee/genesis/controller/rest/responses/QuestionnaireController.java index 6b631e83..36f35888 100644 --- a/src/main/java/fr/insee/genesis/controller/rest/responses/QuestionnaireController.java +++ b/src/main/java/fr/insee/genesis/controller/rest/responses/QuestionnaireController.java @@ -47,17 +47,16 @@ public ResponseEntity> getQuestionnairesWithCamp } - //========= OPTIMISATIONS PERFS (START) ========== /** * @author Adrien Marchal */ @Operation(summary = "List questionnaires used for a given campaign (using a DISTINCT query)") - @GetMapping(path = "/by-campaignV2") - public ResponseEntity> getQuestionnairesByCampaignV2(@RequestParam("campaignId") String campaignId) { - Set questionnaires = surveyUnitService.findQuestionnaireIdsByCampaignIdV2(campaignId); + @GetMapping(path = "/by-campaign") + public ResponseEntity> getQuestionnairesByCampaign(@RequestParam("campaignId") String campaignId) { + Set questionnaires = surveyUnitService.findQuestionnaireIdsByCampaignId(campaignId); return ResponseEntity.ok(questionnaires); } - //========= OPTIMISATIONS PERFS (END) ========== + @Operation(summary = "Get the questionnaireId corresponding to an interrogationId") @GetMapping(path = "/by-interrogation") diff --git a/src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java b/src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java index d524dda0..d0ee49e9 100644 --- a/src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java +++ b/src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java @@ -302,9 +302,6 @@ public ResponseEntity getLatestByInterrogationOneObject(@R } - - - //========= OPTIMISATIONS PERFS (START) ========== /** * @author Adrien Marchal */ @@ -315,10 +312,10 @@ public ResponseEntity getLatestByInterrogationOneObject(@R @PreAuthorize("hasRole('USER_KRAFTWERK')") public ResponseEntity> getLatestForInterrogationList(@RequestParam("questionnaireId") String questionnaireId, @RequestBody List interrogationIds) { - List enumModes = surveyUnitService.findModesByQuestionnaireIdV2(questionnaireId); + List enumModes = surveyUnitService.findModesByQuestionnaireId(questionnaireId); // => convertion of "List" -> "List" for query using lamda List modes = enumModes.stream().map(Mode::getModeName).toList(); - return getLatestForInterrogationListV2(questionnaireId, modes, interrogationIds); + return getLatestForInterrogationListWithModes(questionnaireId, modes, interrogationIds); } @@ -330,7 +327,7 @@ public ResponseEntity> getLatestForInterrogationList( "For a given id, the endpoint returns a document by collection mode (if there is more than one).") @PostMapping(path = "/simplified/by-list-interrogation-and-questionnaire-and-modes/latest") @PreAuthorize("hasRole('USER_KRAFTWERK')") - public ResponseEntity> getLatestForInterrogationListV2(@RequestParam("questionnaireId") String questionnaireId, + public ResponseEntity> getLatestForInterrogationListWithModes(@RequestParam("questionnaireId") String questionnaireId, @RequestParam List modes, @RequestBody List interrogationIds) { List results = new ArrayList<>(); @@ -397,7 +394,6 @@ private SurveyUnitSimplified fusionWithLastUpdated(List respons return simplifiedResponse; } - //========= OPTIMISATIONS PERFS (END) ========== @Operation(summary = "Save edited variables", diff --git a/src/main/java/fr/insee/genesis/domain/ports/api/SurveyUnitApiPort.java b/src/main/java/fr/insee/genesis/domain/ports/api/SurveyUnitApiPort.java index 26f246a8..ddfe3ee0 100644 --- a/src/main/java/fr/insee/genesis/domain/ports/api/SurveyUnitApiPort.java +++ b/src/main/java/fr/insee/genesis/domain/ports/api/SurveyUnitApiPort.java @@ -28,9 +28,7 @@ public interface SurveyUnitApiPort { //NEW VERSION OF THE METHOD List findLatestByIdAndByQuestionnaireId(String interrogationId, String questionnaireId); - //========= OPTIMISATIONS PERFS (START) ========== List> findLatestByIdAndByQuestionnaireIdAndModeOrdered(String questionnaireId, String mode, List interrogationIds); - //========= OPTIMISATIONS PERFS (END) ========== SurveyUnitDto findLatestValuesByStateByIdAndByQuestionnaireId(String interrogationId, String questionnaireId); @@ -41,31 +39,25 @@ public interface SurveyUnitApiPort { */ List findDistinctInterrogationIdsByQuestionnaireId(String questionnaireId); - //========= OPTIMISATIONS PERFS (START) ========== long countInterrogationIdsByQuestionnaireId(String questionnaireId); List findDistinctPageableInterrogationIdsByQuestionnaireId(String questionnaireId, long totalSize, long blockSize, long page); - List findModesByQuestionnaireIdV2(String questionnaireId); - //========= OPTIMISATIONS PERFS (END) ========== + List findModesByQuestionnaireId(String questionnaireId); - //========= OPTIMISATIONS PERFS (START) ========== - List findModesByCampaignIdV2(String campaignId); - //========= OPTIMISATIONS PERFS (END) ========== + List findModesByCampaignId(String campaignId); Long deleteByQuestionnaireId(String questionnaireId); long countResponses(); - //========= OPTIMISATIONS PERFS (START) ========== /** * @author Adrien Marchal */ - Set findQuestionnaireIdsByCampaignIdV2(String campaignId); - //========= OPTIMISATIONS PERFS (END) ========== + Set findQuestionnaireIdsByCampaignId(String campaignId); Set findDistinctCampaignIds(); @@ -73,9 +65,7 @@ List findDistinctPageableInterrogationIdsByQuestionnaireId(Stri Set findDistinctQuestionnaireIds(); - //========= OPTIMISATIONS PERFS (START) ========== - List findCampaignsWithQuestionnairesV2(); - //========= OPTIMISATIONS PERFS (END) ========== + List findCampaignsWithQuestionnaires(); List findQuestionnairesWithCampaigns(); diff --git a/src/main/java/fr/insee/genesis/domain/ports/spi/SurveyUnitPersistencePort.java b/src/main/java/fr/insee/genesis/domain/ports/spi/SurveyUnitPersistencePort.java index 9a8ff8fb..d003180d 100644 --- a/src/main/java/fr/insee/genesis/domain/ports/spi/SurveyUnitPersistencePort.java +++ b/src/main/java/fr/insee/genesis/domain/ports/spi/SurveyUnitPersistencePort.java @@ -12,12 +12,10 @@ public interface SurveyUnitPersistencePort { List findByIds(String interrogationId, String questionnaireId); - //========= OPTIMISATIONS PERFS (START) ========== /** * @author Adrien Marchal */ List findBySetOfIdsAndQuestionnaireIdAndMode(String questionnaireId, String mode, List interrogationIdSet); - //========= OPTIMISATIONS PERFS (START) ========== List findByInterrogationId(String interrogationId); @@ -30,15 +28,13 @@ public interface SurveyUnitPersistencePort { */ List findInterrogationIdsByQuestionnaireId(String questionnaireId); - //======== OPTIMISATIONS PERFS (START) ======== long countInterrogationIdsByQuestionnaireId(String questionnaireId); List findPageableInterrogationIdsByQuestionnaireId(String questionnaireId, Long skip, Long limit); - List findModesByCampaignIdV2(String campaignId); + List findModesByCampaignId(String campaignId); - List findModesByQuestionnaireIdV2(String questionnaireId); - //======= OPTIMISATIONS PERFS (END) ========= + List findModesByQuestionnaireId(String questionnaireId); Long deleteByQuestionnaireId(String questionnaireId); @@ -46,12 +42,10 @@ public interface SurveyUnitPersistencePort { long count(); - //========= OPTIMISATIONS PERFS (START) ========== /** * @author Adrien Marchal */ - Set findQuestionnaireIdsByCampaignIdV2(String campaignId); - //========= OPTIMISATIONS PERFS (END) ========== + Set findQuestionnaireIdsByCampaignId(String campaignId); Set findDistinctCampaignIds(); diff --git a/src/main/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitService.java b/src/main/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitService.java index b2e3e990..828fa89c 100644 --- a/src/main/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitService.java +++ b/src/main/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitService.java @@ -64,8 +64,6 @@ public Stream findByQuestionnaireId(String questionnaireId) { } - - //========= OPTIMISATIONS PERFS (START) ========== /** * In this method we want to get the latest update for each variable of a survey unit * But we need to separate the updates by mode @@ -76,7 +74,7 @@ public Stream findByQuestionnaireId(String questionnaireId) { */ @Override public List findLatestByIdAndByQuestionnaireId(String interrogationId, String questionnaireId) { - List enumModes = findModesByQuestionnaireIdV2(questionnaireId); + List enumModes = findModesByQuestionnaireId(questionnaireId); // => convertion of "List" -> "List" for query using lamda List modes = enumModes.stream().map(Mode::getModeName).toList(); List interrogationIds = List.of(new InterrogationId(interrogationId)); @@ -179,7 +177,6 @@ private List extractLatestUpdates(List allResp return latestUpdatesbyVariables; } - //========= OPTIMISATIONS PERFS (END) ========== @Override @@ -219,8 +216,6 @@ public List findDistinctInterrogationIdsByQuestionnaireId(Strin } - //============ OPTIMISATIONS PERFS (START) ============ - /** * @author Adrien Marchal * Calculations made to establish the data a worker will be responsible of, among the whole data to be processed. @@ -258,7 +253,6 @@ public List findDistinctPageableInterrogationIdsByQuestionnaire public long countInterrogationIdsByQuestionnaireId(String questionnaireId) { return surveyUnitPersistencePort.countInterrogationIdsByQuestionnaireId(questionnaireId); } - //=========== OPTIMISATIONS PERFS (END) ============= /** @@ -271,23 +265,22 @@ public List findInterrogationIdsAndModesByQuestionnaireId(Strin } - //========= OPTIMISATIONS PERFS (START) ========== @Override - public List findModesByQuestionnaireIdV2(String questionnaireId) { - List surveyUnitModels = surveyUnitPersistencePort.findModesByQuestionnaireIdV2(questionnaireId); + public List findModesByQuestionnaireId(String questionnaireId) { + List surveyUnitModels = surveyUnitPersistencePort.findModesByQuestionnaireId(questionnaireId); List sources = new ArrayList<>(); surveyUnitModels.forEach(surveyUnitModel -> sources.add(surveyUnitModel.getMode())); return sources.stream().distinct().toList(); } @Override - public List findModesByCampaignIdV2(String campaignId) { - List surveyUnitModels = surveyUnitPersistencePort.findModesByCampaignIdV2(campaignId); + public List findModesByCampaignId(String campaignId) { + List surveyUnitModels = surveyUnitPersistencePort.findModesByCampaignId(campaignId); List sources = new ArrayList<>(); surveyUnitModels.forEach(surveyUnitModel -> sources.add(surveyUnitModel.getMode())); return sources.stream().distinct().toList(); } - //========= OPTIMISATIONS PERFS (END) ========== + @Override public Long deleteByQuestionnaireId(String questionnaireId) { @@ -300,15 +293,14 @@ public long countResponses() { } - //========= OPTIMISATIONS PERFS (START) ========== /** * @author Adrien Marchal */ @Override - public Set findQuestionnaireIdsByCampaignIdV2(String campaignId) { - return surveyUnitPersistencePort.findQuestionnaireIdsByCampaignIdV2(campaignId); + public Set findQuestionnaireIdsByCampaignId(String campaignId) { + return surveyUnitPersistencePort.findQuestionnaireIdsByCampaignId(campaignId); } - //========= OPTIMISATIONS PERFS (END) ========== + @Override public Set findDistinctCampaignIds() { @@ -316,17 +308,16 @@ public Set findDistinctCampaignIds() { } - //========= OPTIMISATIONS PERFS (START) ========== @Override - public List findCampaignsWithQuestionnairesV2() { + public List findCampaignsWithQuestionnaires() { List campaignsWithQuestionnaireList = new ArrayList<>(); for(String campaignId : findDistinctCampaignIds()){ - Set questionnaires = findQuestionnaireIdsByCampaignIdV2(campaignId); + Set questionnaires = findQuestionnaireIdsByCampaignId(campaignId); campaignsWithQuestionnaireList.add(new CampaignWithQuestionnaire(campaignId,questionnaires)); } return campaignsWithQuestionnaireList; } - //========= OPTIMISATIONS PERFS (END) ========== + @Override public long countResponsesByCampaignId(String campaignId){ diff --git a/src/main/java/fr/insee/genesis/infrastructure/adapter/SurveyUnitMongoAdapter.java b/src/main/java/fr/insee/genesis/infrastructure/adapter/SurveyUnitMongoAdapter.java index 1fa80d8d..a3c755f1 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/adapter/SurveyUnitMongoAdapter.java +++ b/src/main/java/fr/insee/genesis/infrastructure/adapter/SurveyUnitMongoAdapter.java @@ -49,7 +49,6 @@ public List findByIds(String interrogationId, String questionna } - //========= OPTIMISATIONS PERFS (START) ========== /** * @author Adrien Marchal */ @@ -58,7 +57,7 @@ public List findBySetOfIdsAndQuestionnaireIdAndMode(String ques List surveyUnits = mongoRepository.findBySetOfIdsAndQuestionnaireIdAndMode(questionnaireId, mode, interrogationIdSet); return surveyUnits.isEmpty() ? Collections.emptyList() : SurveyUnitDocumentMapper.INSTANCE.listDocumentToListModel(surveyUnits); } - //========= OPTIMISATIONS PERFS (END) ========== + @Override public List findByInterrogationId(String questionnaireId) { @@ -94,14 +93,13 @@ public long count() { } - //========= OPTIMISATIONS PERFS (START) ========== /** * @author Adrien Marchal */ @Override - public Set findQuestionnaireIdsByCampaignIdV2(String campaignId){ + public Set findQuestionnaireIdsByCampaignId(String campaignId){ Set mongoResponse = - mongoRepository.findQuestionnaireIdsByCampaignIdV2(campaignId); + mongoRepository.findQuestionnaireIdsByCampaignId(campaignId); //Extract questionnaireIds from JSON response Set questionnaireIds = new HashSet<>(); @@ -117,7 +115,7 @@ public Set findQuestionnaireIdsByCampaignIdV2(String campaignId){ return questionnaireIds; } - //========= OPTIMISATIONS PERFS (END) ========== + @Override public Set findDistinctCampaignIds() { @@ -138,7 +136,7 @@ public List findInterrogationIdsByQuestionnaireId(String questi return surveyUnits.isEmpty() ? Collections.emptyList() : SurveyUnitDocumentMapper.INSTANCE.listDocumentToListModel(surveyUnits); } - //========== OPTIMISATIONS PERFS (START) =========== + @Override public long countInterrogationIdsByQuestionnaireId(String questionnaireId) { return mongoRepository.countInterrogationIdsByQuestionnaireId(questionnaireId); @@ -151,18 +149,17 @@ public List findPageableInterrogationIdsByQuestionnaireId(Strin } @Override - public List findModesByCampaignIdV2(String campaignId) { - List surveyUnits = mongoRepository.findModesByCampaignIdV2(campaignId); + public List findModesByCampaignId(String campaignId) { + List surveyUnits = mongoRepository.findModesByCampaignId(campaignId); return surveyUnits.isEmpty() ? Collections.emptyList() : SurveyUnitDocumentMapper.INSTANCE.listDocumentToListModel(surveyUnits); } @Override - public List findModesByQuestionnaireIdV2(String questionnaireId) { - List surveyUnits = mongoRepository.findModesByQuestionnaireIdV2(questionnaireId); + public List findModesByQuestionnaireId(String questionnaireId) { + List surveyUnits = mongoRepository.findModesByQuestionnaireId(questionnaireId); return surveyUnits.isEmpty() ? Collections.emptyList() : SurveyUnitDocumentMapper.INSTANCE.listDocumentToListModel(surveyUnits); } - //========== OPTIMISATIONS PERFS (END) ============ public long countByCampaignId(String campaignId){ diff --git a/src/main/java/fr/insee/genesis/infrastructure/repository/SurveyUnitMongoDBRepository.java b/src/main/java/fr/insee/genesis/infrastructure/repository/SurveyUnitMongoDBRepository.java index c1ac3086..86f7f7f3 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/repository/SurveyUnitMongoDBRepository.java +++ b/src/main/java/fr/insee/genesis/infrastructure/repository/SurveyUnitMongoDBRepository.java @@ -18,13 +18,13 @@ public interface SurveyUnitMongoDBRepository extends MongoRepository findByInterrogationIdAndQuestionnaireId(String interrogationId, String questionnaireId); - //========= OPTIMISATIONS PERFS (START) ========== + /** * @author Adrien Marchal */ @Query("{ 'questionnaireId' : ?0, 'mode' : ?1, 'interrogationId' : { $in: ?2 } }") List findBySetOfIdsAndQuestionnaireIdAndMode(String questionnaireId, String mode, List interrogationIdSet); - //========= OPTIMISATIONS PERFS (END) ========== + /** * !!!WARNING!!! : A CALL WITH THIS ENDPOINT ON A BIG COLLECTION (> 300k) MAY KILL THE GENESIS-API APP.!!! @@ -32,7 +32,7 @@ public interface SurveyUnitMongoDBRepository extends MongoRepository findInterrogationIdsByQuestionnaireId(String questionnaireId); - //========= OPTIMISATIONS PERFS (START) ========== + /** * @author Adrien Marchal */ @@ -56,15 +56,14 @@ public interface SurveyUnitMongoDBRepository extends MongoRepository findModesByCampaignIdV2(String campaignId); + List findModesByCampaignId(String campaignId); @Aggregation(pipeline = { "{ '$match': { 'questionnaireId' : ?0 } }", "{ '$group': { '_id': '$mode' } }", "{ '$set': { 'mode': '$_id', '_id': '$$REMOVE' } }" }) - List findModesByQuestionnaireIdV2(String questionnaireId); - //========= OPTIMISATIONS PERFS (END) ========== + List findModesByQuestionnaireId(String questionnaireId); Long deleteByQuestionnaireId(String questionnaireId); @@ -75,7 +74,6 @@ public interface SurveyUnitMongoDBRepository extends MongoRepository findQuestionnaireIdsByCampaignIdV2(String campaignId); - //========= OPTIMISATIONS PERFS (END) ========== + Set findQuestionnaireIdsByCampaignId(String campaignId); + long countByCampaignId(String campaignId); diff --git a/src/test/java/fr/insee/genesis/controller/rest/responses/ModeControllerTest.java b/src/test/java/fr/insee/genesis/controller/rest/responses/ModeControllerTest.java index 855c58a3..aca7ca46 100644 --- a/src/test/java/fr/insee/genesis/controller/rest/responses/ModeControllerTest.java +++ b/src/test/java/fr/insee/genesis/controller/rest/responses/ModeControllerTest.java @@ -38,7 +38,7 @@ void reset() throws IOException { //When + Then @Test void getModesByQuestionnaireTest() { - ResponseEntity> response = modeControllerStatic.getModesByQuestionnaireV2(DEFAULT_QUESTIONNAIRE_ID); + ResponseEntity> response = modeControllerStatic.getModesByQuestionnaire(DEFAULT_QUESTIONNAIRE_ID); Assertions.assertThat(response.getStatusCode().is2xxSuccessful()).isTrue(); Assertions.assertThat(response.getBody()).isNotNull().isNotEmpty().hasSize(1); @@ -47,31 +47,11 @@ void getModesByQuestionnaireTest() { @Test void getModesByCampaignTest() { - ResponseEntity> response = modeControllerStatic.getModesByCampaignV2("TESTCAMPAIGNID"); + ResponseEntity> response = modeControllerStatic.getModesByCampaign("TESTCAMPAIGNID"); Assertions.assertThat(response.getStatusCode().is2xxSuccessful()).isTrue(); Assertions.assertThat(response.getBody()).isNotNull().isNotEmpty().hasSize(1); Assertions.assertThat(response.getBody().getFirst()).isEqualTo(Mode.WEB); } - //========= OPTIMISATIONS PERFS (START) ========== - @Test - void getModesByQuestionnaireV2Test() { - ResponseEntity> response = modeControllerStatic.getModesByQuestionnaireV2(DEFAULT_QUESTIONNAIRE_ID); - - Assertions.assertThat(response.getStatusCode().is2xxSuccessful()).isTrue(); - Assertions.assertThat(response.getBody()).isNotNull().isNotEmpty().hasSize(1); - Assertions.assertThat(response.getBody().getFirst()).isEqualTo(Mode.WEB); - } - - @Test - void getModesByCampaignV2Test() { - ResponseEntity> response = modeControllerStatic.getModesByCampaignV2("TESTCAMPAIGNID"); - - Assertions.assertThat(response.getStatusCode().is2xxSuccessful()).isTrue(); - Assertions.assertThat(response.getBody()).isNotNull().isNotEmpty().hasSize(1); - Assertions.assertThat(response.getBody().getFirst()).isEqualTo(Mode.WEB); - } - //========= OPTIMISATIONS PERFS (END) ========== - } diff --git a/src/test/java/fr/insee/genesis/controller/rest/responses/QuestionnaireControllerTest.java b/src/test/java/fr/insee/genesis/controller/rest/responses/QuestionnaireControllerTest.java index 995644c2..de1767a0 100644 --- a/src/test/java/fr/insee/genesis/controller/rest/responses/QuestionnaireControllerTest.java +++ b/src/test/java/fr/insee/genesis/controller/rest/responses/QuestionnaireControllerTest.java @@ -53,7 +53,7 @@ void getQuestionnairesTest() { void getQuestionnairesByCampaignTest() { Utils.addAdditionalSurveyUnitModelToMongoStub("TESTQUESTIONNAIRE2", surveyUnitPersistencePortStub); - ResponseEntity> response = questionnaireControllerStatic.getQuestionnairesByCampaignV2("TESTCAMPAIGNID"); + ResponseEntity> response = questionnaireControllerStatic.getQuestionnairesByCampaign("TESTCAMPAIGNID"); Assertions.assertThat(response.getStatusCode().is2xxSuccessful()).isTrue(); Assertions.assertThat(response.getBody()).isNotNull().isNotEmpty().containsOnly( diff --git a/src/test/java/fr/insee/genesis/domain/service/SurveyUnitServiceTest.java b/src/test/java/fr/insee/genesis/domain/service/SurveyUnitServiceTest.java index 3d6e220c..0d897b13 100644 --- a/src/test/java/fr/insee/genesis/domain/service/SurveyUnitServiceTest.java +++ b/src/test/java/fr/insee/genesis/domain/service/SurveyUnitServiceTest.java @@ -237,20 +237,19 @@ void findDistinctInterrogationIdsByQuestionnaireIdTest(){ } - //========= OPTIMISATIONS PERFS (START) ========== @Test - void findModesByQuestionnaireIdV2Test(){ - Assertions.assertThat(surveyUnitServiceStatic.findModesByQuestionnaireIdV2(DEFAULT_QUESTIONNAIRE_ID)).filteredOn( + void findModesByQuestionnaireIdTest(){ + Assertions.assertThat(surveyUnitServiceStatic.findModesByQuestionnaireId(DEFAULT_QUESTIONNAIRE_ID)).filteredOn( mode -> mode.equals(Mode.WEB) ).isNotEmpty(); } - //========= OPTIMISATIONS PERFS (END) ========== + @Test void getQuestionnairesByCampaignTest() { addAdditionnalSurveyUnitModelToMongoStub("TESTQUESTIONNAIRE2"); - Assertions.assertThat(surveyUnitServiceStatic.findQuestionnaireIdsByCampaignIdV2("TESTCAMPAIGNID")).isNotEmpty().hasSize(2); + Assertions.assertThat(surveyUnitServiceStatic.findQuestionnaireIdsByCampaignId("TESTCAMPAIGNID")).isNotEmpty().hasSize(2); } diff --git a/src/test/java/fr/insee/genesis/stubs/SurveyUnitPersistencePortStub.java b/src/test/java/fr/insee/genesis/stubs/SurveyUnitPersistencePortStub.java index b17db5b2..ea6e01de 100644 --- a/src/test/java/fr/insee/genesis/stubs/SurveyUnitPersistencePortStub.java +++ b/src/test/java/fr/insee/genesis/stubs/SurveyUnitPersistencePortStub.java @@ -30,7 +30,7 @@ public List findByIds(String interrogationId, String questionna return surveyUnitModelList; } - //========= OPTIMISATIONS PERFS (START) ========== + /** * @author Adrien Marchal */ @@ -43,7 +43,6 @@ public List findBySetOfIdsAndQuestionnaireIdAndMode(String ques return surveyUnitModelList; } - //========= OPTIMISATIONS PERFS (START) ========== @Override @@ -98,7 +97,6 @@ public List findInterrogationIdsByQuestionnaireId(String questi } - //======== OPTIMISATIONS PERFS (START) ======== /** * @author Adrien Marchal */ @@ -122,7 +120,7 @@ public List findPageableInterrogationIdsByQuestionnaireId(Strin @Override - public List findModesByCampaignIdV2(String campaignId) { + public List findModesByCampaignId(String campaignId) { List surveyUnitModelList = new ArrayList<>(); for(SurveyUnitModel SurveyUnitModel : mongoStub){ if(SurveyUnitModel.getCampaignId().equals(campaignId)) @@ -135,7 +133,7 @@ public List findModesByCampaignIdV2(String campaignId) { } @Override - public List findModesByQuestionnaireIdV2(String questionnaireId) { + public List findModesByQuestionnaireId(String questionnaireId) { List surveyUnitModelList = new ArrayList<>(); for(SurveyUnitModel SurveyUnitModel : mongoStub){ if(SurveyUnitModel.getQuestionnaireId().equals(questionnaireId)) @@ -146,8 +144,6 @@ public List findModesByQuestionnaireIdV2(String questionnaireId return surveyUnitModelList; } - //======= OPTIMISATIONS PERFS (END) ========= - @Override @@ -161,9 +157,8 @@ public long count() { } - //========= OPTIMISATIONS PERFS (START) ========== @Override - public Set findQuestionnaireIdsByCampaignIdV2(String campaignId) { + public Set findQuestionnaireIdsByCampaignId(String campaignId) { Set questionnaireIdSet = new HashSet<>(); for(SurveyUnitModel SurveyUnitModel : mongoStub){ if(SurveyUnitModel.getCampaignId().equals(campaignId)) @@ -172,7 +167,7 @@ public Set findQuestionnaireIdsByCampaignIdV2(String campaignId) { return questionnaireIdSet; } - //========= OPTIMISATIONS PERFS (END) ========== + @Override public Set findDistinctCampaignIds() { From 8337639f971c0d04906683ad2358872093f49bd4 Mon Sep 17 00:00:00 2001 From: Adrien MARCHAL Date: Wed, 23 Jul 2025 15:14:49 +0200 Subject: [PATCH 3/6] cleanup --- .../fr/insee/genesis/domain/ports/api/SurveyUnitApiPort.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/fr/insee/genesis/domain/ports/api/SurveyUnitApiPort.java b/src/main/java/fr/insee/genesis/domain/ports/api/SurveyUnitApiPort.java index ddfe3ee0..cefa0e46 100644 --- a/src/main/java/fr/insee/genesis/domain/ports/api/SurveyUnitApiPort.java +++ b/src/main/java/fr/insee/genesis/domain/ports/api/SurveyUnitApiPort.java @@ -25,7 +25,6 @@ public interface SurveyUnitApiPort { Stream findByQuestionnaireId(String questionnaireId); - //NEW VERSION OF THE METHOD List findLatestByIdAndByQuestionnaireId(String interrogationId, String questionnaireId); List> findLatestByIdAndByQuestionnaireIdAndModeOrdered(String questionnaireId, String mode, List interrogationIds); From 92c4f42cfc6114f5adce67152053e7eb60532d81 Mon Sep 17 00:00:00 2001 From: Adrien MARCHAL Date: Fri, 25 Jul 2025 10:07:49 +0200 Subject: [PATCH 4/6] externalize services form COntroller --- .../rest/responses/ResponseController.java | 63 +-------------- .../domain/ports/api/SurveyUnitApiPort.java | 3 +- .../service/surveyunit/SurveyUnitService.java | 79 ++++++++++++++++--- 3 files changed, 73 insertions(+), 72 deletions(-) diff --git a/src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java b/src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java index 9b7741e0..a9347b6f 100644 --- a/src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java +++ b/src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java @@ -300,72 +300,11 @@ public ResponseEntity> getLatestForInterrogationList( public ResponseEntity> getLatestForInterrogationListWithModes(@RequestParam("questionnaireId") String questionnaireId, @RequestParam List modes, @RequestBody List interrogationIds) { - List results = new ArrayList<>(); - - //!!!WARNING!!! : FOR PERFORMANCES PURPOSES, WE DONT'MAKE REQUESTS ON INDIVIDUAL ELEMENTS ANYMORE, BUT ON A SUBLIST OF THE INPUTLIST - final int SUBBLOCK_SIZE = 100; - int offset = 0; - List interrogationIdsSubList = null; - - for(String mode : modes) { - - while(offset <= interrogationIds.size()) { - //extract part of input list - int endOffset = Math.min(offset + SUBBLOCK_SIZE, interrogationIds.size()); - interrogationIdsSubList = interrogationIds.subList(offset, endOffset); - - //1) For each InterrogationId, we collect all responses versions, in which ONLY THE LATEST VERSION of each variable is kept. - List> responses = surveyUnitService.findLatestByIdAndByQuestionnaireIdAndModeOrdered(questionnaireId, mode, interrogationIdsSubList); - - responses.forEach(responsesForSingleInterrId -> { - SurveyUnitSimplified simplifiedResponse = fusionWithLastUpdated(responsesForSingleInterrId, mode); - if(simplifiedResponse != null) { - results.add(simplifiedResponse); - } - }); - - offset = offset + SUBBLOCK_SIZE; - } - } - + List results = surveyUnitService.getLatestForInterrogationListWithModes(questionnaireId, modes, interrogationIds); return ResponseEntity.ok(results); } - private SurveyUnitSimplified fusionWithLastUpdated(List responsesForSingleInterrId, String mode) { - //NOTE : 1) "responses" in input here corresponds to all collected responses versions of a given "InterrogationId", - // in which ONLY THE LATEST VERSION of each variable is kept. - - //return simplifiedResponse - SurveyUnitSimplified simplifiedResponse = null; - - //2) storage of the !!!FUSION!!! OF ALL LATEST UPDATED variables (located in the different versions of the stored "InterrogationId") - List outputVariables = new ArrayList<>(); - List outputExternalVariables = new ArrayList<>(); - - responsesForSingleInterrId.forEach(response -> { - outputVariables.addAll(response.getCollectedVariables()); - outputExternalVariables.addAll(response.getExternalVariables()); - }); - - //3) add to the result list the compiled fusion of all the latest variables - if (!outputVariables.isEmpty() || !outputExternalVariables.isEmpty()) { - Mode modeWrapped = Mode.getEnumFromModeName(mode); - - simplifiedResponse = SurveyUnitSimplified.builder() - .questionnaireId(responsesForSingleInterrId.getFirst().getQuestionnaireId()) - .campaignId(responsesForSingleInterrId.getFirst().getCampaignId()) - .interrogationId(responsesForSingleInterrId.getFirst().getInterrogationId()) - .mode(modeWrapped) - .variablesUpdate(outputVariables) - .externalVariables(outputExternalVariables) - .build(); - } - - return simplifiedResponse; - } - - @Operation(summary = "Save edited variables", description = "Save edited variables document into database") @PostMapping(path = "/save-edited") diff --git a/src/main/java/fr/insee/genesis/domain/ports/api/SurveyUnitApiPort.java b/src/main/java/fr/insee/genesis/domain/ports/api/SurveyUnitApiPort.java index 06dcd985..e69cbd14 100644 --- a/src/main/java/fr/insee/genesis/domain/ports/api/SurveyUnitApiPort.java +++ b/src/main/java/fr/insee/genesis/domain/ports/api/SurveyUnitApiPort.java @@ -6,6 +6,7 @@ import fr.insee.genesis.controller.dto.QuestionnaireWithCampaign; import fr.insee.genesis.controller.dto.SurveyUnitDto; import fr.insee.genesis.controller.dto.SurveyUnitInputDto; +import fr.insee.genesis.controller.dto.SurveyUnitSimplified; import fr.insee.genesis.domain.model.surveyunit.Mode; import fr.insee.genesis.domain.model.surveyunit.SurveyUnitModel; import fr.insee.genesis.exceptions.GenesisException; @@ -27,7 +28,7 @@ public interface SurveyUnitApiPort { List findLatestByIdAndByQuestionnaireId(String interrogationId, String questionnaireId); - List> findLatestByIdAndByQuestionnaireIdAndModeOrdered(String questionnaireId, String mode, List interrogationIds); + List getLatestForInterrogationListWithModes(String questionnaireId, List modes, List interrogationIds); SurveyUnitDto findLatestValuesByStateByIdAndByQuestionnaireId(String interrogationId, String questionnaireId); diff --git a/src/main/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitService.java b/src/main/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitService.java index 886995ed..a01ceef9 100644 --- a/src/main/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitService.java +++ b/src/main/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitService.java @@ -2,13 +2,7 @@ import fr.insee.bpm.metadata.model.VariableType; import fr.insee.bpm.metadata.model.VariablesMap; -import fr.insee.genesis.controller.dto.CampaignWithQuestionnaire; -import fr.insee.genesis.controller.dto.QuestionnaireWithCampaign; -import fr.insee.genesis.controller.dto.SurveyUnitDto; -import fr.insee.genesis.controller.dto.SurveyUnitInputDto; -import fr.insee.genesis.controller.dto.VariableDto; -import fr.insee.genesis.controller.dto.VariableInputDto; -import fr.insee.genesis.controller.dto.VariableStateDto; +import fr.insee.genesis.controller.dto.*; import fr.insee.genesis.controller.services.MetadataService; import fr.insee.genesis.domain.model.surveyunit.DataState; import fr.insee.genesis.domain.model.surveyunit.InterrogationId; @@ -163,6 +157,40 @@ public List findLatestByIdAndByQuestionnaireId(String interroga } + @Override + public List getLatestForInterrogationListWithModes(String questionnaireId, List modes, List interrogationIds) { + List results = new ArrayList<>(); + + //!!!WARNING!!! : FOR PERFORMANCES PURPOSES, WE DONT'MAKE REQUESTS ON INDIVIDUAL ELEMENTS ANYMORE, BUT ON A SUBLIST OF THE INPUTLIST + final int SUBBLOCK_SIZE = 100; + int offset = 0; + List interrogationIdsSubList = null; + + for(String mode : modes) { + + while(offset <= interrogationIds.size()) { + //extract part of input list + int endOffset = Math.min(offset + SUBBLOCK_SIZE, interrogationIds.size()); + interrogationIdsSubList = interrogationIds.subList(offset, endOffset); + + //1) For each InterrogationId, we collect all responses versions, in which ONLY THE LATEST VERSION of each variable is kept. + List> responses = findLatestByIdAndByQuestionnaireIdAndModeOrdered(questionnaireId, mode, interrogationIdsSubList); + + responses.forEach(responsesForSingleInterrId -> { + SurveyUnitSimplified simplifiedResponse = fusionWithLastUpdated(responsesForSingleInterrId, mode); + if(simplifiedResponse != null) { + results.add(simplifiedResponse); + } + }); + + offset = offset + SUBBLOCK_SIZE; + } + } + + return results; + } + + /** * In this method we want to get the latest update for each variable of a survey unit * But we need to separate the updates by mode @@ -173,8 +201,7 @@ public List findLatestByIdAndByQuestionnaireId(String interroga * @return the latest update for each variable of a survey unit * @author Adrien Marchal */ - @Override - public List> findLatestByIdAndByQuestionnaireIdAndModeOrdered(String questionnaireId, String mode, + private List> findLatestByIdAndByQuestionnaireIdAndModeOrdered(String questionnaireId, String mode, List interrogationIds) { //return object List> listLatestUpdatesbyVariables = new ArrayList<>(); @@ -251,6 +278,40 @@ private List extractLatestUpdates(List allResp } + private SurveyUnitSimplified fusionWithLastUpdated(List responsesForSingleInterrId, String mode) { + //NOTE : 1) "responses" in input here corresponds to all collected responses versions of a given "InterrogationId", + // in which ONLY THE LATEST VERSION of each variable is kept. + + //return simplifiedResponse + SurveyUnitSimplified simplifiedResponse = null; + + //2) storage of the !!!FUSION!!! OF ALL LATEST UPDATED variables (located in the different versions of the stored "InterrogationId") + List outputVariables = new ArrayList<>(); + List outputExternalVariables = new ArrayList<>(); + + responsesForSingleInterrId.forEach(response -> { + outputVariables.addAll(response.getCollectedVariables()); + outputExternalVariables.addAll(response.getExternalVariables()); + }); + + //3) add to the result list the compiled fusion of all the latest variables + if (!outputVariables.isEmpty() || !outputExternalVariables.isEmpty()) { + Mode modeWrapped = Mode.getEnumFromModeName(mode); + + simplifiedResponse = SurveyUnitSimplified.builder() + .questionnaireId(responsesForSingleInterrId.getFirst().getQuestionnaireId()) + .campaignId(responsesForSingleInterrId.getFirst().getCampaignId()) + .interrogationId(responsesForSingleInterrId.getFirst().getInterrogationId()) + .mode(modeWrapped) + .variablesUpdate(outputVariables) + .externalVariables(outputExternalVariables) + .build(); + } + + return simplifiedResponse; + } + + @Override public SurveyUnitDto findLatestValuesByStateByIdAndByQuestionnaireId(String interrogationId, String questionnaireId) { From 90f45c29b071a4218d06e79c739c073adaa4c0b3 Mon Sep 17 00:00:00 2001 From: Adrien MARCHAL Date: Fri, 25 Jul 2025 10:48:22 +0200 Subject: [PATCH 5/6] review TODOs decisions after merge with main branch --- .../service/surveyunit/SurveyUnitService.java | 67 +------------------ 1 file changed, 1 insertion(+), 66 deletions(-) diff --git a/src/main/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitService.java b/src/main/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitService.java index a01ceef9..ace65499 100644 --- a/src/main/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitService.java +++ b/src/main/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitService.java @@ -79,8 +79,6 @@ public Stream findByQuestionnaireId(String questionnaireId) { */ @Override public List findLatestByIdAndByQuestionnaireId(String interrogationId, String questionnaireId) { - //TODO : DIFFERENCES TO BE REVIEWED - /* List enumModes = findModesByQuestionnaireId(questionnaireId); // => convertion of "List" -> "List" for query using lamda List modes = enumModes.stream().map(Mode::getModeName).toList(); @@ -95,65 +93,6 @@ public List findLatestByIdAndByQuestionnaireId(String interroga } return responses; - */ - List latestUpdatesbyVariables = new ArrayList<>(); - List surveyUnitModels = surveyUnitPersistencePort.findByIds(interrogationId, questionnaireId); - List modes = getDistinctsModes(surveyUnitModels); - modes.forEach(mode ->{ - List suByMode = surveyUnitModels.stream() - .filter(surveyUnitModel -> surveyUnitModel.getMode().equals(mode)) - .sorted((o1, o2) -> o2.getRecordDate().compareTo(o1.getRecordDate())) //Sorting update by date (latest updates first by date of upload in database) - .toList(); - - //We had all the variables of the oldest update - latestUpdatesbyVariables.add(suByMode.getFirst()); - //We keep the name of already added variables to skip them in older updates - List addedVariables = new ArrayList<>(); - SurveyUnitModel latestUpdate = suByMode.getFirst(); - - if(latestUpdate.getCollectedVariables() == null){ - latestUpdate.setCollectedVariables(new ArrayList<>()); - } - if(latestUpdate.getExternalVariables() == null){ - latestUpdate.setExternalVariables(new ArrayList<>()); - } - - latestUpdate.getCollectedVariables().forEach(colVar -> addedVariables.add(new VarIdScopeTuple(colVar.varId(), - colVar.scope(), colVar.iteration()))); - latestUpdate.getExternalVariables().forEach(extVar -> addedVariables.add(new VarIdScopeTuple(extVar.varId(), extVar.scope(), extVar.iteration()))); - - suByMode.forEach(surveyUnitModel -> { - List collectedVariablesToKeep = new ArrayList<>(); - List externalVariablesToKeep = new ArrayList<>(); - // We iterate over the variables of the update and add them to the list if they are not already added - if (surveyUnitModel.getCollectedVariables() != null) { - surveyUnitModel.getCollectedVariables().stream() - .filter(colVar -> !addedVariables.contains(new VarIdScopeTuple(colVar.varId(), colVar.scope() - , colVar.iteration()))) - .forEach(colVar -> { - collectedVariablesToKeep.add(colVar); - addedVariables.add(new VarIdScopeTuple(colVar.varId(), colVar.scope(), colVar.iteration())); - }); - } - if (surveyUnitModel.getExternalVariables() != null){ - surveyUnitModel.getExternalVariables().stream() - .filter(extVar -> !addedVariables.contains(new VarIdScopeTuple(extVar.varId(), extVar.scope(), - extVar.iteration()))) - .forEach(extVar -> { - externalVariablesToKeep.add(extVar); - addedVariables.add(new VarIdScopeTuple(extVar.varId(), extVar.scope(), extVar.iteration())); - }); - } - - // If there are new variables, we add the update to the list of latest updates - if (!collectedVariablesToKeep.isEmpty() || !externalVariablesToKeep.isEmpty()){ - surveyUnitModel.setCollectedVariables(collectedVariablesToKeep); - surveyUnitModel.setExternalVariables(externalVariablesToKeep); - latestUpdatesbyVariables.add(surveyUnitModel); - } - }); - }); - return latestUpdatesbyVariables; } @@ -161,7 +100,7 @@ public List findLatestByIdAndByQuestionnaireId(String interroga public List getLatestForInterrogationListWithModes(String questionnaireId, List modes, List interrogationIds) { List results = new ArrayList<>(); - //!!!WARNING!!! : FOR PERFORMANCES PURPOSES, WE DONT'MAKE REQUESTS ON INDIVIDUAL ELEMENTS ANYMORE, BUT ON A SUBLIST OF THE INPUTLIST + //!!!WARNING!!! : FOR PERFORMANCES PURPOSES, WE DON'T MAKE REQUESTS ON INDIVIDUAL ELEMENTS ANYMORE, BUT ON A SUBLIST OF THE INPUT LIST final int SUBBLOCK_SIZE = 100; int offset = 0; List interrogationIdsSubList = null; @@ -331,9 +270,6 @@ public SurveyUnitDto findLatestValuesByStateByIdAndByQuestionnaireId(String inte .filter(surveyUnitModel -> surveyUnitModel.getMode().equals(mode)) .sorted((o1, o2) -> o2.getRecordDate().compareTo(o1.getRecordDate())) //Sorting update by date (latest updates first by date of upload in database) .toList(); - //TODO : REVIEW DIFFERENCES BETWEEN THE 2 VERSIONS! - //suByMode.forEach(surveyUnitModel -> extractVariables(surveyUnitModel, collectedVariableMap,externalVariableMap)); - //---------- VariablesMap variablesMap = null; for(SurveyUnitModel surveyUnitModel : suByMode){ if(variablesMap == null){ @@ -346,7 +282,6 @@ public SurveyUnitDto findLatestValuesByStateByIdAndByQuestionnaireId(String inte extractVariables(surveyUnitModel, collectedVariableMap, externalVariableMap, variablesMap); } - //---------- }); collectedVariableMap.keySet().forEach(variableTuple -> surveyUnitDto.getCollectedVariables().add(collectedVariableMap.get(variableTuple))); externalVariableMap.keySet().forEach(variableName -> surveyUnitDto.getExternalVariables().add(externalVariableMap.get(variableName))); From 98917ff0f6a90ecdd697f42ca7893007c2fef970 Mon Sep 17 00:00:00 2001 From: Adrien MARCHAL Date: Fri, 25 Jul 2025 12:06:49 +0200 Subject: [PATCH 6/6] add performance test --- .../responses/ResponseControllerTest.java | 30 +++++++++++++ .../controller/rest/responses/Utils.java | 44 +++++++++++++++++-- 2 files changed, 70 insertions(+), 4 deletions(-) diff --git a/src/test/java/fr/insee/genesis/controller/rest/responses/ResponseControllerTest.java b/src/test/java/fr/insee/genesis/controller/rest/responses/ResponseControllerTest.java index 4b37f937..d1b107a7 100644 --- a/src/test/java/fr/insee/genesis/controller/rest/responses/ResponseControllerTest.java +++ b/src/test/java/fr/insee/genesis/controller/rest/responses/ResponseControllerTest.java @@ -782,4 +782,34 @@ void saveEditedTest_null() { Assertions.assertThat(docSaved.getModifiedBy()).isNull(); } + + + @Test + void getLatestForInterrogationListWithModes_performance() throws GenesisException { + //GIVEN + Utils.fillMongoStubForPerformances(DataState.COLLECTED, + LocalDateTime.of(1999,2,2,0,0,0), + LocalDateTime.of(1999,2,2,0,0,0), + surveyUnitPersistencePortStub + ); + + + //WHEN + List modes = new ArrayList<>(); + modes.add("WEB"); + List interrogationIds = Utils.generateInterrogationIds(); + long start = System.currentTimeMillis(); + ResponseEntity> response = responseControllerStatic.getLatestForInterrogationListWithModes( + DEFAULT_QUESTIONNAIRE_ID, + modes, + interrogationIds + ); + long end = System.currentTimeMillis(); + long delta = end - start; + + //THEN + Assertions.assertThat(delta < 3000).isTrue(); + } + + } diff --git a/src/test/java/fr/insee/genesis/controller/rest/responses/Utils.java b/src/test/java/fr/insee/genesis/controller/rest/responses/Utils.java index a4602276..25085637 100644 --- a/src/test/java/fr/insee/genesis/controller/rest/responses/Utils.java +++ b/src/test/java/fr/insee/genesis/controller/rest/responses/Utils.java @@ -1,11 +1,9 @@ package fr.insee.genesis.controller.rest.responses; import cucumber.TestConstants; -import fr.insee.genesis.domain.model.surveyunit.DataState; -import fr.insee.genesis.domain.model.surveyunit.Mode; -import fr.insee.genesis.domain.model.surveyunit.SurveyUnitModel; -import fr.insee.genesis.domain.model.surveyunit.VariableModel; +import fr.insee.genesis.domain.model.surveyunit.*; import fr.insee.genesis.stubs.SurveyUnitPersistencePortStub; +import org.apache.commons.lang3.StringUtils; import java.io.File; import java.io.IOException; @@ -310,4 +308,42 @@ static void addAdditionalSurveyUnitModelToMongoStub(DataState state, surveyUnitPersistencePortStub.getMongoStub().add(recentSurveyUnitModel); } + + static void fillMongoStubForPerformances(DataState state, + LocalDateTime fileDate, + LocalDateTime recordDate, + SurveyUnitPersistencePortStub surveyUnitPersistencePortStub) { + List sumList = new ArrayList<>(); + + + for (int i = 0; i < 1000 ; i++) { + SurveyUnitModel recentSurveyUnitModel = SurveyUnitModel.builder() + .campaignId("TEST-TABLEAUX") + .mode(Mode.WEB) + //interrogationId example : "UE000000001" + .interrogationId(StringUtils.leftPad(String.valueOf(i), 9, "0")) + .questionnaireId(DEFAULT_QUESTIONNAIRE_ID) + .state(state) + .fileDate(fileDate) + .recordDate(recordDate) + .externalVariables(new ArrayList<>()) + .collectedVariables(new ArrayList<>()) + .build(); + sumList.add(recentSurveyUnitModel); + } + + surveyUnitPersistencePortStub.getMongoStub().addAll(sumList); + } + + + static List generateInterrogationIds() { + List ids = new ArrayList<>(); + for (int i = 0; i < 1000; i++) { + InterrogationId interrogationId = new InterrogationId(StringUtils.leftPad(String.valueOf(i), 9, "0")); + ids.add(interrogationId); + } + return ids; + } + + }