Skip to content

devPerformances cleanup #260

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
amarchal-insee wants to merge 7 commits into
base: main
Choose a base branch
from
Open

devPerformances cleanup #260

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
72acda0
!!!WARNING!!! - REMOVED OLD V1 METHODS
amarchal-insee Jul 23, 2025
f3478ae
!!!WARNING!!! - REMOVE ALL V2 REFS
amarchal-insee Jul 23, 2025
8337639
cleanup
amarchal-insee Jul 23, 2025
ee5693a
Merge branch 'main' into horizontalScaling
amarchal-insee Jul 25, 2025
92c4f42
externalize services form COntroller
amarchal-insee Jul 25, 2025
90f45c2
review TODOs decisions after merge with main branch
amarchal-insee Jul 25, 2025
98917ff
add performance test
amarchal-insee Jul 25, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions src/main/java/fr/insee/genesis/controller/rest/responses/InterrogationController.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,9 @@ public InterrogationController(SurveyUnitApiPort surveyUnitService) {
}


/**
* !!!WARNING!!! : A CALL WITH THIS ENDPOINT ON A BIG COLLECTION (> 300k) MAY KILL THE GENESIS-API APP.!!!
*/
@Operation(summary = "Retrieve all interrogations for a given questionnaire")
@GetMapping(path = "/by-questionnaire")
public ResponseEntity<List<InterrogationId>> getAllInterrogationIdsByQuestionnaire(@RequestParam("questionnaireId") String questionnaireId) {
Expand All @@ -36,7 +39,6 @@ public ResponseEntity<List<InterrogationId>> getAllInterrogationIdsByQuestionnai
}


//========= OPTIMISATIONS PERFS (START) ==========
/**
* @author Adrien Marchal
*/
Expand All @@ -63,7 +65,6 @@ public ResponseEntity<List<InterrogationId>> getPaginatedInterrogationIdsByQuest
List<InterrogationId> responses = surveyUnitService.findDistinctPageableInterrogationIdsByQuestionnaireId(questionnaireId, totalSize, blockSize, page);
return ResponseEntity.ok(responses);
}
//======== OPTIMISATIONS PERFS (END) ===========


}
16 changes: 0 additions & 16 deletions src/main/java/fr/insee/genesis/controller/rest/responses/ModeController.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,20 +40,4 @@ public ResponseEntity<List<Mode>> getModesByCampaign(@RequestParam("campaignId")
return ResponseEntity.ok(modes);
}

//========= OPTIMISATIONS PERFS (START) ==========
@Operation(summary = "List sources/modes used for a given questionnaire")
@GetMapping(path = "/by-questionnaireV2")
public ResponseEntity<List<Mode>> getModesByQuestionnaireV2(@RequestParam("questionnaireId") String questionnaireId) {
List<Mode> modes = surveyUnitService.findModesByQuestionnaireIdV2(questionnaireId);
return ResponseEntity.ok(modes);
}

@Operation(summary = "List sources/modes used for a given campaign")
@GetMapping(path = "/by-campaignV2")
public ResponseEntity<List<Mode>> getModesByCampaignV2(@RequestParam("campaignId") String campaignId) {
List<Mode> modes = surveyUnitService.findModesByCampaignIdV2(campaignId);
return ResponseEntity.ok(modes);
}
//========= OPTIMISATIONS PERFS (END) ==========

}
15 changes: 4 additions & 11 deletions src/main/java/fr/insee/genesis/controller/rest/responses/QuestionnaireController.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,24 +46,17 @@ public ResponseEntity<List<QuestionnaireWithCampaign>> getQuestionnairesWithCamp
return ResponseEntity.ok(questionnaireWithCampaignList);
}

@Operation(summary = "List questionnaires used for a given campaign")
@GetMapping(path = "/by-campaign")
public ResponseEntity<Set<String>> getQuestionnairesByCampaign(@RequestParam("campaignId") String campaignId) {
Set<String> questionnaires = surveyUnitService.findQuestionnaireIdsByCampaignId(campaignId);
return ResponseEntity.ok(questionnaires);
}

//========= OPTIMISATIONS PERFS (START) ==========
/**
* @author Adrien Marchal
*/
@Operation(summary = "List questionnaires used for a given campaign (using a DISTINCT query)")
@GetMapping(path = "/by-campaignV2")
public ResponseEntity<Set<String>> getQuestionnairesByCampaignV2(@RequestParam("campaignId") String campaignId) {
Set<String> questionnaires = surveyUnitService.findQuestionnaireIdsByCampaignIdV2(campaignId);
@GetMapping(path = "/by-campaign")
public ResponseEntity<Set<String>> getQuestionnairesByCampaign(@RequestParam("campaignId") String campaignId) {
Set<String> questionnaires = surveyUnitService.findQuestionnaireIdsByCampaignId(campaignId);
return ResponseEntity.ok(questionnaires);
}
//========= OPTIMISATIONS PERFS (END) ==========


@Operation(summary = "Get the questionnaireId corresponding to an interrogationId")
@GetMapping(path = "/by-interrogation")
Expand Down
104 changes: 11 additions & 93 deletions src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,11 +57,9 @@
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
import java.util.stream.Stream;

@RequestMapping(path = "/responses" )
@Controller
Expand Down Expand Up @@ -274,119 +272,39 @@ public ResponseEntity<SurveyUnitSimplified> getLatestByInterrogationOneObject(@R
}


/**
* @author Adrien Marchal
*/
@Operation(summary = "Retrieve all responses for a questionnaire and a list of UE",
description = "Return the latest state for each variable for the given ids and a given questionnaire.<br>" +
"For a given id, the endpoint returns a document by collection mode (if there is more than one).")
@PostMapping(path = "/simplified/by-list-interrogation-and-questionnaire/latest")
@PreAuthorize("hasRole('USER_KRAFTWERK')")
public ResponseEntity<List<SurveyUnitSimplified>> getLatestForInterrogationList(@RequestParam("questionnaireId") String questionnaireId,
@RequestBody List<InterrogationId> interrogationIds) {
List<SurveyUnitSimplified> results = new ArrayList<>();
List<Mode> modes = surveyUnitService.findModesByQuestionnaireId(questionnaireId);
interrogationIds.forEach(interrogationId -> {
List<SurveyUnitModel> responses = surveyUnitService.findLatestByIdAndByQuestionnaireId(interrogationId.getInterrogationId(), questionnaireId);
modes.forEach(mode -> {
List<VariableModel> outputVariables = new ArrayList<>();
List<VariableModel> outputExternalVariables = new ArrayList<>();
responses.stream().filter(rep -> rep.getMode().equals(mode)).forEach(response -> {
outputVariables.addAll(response.getCollectedVariables());
outputExternalVariables.addAll(response.getExternalVariables());
});
if (!outputVariables.isEmpty() || !outputExternalVariables.isEmpty()) {
results.add(SurveyUnitSimplified.builder()
.questionnaireId(responses.getFirst().getQuestionnaireId())
.campaignId(responses.getFirst().getCampaignId())
.interrogationId(responses.getFirst().getInterrogationId())
.mode(mode)
.variablesUpdate(outputVariables)
.externalVariables(outputExternalVariables)
.build());
}
});
});
return ResponseEntity.ok(results);
@RequestBody List<InterrogationId> interrogationIds) {
List<Mode> enumModes = surveyUnitService.findModesByQuestionnaireId(questionnaireId);
// => convertion of "List<Mode>" -> "List<String>" for query using lamda
List<String> modes = enumModes.stream().map(Mode::getModeName).toList();
return getLatestForInterrogationListWithModes(questionnaireId, modes, interrogationIds);
}


//========= OPTIMISATIONS PERFS (START) ==========
/**
* @author Adrien Marchal
*/
@Operation(summary = "Retrieve all responses for a questionnaire and a list of UE",
description = "Return the latest state for each variable for the given ids and a given questionnaire.<br>" +
"For a given id, the endpoint returns a document by collection mode (if there is more than one).")
@PostMapping(path = "/simplified/by-list-interrogation-and-questionnaire/latestV2")
@PostMapping(path = "/simplified/by-list-interrogation-and-questionnaire-and-modes/latest")
@PreAuthorize("hasRole('USER_KRAFTWERK')")
public ResponseEntity<List<SurveyUnitSimplified>> getLatestForInterrogationListV2(@RequestParam("questionnaireId") String questionnaireId,
public ResponseEntity<List<SurveyUnitSimplified>> getLatestForInterrogationListWithModes(@RequestParam("questionnaireId") String questionnaireId,
@RequestParam List<String> modes,
@RequestBody List<InterrogationId> interrogationIds) {
List<SurveyUnitSimplified> results = new ArrayList<>();

//!!!WARNING!!! : FOR PERFORMANCES PURPOSES, WE DONT'MAKE REQUESTS ON INDIVIDUAL ELEMENTS ANYMORE, BUT ON A SUBLIST OF THE INPUTLIST
final int SUBBLOCK_SIZE = 100;
int offset = 0;
List<InterrogationId> interrogationIdsSubList = null;

for(String mode : modes) {

while(offset <= interrogationIds.size()) {
//extract part of input list
int endOffset = Math.min(offset + SUBBLOCK_SIZE, interrogationIds.size());
interrogationIdsSubList = interrogationIds.subList(offset, endOffset);

//1) For each InterrogationId, we collect all responses versions, in which ONLY THE LATEST VERSION of each variable is kept.
List<List<SurveyUnitModel>> responses = surveyUnitService.findLatestByIdAndByQuestionnaireIdAndModeOrdered(questionnaireId, mode, interrogationIdsSubList);

responses.forEach(responsesForSingleInterrId -> {
SurveyUnitSimplified simplifiedResponse = fusionWithLastUpdated(responsesForSingleInterrId, mode);
if(simplifiedResponse != null) {
results.add(simplifiedResponse);
}
});

offset = offset + SUBBLOCK_SIZE;
}
}

List<SurveyUnitSimplified> results = surveyUnitService.getLatestForInterrogationListWithModes(questionnaireId, modes, interrogationIds);
return ResponseEntity.ok(results);
}


private SurveyUnitSimplified fusionWithLastUpdated(List<SurveyUnitModel> responsesForSingleInterrId, String mode) {
//NOTE : 1) "responses" in input here corresponds to all collected responses versions of a given "InterrogationId",
// in which ONLY THE LATEST VERSION of each variable is kept.

//return simplifiedResponse
SurveyUnitSimplified simplifiedResponse = null;

//2) storage of the !!!FUSION!!! OF ALL LATEST UPDATED variables (located in the different versions of the stored "InterrogationId")
List<VariableModel> outputVariables = new ArrayList<>();
List<VariableModel> outputExternalVariables = new ArrayList<>();

responsesForSingleInterrId.forEach(response -> {
outputVariables.addAll(response.getCollectedVariables());
outputExternalVariables.addAll(response.getExternalVariables());
});

//3) add to the result list the compiled fusion of all the latest variables
if (!outputVariables.isEmpty() || !outputExternalVariables.isEmpty()) {
Mode modeWrapped = Mode.getEnumFromModeName(mode);

simplifiedResponse = SurveyUnitSimplified.builder()
.questionnaireId(responsesForSingleInterrId.getFirst().getQuestionnaireId())
.campaignId(responsesForSingleInterrId.getFirst().getCampaignId())
.interrogationId(responsesForSingleInterrId.getFirst().getInterrogationId())
.mode(modeWrapped)
.variablesUpdate(outputVariables)
.externalVariables(outputExternalVariables)
.build();
}

return simplifiedResponse;
}
//========= OPTIMISATIONS PERFS (END) ==========


@Operation(summary = "Save edited variables",
description = "Save edited variables document into database")
@PostMapping(path = "/save-edited")
Expand Down
22 changes: 7 additions & 15 deletions src/main/java/fr/insee/genesis/domain/ports/api/SurveyUnitApiPort.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import fr.insee.genesis.controller.dto.QuestionnaireWithCampaign;
import fr.insee.genesis.controller.dto.SurveyUnitDto;
import fr.insee.genesis.controller.dto.SurveyUnitInputDto;
import fr.insee.genesis.controller.dto.SurveyUnitSimplified;
import fr.insee.genesis.domain.model.surveyunit.Mode;
import fr.insee.genesis.domain.model.surveyunit.SurveyUnitModel;
import fr.insee.genesis.exceptions.GenesisException;
Expand All @@ -27,45 +28,36 @@ public interface SurveyUnitApiPort {

List<SurveyUnitModel> findLatestByIdAndByQuestionnaireId(String interrogationId, String questionnaireId);

//========= OPTIMISATIONS PERFS (START) ==========
List<List<SurveyUnitModel>> findLatestByIdAndByQuestionnaireIdAndModeOrdered(String questionnaireId, String mode, List<InterrogationId> interrogationIds);
//========= OPTIMISATIONS PERFS (END) ==========
List<SurveyUnitSimplified> getLatestForInterrogationListWithModes(String questionnaireId, List<String> modes, List<InterrogationId> interrogationIds);

SurveyUnitDto findLatestValuesByStateByIdAndByQuestionnaireId(String interrogationId, String questionnaireId);

List<SurveyUnitModel> findInterrogationIdsAndModesByQuestionnaireId(String questionnaireId);

/**
* !!!WARNING!!! : A CALL WITH THIS ENDPOINT ON A BIG COLLECTION (> 300k) MAY KILL THE GENESIS-API APP.!!!
*/
List<InterrogationId> findDistinctInterrogationIdsByQuestionnaireId(String questionnaireId);

//========= OPTIMISATIONS PERFS (START) ==========
long countInterrogationIdsByQuestionnaireId(String questionnaireId);

List<InterrogationId> findDistinctPageableInterrogationIdsByQuestionnaireId(String questionnaireId,
long totalSize, long blockSize, long page);

List<Mode> findModesByQuestionnaireIdV2(String questionnaireId);
//========= OPTIMISATIONS PERFS (END) ==========

List<Mode> findModesByQuestionnaireId(String questionnaireId);

List<Mode> findModesByCampaignId(String campaignId);

//========= OPTIMISATIONS PERFS (START) ==========
List<Mode> findModesByCampaignIdV2(String campaignId);
//========= OPTIMISATIONS PERFS (END) ==========
List<Mode> findModesByCampaignId(String campaignId);

Long deleteByQuestionnaireId(String questionnaireId);

long countResponses();

Set<String> findQuestionnaireIdsByCampaignId(String campaignId);

//========= OPTIMISATIONS PERFS (START) ==========
/**
* @author Adrien Marchal
*/
Set<String> findQuestionnaireIdsByCampaignIdV2(String campaignId);
//========= OPTIMISATIONS PERFS (END) ==========
Set<String> findQuestionnaireIdsByCampaignId(String campaignId);

Set<String> findDistinctCampaignIds();

Expand Down
17 changes: 6 additions & 11 deletions src/main/java/fr/insee/genesis/domain/ports/spi/SurveyUnitPersistencePort.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,45 +12,40 @@ public interface SurveyUnitPersistencePort {

List<SurveyUnitModel> findByIds(String interrogationId, String questionnaireId);

//========= OPTIMISATIONS PERFS (START) ==========
/**
* @author Adrien Marchal
*/
List<SurveyUnitModel> findBySetOfIdsAndQuestionnaireIdAndMode(String questionnaireId, String mode, List<String> interrogationIdSet);
//========= OPTIMISATIONS PERFS (START) ==========

List<SurveyUnitModel> findByInterrogationId(String interrogationId);

List<SurveyUnitModel> findByInterrogationIdsAndQuestionnaireId(List<SurveyUnitModel> interrogationIds, String questionnaireId);

Stream<SurveyUnitModel> findByQuestionnaireId(String questionnaireId);

/**
* !!!WARNING!!! : A CALL WITH THIS ENDPOINT ON A BIG COLLECTION (> 300k) MAY KILL THE GENESIS-API APP.!!!
*/
List<SurveyUnitModel> findInterrogationIdsByQuestionnaireId(String questionnaireId);

//======== OPTIMISATIONS PERFS (START) ========
long countInterrogationIdsByQuestionnaireId(String questionnaireId);

List<SurveyUnitModel> findPageableInterrogationIdsByQuestionnaireId(String questionnaireId, Long skip, Long limit);

List<SurveyUnitModel> findModesByCampaignIdV2(String campaignId);
List<SurveyUnitModel> findModesByCampaignId(String campaignId);

List<SurveyUnitModel> findModesByQuestionnaireIdV2(String questionnaireId);
//======= OPTIMISATIONS PERFS (END) =========
List<SurveyUnitModel> findModesByQuestionnaireId(String questionnaireId);

List<SurveyUnitModel> findInterrogationIdsByCampaignId(String campaignId);

Long deleteByQuestionnaireId(String questionnaireId);

long count();

Set<String> findQuestionnaireIdsByCampaignId(String campaignId);

//========= OPTIMISATIONS PERFS (START) ==========
/**
* @author Adrien Marchal
*/
Set<String> findQuestionnaireIdsByCampaignIdV2(String campaignId);
//========= OPTIMISATIONS PERFS (END) ==========
Set<String> findQuestionnaireIdsByCampaignId(String campaignId);

Set<String> findDistinctCampaignIds();

Expand Down
Loading
Loading