diff --git a/.mvn/wrapper/maven-wrapper.properties b/.mvn/wrapper/maven-wrapper.properties new file mode 100644 index 00000000..8659fbea --- /dev/null +++ b/.mvn/wrapper/maven-wrapper.properties @@ -0,0 +1,2 @@ +distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.9.11/apache-maven-3.9.11-bin.zip +wrapperUrl=https://repo.maven.apache.org/maven2/org/apache/maven/wrapper/maven-wrapper/3.3.4/maven-wrapper-3.3.4.jar diff --git a/pom.xml b/pom.xml index a43e4dbc..a7ac3cbd 100644 --- a/pom.xml +++ b/pom.xml @@ -98,7 +98,16 @@ springdoc-openapi-starter-webmvc-ui ${springdoc.version} - + + fr.insee + modelefiliere + 2.0.0 + + + com.networknt + json-schema-validator + 1.5.9 + @@ -107,13 +116,6 @@ 12.9 - - - com.networknt - json-schema-validator - 2.0.0 - - org.mapstruct diff --git a/src/main/java/fr/insee/genesis/Constants.java b/src/main/java/fr/insee/genesis/Constants.java index 9ce082a2..a4f12760 100644 --- a/src/main/java/fr/insee/genesis/Constants.java +++ b/src/main/java/fr/insee/genesis/Constants.java @@ -23,8 +23,7 @@ public class Constants { public static final String MONGODB_CONTEXTUAL_EXTERNAL_COLLECTION_NAME = "editedExternal"; public static final String LOOP_NAME_PREFIX = "BOUCLE"; public static final String MONGODB_RESPONSE_COLLECTION_NAME = "responses"; - public static final String MONGODB_RESPONSE_RAW_COLLECTION_NAME = "lunaticjsondata"; - public static final String MONGODB_VARIABLETYPE_COLLECTION_NAME = "variabletypes"; + public static final String MONGODB_RAW_RESPONSES_COLLECTION_NAME = "rawResponses"; public static final String VOLUMETRY_FOLDER_NAME = "genesis_volumetries"; public static final String VOLUMETRY_FILE_SUFFIX = "_VOLUMETRY"; public static final String VOLUMETRY_RAW_FILE_SUFFIX = "_RAW_VOLUMETRY"; diff --git a/src/main/java/fr/insee/genesis/controller/adapter/LunaticJsonAdapter.java b/src/main/java/fr/insee/genesis/controller/adapter/LunaticJsonAdapter.java index edded928..33178f1c 100644 --- a/src/main/java/fr/insee/genesis/controller/adapter/LunaticJsonAdapter.java +++ b/src/main/java/fr/insee/genesis/controller/adapter/LunaticJsonAdapter.java @@ -13,7 +13,7 @@ public class LunaticJsonAdapter { public SurveyUnitModel convert(LunaticJsonSurveyUnit su){ return SurveyUnitModel.builder() - .questionnaireId(su.getQuestionnaireId()) + .collectionInstrumentId(su.getQuestionnaireId()) .campaignId("") .interrogationId(su.getInterrogationId()) .state(DataState.COLLECTED) diff --git a/src/main/java/fr/insee/genesis/controller/adapter/LunaticXmlAdapter.java b/src/main/java/fr/insee/genesis/controller/adapter/LunaticXmlAdapter.java index 143965f5..ee78e099 100644 --- a/src/main/java/fr/insee/genesis/controller/adapter/LunaticXmlAdapter.java +++ b/src/main/java/fr/insee/genesis/controller/adapter/LunaticXmlAdapter.java @@ -69,7 +69,7 @@ public static List convert(LunaticXmlSurveyUnit su, VariablesMa */ private static SurveyUnitModel getStateDataFromSurveyUnit(LunaticXmlSurveyUnit su, VariablesMap variablesMap, String campaignId, DataState dataState, Mode mode) { SurveyUnitModel surveyUnitModel = SurveyUnitModel.builder() - .questionnaireId(su.getQuestionnaireModelId().toUpperCase()) + .collectionInstrumentId(su.getQuestionnaireModelId().toUpperCase()) .campaignId(campaignId) .interrogationId(su.getId()) .state(dataState) diff --git a/src/main/java/fr/insee/genesis/controller/dto/ScheduleDto.java b/src/main/java/fr/insee/genesis/controller/dto/ScheduleDto.java index 976bfe7e..bf67427e 100644 --- a/src/main/java/fr/insee/genesis/controller/dto/ScheduleDto.java +++ b/src/main/java/fr/insee/genesis/controller/dto/ScheduleDto.java @@ -8,6 +8,7 @@ @Builder public record ScheduleDto (String surveyName, + String collectionInstrumentId, LocalDateTime lastExecution, List kraftwerkExecutionScheduleList ){} diff --git a/src/main/java/fr/insee/genesis/controller/dto/SurveyUnitSimplified.java b/src/main/java/fr/insee/genesis/controller/dto/SurveyUnitSimplified.java index c1d4abec..57ea1d84 100644 --- a/src/main/java/fr/insee/genesis/controller/dto/SurveyUnitSimplified.java +++ b/src/main/java/fr/insee/genesis/controller/dto/SurveyUnitSimplified.java @@ -11,10 +11,14 @@ @Data public class SurveyUnitSimplified { - private String questionnaireId; + private String collectionInstrumentId; + /** + * @deprecated We will not reveive this piece of information anymore + */ + @Deprecated(forRemoval = true, since = "2026-01-01") private String campaignId; private String interrogationId; - private String surveyUnitId; + private String usualSurveyUnitId; private Mode mode; private List variablesUpdate; private List externalVariables; diff --git a/src/main/java/fr/insee/genesis/controller/mappers/DataProcessingContextMapperDto.java b/src/main/java/fr/insee/genesis/controller/mappers/DataProcessingContextMapperDto.java index faa5caf6..256660b1 100644 --- a/src/main/java/fr/insee/genesis/controller/mappers/DataProcessingContextMapperDto.java +++ b/src/main/java/fr/insee/genesis/controller/mappers/DataProcessingContextMapperDto.java @@ -23,4 +23,13 @@ public List dataProcessingContextListToScheduleDtoList(List saveContext( @Parameter(description = "Identifier of the partition", required = true) @RequestParam("partitionId") String partitionId, @@ -60,8 +51,40 @@ public ResponseEntity saveContext( return ResponseEntity.ok().build(); } + @Operation(summary = "Create or update a data processing context") + @PutMapping(path = "/contexts/{collectionInstrumentId}/review") + @PreAuthorize("hasAnyRole('USER_PLATINE', 'USER_BACK_OFFICE', 'SCHEDULER')") + public ResponseEntity saveContextWithCollectionInstrumentId( + @PathVariable("collectionInstrumentId") String collectionInstrumentId, + @Parameter(description = "Allow reviewing") @RequestParam(value = "withReview", defaultValue = "false") Boolean withReview + ){ + try { + withReview = withReview != null && withReview; //False if null + dataProcessingContextApiPort.saveContextByCollectionInstrumentId(collectionInstrumentId, withReview); + }catch (GenesisException e){ + return new ResponseEntity<>(e.getMessage(), HttpStatusCode.valueOf(e.getStatus())); + } + return ResponseEntity.ok().build(); + } + + + @Operation(summary = "Returns partition review indicator") + @GetMapping(path = "/contexts/{collectionInstrumentId}/review") + @PreAuthorize("hasAnyRole('USER_BACK_OFFICE','SCHEDULER','USER_PLATINE')") + public ResponseEntity getReviewIndicatorByCollectionInstrumentId( + @PathVariable("collectionInstrumentId") String collectionInstrumentId + ){ + try { + boolean withReview = dataProcessingContextApiPort.getReviewByCollectionInstrumentId(collectionInstrumentId); + return ResponseEntity.ok(withReview); + }catch (GenesisException e){ + return new ResponseEntity<>(e.getMessage(), HttpStatusCode.valueOf(e.getStatus())); + } + } + + @Deprecated(forRemoval = true) @Operation(summary = "Returns partition review indicator") - @GetMapping(path = "/review") + @GetMapping(path = "/context/review") @PreAuthorize("hasAnyRole('USER_BACK_OFFICE','SCHEDULER','USER_PLATINE')") public ResponseEntity getReviewIndicator( @Parameter(description = "Identifier of the partition", required = true) @RequestParam("partitionId") String partitionId @@ -74,8 +97,9 @@ public ResponseEntity getReviewIndicator( } } + @Deprecated(forRemoval = true) @Operation(summary = "Schedule a Kraftwerk execution") - @PutMapping(path = "/schedules") + @PutMapping(path = "/context/schedules") @PreAuthorize("hasRole('USER_KRAFTWERK')") public ResponseEntity saveSchedule( @Parameter(description = "Partition identifier to call Kraftwerk on") @RequestParam("partitionId") String partitionId, @@ -119,8 +143,55 @@ public ResponseEntity saveSchedule( return ResponseEntity.ok().build(); } + // Should be refactored to make it restfull + @Operation(summary = "Schedule a Kraftwerk execution using the collection instrument") + @PutMapping(path = "/contexts/schedules") + @PreAuthorize("hasRole('USER_KRAFTWERK')") + public ResponseEntity saveScheduleWithCollectionInstrumentId( + @Parameter(description = "Collection instrument to call Kraftwerk on") @RequestParam("collectionInstrumentId") String collectionInstrumentId, + @Parameter(description = "Kraftwerk endpoint") @RequestParam(value = "serviceTocall", defaultValue = Constants.KRAFTWERK_MAIN_ENDPOINT) ServiceToCall serviceToCall, + @Parameter(description = "Frequency in Spring cron format (6 inputs, go to https://crontab.cronhub.io/ for generator) \n Example : 0 0 6 * * *") @RequestParam("frequency") String frequency, + @Parameter(description = "Schedule effective date and time", example = "2024-01-01T12:00:00") @RequestParam("scheduleBeginDate") LocalDateTime scheduleBeginDate, + @Parameter(description = "Schedule end date and time", example = "2024-01-01T12:00:00") @RequestParam("scheduleEndDate") LocalDateTime scheduleEndDate, + @Parameter(description = "Encrypt after process ? Ignore next parameters if false") @RequestParam(value = + "useEncryption", + defaultValue = "false") boolean useEncryption, + @Parameter(description = "(Encryption) vault path") @RequestParam(value = "encryptionVaultPath", defaultValue = "") String encryptionVaultPath, + @Parameter(description = "(Encryption) output folder") @RequestParam(value = "encryptionOutputFolder", + defaultValue = "") String encryptionOutputFolder, + @Parameter(description = "(Encryption) Use signature system") @RequestParam(value = "useSignature", defaultValue = "false") boolean useSignature + ) { + try { + //Check frequency + if(!CronExpression.isValidExpression(frequency)) { + log.warn("Returned error for wrong frequency : {}", frequency); + throw new GenesisException(400, "Wrong frequency syntax"); + } + + TrustParameters trustParameters = null; + if(useEncryption) { + trustParameters = new TrustParameters( + fileUtils.getKraftwerkOutFolder(collectionInstrumentId), + encryptionOutputFolder, + encryptionVaultPath, + useSignature + ); + } + dataProcessingContextApiPort.saveKraftwerkExecutionScheduleByCollectionInstrumentId( + collectionInstrumentId, + serviceToCall == null ? ServiceToCall.MAIN : serviceToCall, + frequency, + scheduleBeginDate, scheduleEndDate, trustParameters + ); + }catch (GenesisException e){ + return new ResponseEntity<>(e.getMessage(), HttpStatusCode.valueOf(e.getStatus())); + } + return ResponseEntity.ok().build(); + } + + @Deprecated(forRemoval = true) @Operation(summary = "Fetch all schedules") - @GetMapping(path = "/schedules") + @GetMapping(path = "/context/schedules") @PreAuthorize("hasAnyRole('SCHEDULER','READER')") public ResponseEntity getAllSchedules() { log.debug("Got GET all schedules request"); @@ -131,8 +202,23 @@ public ResponseEntity getAllSchedules() { return ResponseEntity.ok(surveyScheduleDocumentModels); } + //It is just a change of path in the url + @Operation(summary = "Fetch all schedules") + @GetMapping(path = "/contexts/schedules") + @PreAuthorize("hasAnyRole('SCHEDULER','READER')") + public ResponseEntity getAllSchedulesV2() { + log.debug("Got GET all schedules request"); + + List surveyScheduleDocumentModels = dataProcessingContextApiPort.getAllSchedules(); + + log.info("Returning {} schedule documents...", surveyScheduleDocumentModels.size()); + return ResponseEntity.ok(surveyScheduleDocumentModels); + } + + + @Deprecated(forRemoval = true) @Operation(summary = "Set last execution date of a partition with new date or nothing") - @PostMapping(path = "/schedules/lastExecutionDate") + @PostMapping(path = "/context/schedules/lastExecutionDate") @PreAuthorize("hasRole('SCHEDULER')") public ResponseEntity setSurveyLastExecution( @Parameter(description = "Survey name to call Kraftwerk on") @RequestBody String partitionId, @@ -147,8 +233,25 @@ public ResponseEntity setSurveyLastExecution( return ResponseEntity.ok().build(); } + @Operation(summary = "Update the date of the last extraction of data corresponding to a collection instrument") + @PutMapping(path = "/contexts/{collectionInstrumentId}/lastExecutionDate") + @PreAuthorize("hasRole('SCHEDULER')") + public ResponseEntity setSurveyLastExecutionByCollectionInstrumentId( + @PathVariable("collectionInstrumentId") @RequestBody String collectionInstrumentId, + @Parameter(description = "Date to save as last execution date", example = "2024-01-01T12:00:00") @RequestParam("newDate") LocalDateTime newDate + ) { + try { + dataProcessingContextApiPort.updateLastExecutionDateByCollectionInstrumentId(collectionInstrumentId, newDate); + log.info("{} last execution updated at {} !", collectionInstrumentId, newDate); + }catch (GenesisException e){ + return new ResponseEntity<>(e.getMessage(), HttpStatusCode.valueOf(e.getStatus())); + } + return ResponseEntity.ok().build(); + } + + @Deprecated(forRemoval = true) @Operation(summary = "Delete the Kraftwerk execution schedules of a partition") - @DeleteMapping(path = "/schedules") + @DeleteMapping(path = "/context/schedules") @PreAuthorize("hasRole('USER_KRAFTWERK')") public ResponseEntity deleteSchedules( @Parameter(description = "Survey name of the schedule(s) to delete") @RequestParam("partitionId") String partitionId @@ -162,36 +265,31 @@ public ResponseEntity deleteSchedules( return ResponseEntity.ok().build(); } + @Operation(summary = "Delete the Kraftwerk execution schedules of a collection instrument id") + @DeleteMapping(path = "/context/{collectionInstrumentId}/schedules") + @PreAuthorize("hasRole('USER_KRAFTWERK')") + public ResponseEntity deleteSchedulesByCollectionInstrumentId( + @PathVariable("collectionInstrumentId") String collectionInstrumentId + ){ + try { + dataProcessingContextApiPort.deleteSchedulesByCollectionInstrumentId(collectionInstrumentId); + }catch (GenesisException e){ + return new ResponseEntity<>(e.getMessage(), HttpStatusCode.valueOf(e.getStatus())); + } + log.info("Schedule deleted for survey {}", collectionInstrumentId); + return ResponseEntity.ok().build(); + } + @Operation(summary = "Delete expired schedules") - @DeleteMapping(path = "/schedules/expired-schedules") + @DeleteMapping(path = "/context/schedules/expired-schedules") @PreAuthorize("hasRole('SCHEDULER')") - public ResponseEntity deleteExpiredSchedules() throws GenesisException, IOException { - Set storedSurveySchedulesNames = new HashSet<>(); - for(ScheduleDto scheduleDto : dataProcessingContextApiPort.getAllSchedules()){ - storedSurveySchedulesNames.add(scheduleDto.surveyName()); - } - for (String surveyScheduleName : storedSurveySchedulesNames) { - List deletedKraftwerkExecutionSchedules = dataProcessingContextApiPort.deleteExpiredSchedules(surveyScheduleName); - //Save in JSON log - if(!deletedKraftwerkExecutionSchedules.isEmpty()) { - Path jsonLogPath = Path.of(fileUtils.getLogFolder(), Constants.SCHEDULE_ARCHIVE_FOLDER_NAME, - surveyScheduleName + ".json"); - ObjectMapper objectMapper = new ObjectMapper().findAndRegisterModules(); - objectMapper.registerModule(new JavaTimeModule()); - String jsonToWrite = objectMapper.writeValueAsString(deletedKraftwerkExecutionSchedules); - if(Files.exists(jsonLogPath)){ - //Remove last ] and append survey - StringBuilder content = new StringBuilder(Files.readString(jsonLogPath)); - content.setCharAt(content.length()-1, ','); - content.append(jsonToWrite, 1, jsonToWrite.length()-1); - content.append(']'); - Files.write(jsonLogPath, content.toString().getBytes(), StandardOpenOption.TRUNCATE_EXISTING); - }else { - Files.createDirectories(jsonLogPath.getParent()); - Files.write(jsonLogPath, jsonToWrite.getBytes()); - } - } + public ResponseEntity deleteExpiredSchedules(){ + try{ + dataProcessingContextApiPort.deleteExpiredSchedules(fileUtils.getLogFolder()); + } catch (GenesisException e){ + return new ResponseEntity<>(e.getMessage(), HttpStatusCode.valueOf(e.getStatus())); } + log.info("Expired schedules deleted"); return ResponseEntity.ok().build(); } } diff --git a/src/main/java/fr/insee/genesis/controller/rest/JsonExtractionController.java b/src/main/java/fr/insee/genesis/controller/rest/JsonExtractionController.java index 595ec153..79d60d16 100644 --- a/src/main/java/fr/insee/genesis/controller/rest/JsonExtractionController.java +++ b/src/main/java/fr/insee/genesis/controller/rest/JsonExtractionController.java @@ -30,11 +30,11 @@ public class JsonExtractionController { @PutMapping(path = "/json") @PreAuthorize("hasAnyRole('USER_KRAFTWERK','SCHEDULER')") public ResponseEntity saveLastJsonExtractionDate( - @RequestParam("questionnaireId") String questionnaireId, + @RequestParam("collectionInstrumentId") String collectionInstrumentId, @RequestParam(value = "mode", required = false) Mode mode){ LocalDateTime extractDate = LocalDateTime.now(); LastJsonExtractionModel extract = LastJsonExtractionModel.builder() - .questionnaireModelId(questionnaireId) + .collectionInstrumentId(collectionInstrumentId) .mode(mode) .lastExtractionDate(extractDate) .build(); @@ -46,10 +46,10 @@ public ResponseEntity saveLastJsonExtractionDate( @GetMapping(path = "/json") @PreAuthorize("hasAnyRole('USER_KRAFTWERK','SCHEDULER')") public ResponseEntity getLastJsonExtractionDate( - @RequestParam("questionnaireId") String questionnaireId, + @RequestParam("collectionInstrumentId") String collectionInstrumentId, @RequestParam(value = "mode", required = false) Mode mode){ try{ - LastJsonExtractionModel lastJsonExtraction = lastJsonExtractionApiPort.getLastExtractionDate(questionnaireId,mode); + LastJsonExtractionModel lastJsonExtraction = lastJsonExtractionApiPort.getLastExtractionDate(collectionInstrumentId,mode); return ResponseEntity.ok(new LastExtractionResponseDto(lastJsonExtraction.getLastExtractionDate())); } catch (GenesisException e){ return ResponseEntity.notFound().build(); diff --git a/src/main/java/fr/insee/genesis/controller/rest/responses/ModeController.java b/src/main/java/fr/insee/genesis/controller/rest/responses/ModeController.java index d7fef093..ac3b071d 100644 --- a/src/main/java/fr/insee/genesis/controller/rest/responses/ModeController.java +++ b/src/main/java/fr/insee/genesis/controller/rest/responses/ModeController.java @@ -26,10 +26,10 @@ public ModeController(SurveyUnitApiPort surveyUnitService) { } - @Operation(summary = "List sources/modes used for a given questionnaire") + @Operation(summary = "List sources/modes used for a given collection instrument (ex questionnaire)") @GetMapping(path = "/by-questionnaire") - public ResponseEntity> getModesByQuestionnaire(@RequestParam("questionnaireId") String questionnaireId) { - List modes = surveyUnitService.findModesByQuestionnaireId(questionnaireId); + public ResponseEntity> getModesByQuestionnaire(@RequestParam("questionnaireId") String collectionInstrumentId) { + List modes = surveyUnitService.findModesByCollectionInstrumentId(collectionInstrumentId); return ResponseEntity.ok(modes); } diff --git a/src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseController.java b/src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseController.java index 33df8669..b51aeb0e 100644 --- a/src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseController.java +++ b/src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseController.java @@ -1,20 +1,28 @@ package fr.insee.genesis.controller.rest.responses; import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; -import com.networknt.schema.Error; -import com.networknt.schema.Schema; -import com.networknt.schema.SchemaRegistry; -import com.networknt.schema.dialect.Dialects; +import com.fasterxml.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; import fr.insee.genesis.controller.dto.rawdata.LunaticJsonRawDataUnprocessedDto; +import fr.insee.genesis.controller.utils.ExtendedJsonNormalizer; +import fr.insee.genesis.controller.utils.JsonSchemaValidator; +import fr.insee.genesis.controller.utils.SchemaType; import fr.insee.genesis.domain.model.surveyunit.Mode; import fr.insee.genesis.domain.model.surveyunit.rawdata.DataProcessResult; import fr.insee.genesis.domain.model.surveyunit.rawdata.LunaticJsonRawDataModel; import fr.insee.genesis.domain.ports.api.LunaticJsonRawDataApiPort; +import fr.insee.genesis.domain.ports.api.RawResponseApiPort; import fr.insee.genesis.exceptions.GenesisError; import fr.insee.genesis.exceptions.GenesisException; +import fr.insee.genesis.exceptions.SchemaValidationException; +import fr.insee.genesis.infrastructure.repository.RawResponseInputRepository; +import fr.insee.modelefiliere.RawResponseDto; import io.swagger.v3.oas.annotations.Hidden; import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.Parameter; +import jakarta.validation.Valid; import lombok.extern.slf4j.Slf4j; import org.springframework.data.domain.Page; import org.springframework.data.domain.PageRequest; @@ -30,32 +38,34 @@ import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.PutMapping; import org.springframework.web.bind.annotation.RequestBody; -import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestParam; +import java.io.IOException; import java.time.Instant; import java.time.LocalDateTime; import java.util.ArrayList; import java.util.List; import java.util.Map; -import java.util.stream.Collectors; @Slf4j @Controller -@RequestMapping(path = "/responses/raw") public class RawResponseController { private static final String SUCCESS_MESSAGE = "Interrogation %s saved"; - private static final String PARTITION_ID = "partitionId"; private static final String INTERROGATION_ID = "interrogationId"; private final LunaticJsonRawDataApiPort lunaticJsonRawDataApiPort; + private final RawResponseApiPort rawResponseApiPort; + private final RawResponseInputRepository rawRepository; - public RawResponseController(LunaticJsonRawDataApiPort lunaticJsonRawDataApiPort) { + + public RawResponseController(LunaticJsonRawDataApiPort lunaticJsonRawDataApiPort, RawResponseApiPort rawResponseApiPort, RawResponseInputRepository rawRepository) { this.lunaticJsonRawDataApiPort = lunaticJsonRawDataApiPort; + this.rawResponseApiPort = rawResponseApiPort; + this.rawRepository = rawRepository; } @Operation(summary = "Save lunatic json data from one interrogation in Genesis Database") - @PutMapping(path = "/lunatic-json/save") + @PutMapping(path = "/responses/raw/lunatic-json/save") @PreAuthorize("hasRole('COLLECT_PLATFORM')") public ResponseEntity saveRawResponsesFromJsonBody( @@ -86,67 +96,113 @@ public ResponseEntity saveRawResponsesFromJsonBody( return ResponseEntity.status(201).body(String.format(SUCCESS_MESSAGE, interrogationId)); } + @Operation(summary = "Save lunatic json data from one interrogation in Genesis Database (with json " + "schema validation)") - @PutMapping(path = "/lunatic-json") + @PostMapping(path="/raw-responses/debug") @PreAuthorize("hasRole('COLLECT_PLATFORM')") public ResponseEntity saveRawResponsesFromJsonBodyWithValidation( @RequestBody Map body - ) { - SchemaRegistry schemaRegistry = SchemaRegistry.withDialect(Dialects.getDraft202012(), SchemaRegistry.Builder::build); - Schema jsonSchema = schemaRegistry - .getSchema(RawResponseController.class.getResourceAsStream("/jsonSchemas/RawResponse.json") - ); + ) throws JsonProcessingException { + + ObjectMapper objectMapperLocal = new ObjectMapper(); + objectMapperLocal.registerModule(new JavaTimeModule()); + objectMapperLocal.disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS); // ISO-8601 + objectMapperLocal.disable(DeserializationFeature.ADJUST_DATES_TO_CONTEXT_TIME_ZONE); + + log.info(objectMapperLocal.writeValueAsString(body)); try { - if (jsonSchema == null) { - throw new GenesisException(500, "No RawResponse json schema has been found"); - } - List errors = jsonSchema.validate( - new ObjectMapper().readTree( - new ObjectMapper().writeValueAsString(body) - ) + RawResponseDto rawResponseDto = JsonSchemaValidator.readAndValidateFromClasspath( + ExtendedJsonNormalizer.normalize(objectMapperLocal.readTree( + objectMapperLocal.writeValueAsString(body))), + SchemaType.RAW_RESPONSE.getSchemaFileName(), + RawResponseDto.class, + objectMapperLocal ); - // Throw Genesis exception if errors are present - validate(errors); - //Check required ids - checkRequiredIds(body); - } catch (JsonProcessingException jpe) { - return ResponseEntity.status(400).body(jpe.toString()); - } catch (GenesisException ge) { - return ResponseEntity.status(ge.getStatus()).body(ge.getMessage()); + rawRepository.saveAsRawJson(rawResponseDto); + } catch (SchemaValidationException | IOException e) { + return ResponseEntity.status(400).body(e.toString()); } + return ResponseEntity.status(201).body(String.format(SUCCESS_MESSAGE, body.get(INTERROGATION_ID).toString())); + } - LunaticJsonRawDataModel rawData = LunaticJsonRawDataModel.builder() - .campaignId(body.get(PARTITION_ID).toString()) - .questionnaireId(body.get("questionnaireModelId").toString().toUpperCase()) - .interrogationId(body.get(INTERROGATION_ID).toString()) - .idUE(body.get("surveyUnitId").toString()) - .mode(Mode.getEnumFromJsonName(body.get("mode").toString())) - .data(body) - .recordDate(LocalDateTime.now()) - .build(); + @Operation(summary = "Save lunatic json data from one interrogation in Genesis Database (with json " + + "schema validation)") + @PostMapping(path="/raw-responses") + @PreAuthorize("hasRole('COLLECT_PLATFORM')") + public ResponseEntity saveRawResponsesFromRawResponseDto( + @Valid @RequestBody RawResponseDto dto + ) { + rawRepository.saveAsRawJson(dto); + return ResponseEntity.status(201).body(String.format(SUCCESS_MESSAGE, dto.getInterrogationId())); + } + + //PROCESS + @Operation(summary = "Process raw data for a list of interrogations") + @PostMapping(path = "/raw-responses/process") + @PreAuthorize("hasRole('SCHEDULER')") + public ResponseEntity processRawResponses( + @Parameter( + description = "Id of the collection instrument (old questionnaireId)", + example = "ENQTEST2025X00" + ) + @RequestParam("collectionInstrumentId") String collectionInstrumentId, + @RequestBody List interrogationIdList + ) { + log.info("Try to process raw responses for collectionInstrumentId {} and {} interrogationIds", collectionInstrumentId, interrogationIdList.size()); + List errors = new ArrayList<>(); try { - lunaticJsonRawDataApiPort.save(rawData); - } catch (Exception e) { - return ResponseEntity.status(500).body("Unexpected error"); + DataProcessResult result = rawResponseApiPort.processRawResponses(collectionInstrumentId, interrogationIdList, errors); + return result.formattedDataCount() == 0 ? + ResponseEntity.ok("%d document(s) processed".formatted(result.dataCount())) + : ResponseEntity.ok("%d document(s) processed, including %d FORMATTED after data verification" + .formatted(result.dataCount(), result.formattedDataCount())); + } catch (GenesisException e) { + return ResponseEntity.status(e.getStatus()).body(e.getMessage()); } + } - log.info("Data saved for interrogationId {} and partition {}", body.get(INTERROGATION_ID).toString(), - body.get(PARTITION_ID).toString()); - return ResponseEntity.status(201).body(String.format(SUCCESS_MESSAGE, body.get(INTERROGATION_ID).toString())); + @Operation(summary = "Process raw data for all data of an collection instrument") + @PostMapping(path = "/raw-responses/{collectionInstrumentId}/process") + @PreAuthorize("hasRole('SCHEDULER')") + public ResponseEntity processRawResponsesByCollectionInstrumentId( + @Parameter( + description = "Id of the collection instrument (old questionnaireId)", + example = "ENQTEST2025X00" + ) + @PathVariable("collectionInstrumentId") String collectionInstrumentId + ) { + log.info("Try to process raw responses for collectionInstrumentId {}", collectionInstrumentId); + try { + DataProcessResult result = rawResponseApiPort.processRawResponses(collectionInstrumentId); + return result.formattedDataCount() == 0 ? + ResponseEntity.ok("%d document(s) processed".formatted(result.dataCount())) + : ResponseEntity.ok("%d document(s) processed, including %d FORMATTED after data verification" + .formatted(result.dataCount(), result.formattedDataCount())); + } catch (GenesisException e) { + return ResponseEntity.status(e.getStatus()).body(e.getMessage()); + } + } + + @Operation(summary = "Get the list of collection instruments containing unprocessed interrogations") + @GetMapping(path = "/raw-responses/unprocessed/collection-instrument-ids") + @PreAuthorize("hasRole('SCHEDULER')") + public ResponseEntity> getUnprocessedCollectionInstrument(){ + log.info("Try to get collection instruments containing unprocessed interrogations..."); + return ResponseEntity.ok(rawResponseApiPort.getUnprocessedCollectionInstrumentIds()); } //GET unprocessed @Operation(summary = "Get campaign id and interrogationId from all unprocessed raw json data") - @GetMapping(path = "/lunatic-json/get/unprocessed") + @GetMapping(path = "/responses/raw/lunatic-json/get/unprocessed") @PreAuthorize("hasRole('SCHEDULER')") - public ResponseEntity> getUnproccessedJsonRawData() { + public ResponseEntity> getUnprocessedJsonRawData() { log.info("Try to get unprocessed raw JSON datas..."); return ResponseEntity.ok(lunaticJsonRawDataApiPort.getUnprocessedDataIds()); } @Hidden - @GetMapping(path = "/lunatic-json/get/by-interrogation-mode-and-campaign") + @GetMapping(path = "/responses/raw/lunatic-json/get/by-interrogation-mode-and-campaign") @PreAuthorize("hasRole('ADMIN')") public ResponseEntity getJsonRawData( @RequestParam(INTERROGATION_ID) String interrogationId, @@ -159,7 +215,7 @@ public ResponseEntity getJsonRawData( //PROCESS @Operation(summary = "Process raw data of a campaign") - @PostMapping(path = "/lunatic-json/process") + @PostMapping(path = "/responses/raw/lunatic-json/process") @PreAuthorize("hasRole('SCHEDULER')") @Deprecated(since = "1.13.0") public ResponseEntity processJsonRawData( @@ -182,7 +238,7 @@ public ResponseEntity processJsonRawData( } @Operation(summary = "Process raw data of a questionnaire") - @PostMapping(path = "/{collectionInstrumentId}/process") + @PostMapping(path = "/responses/raw/lunatic-json/{collectionInstrumentId}/process") @PreAuthorize("hasRole('SCHEDULER')") public ResponseEntity processJsonRawData( @PathVariable String collectionInstrumentId @@ -200,7 +256,7 @@ public ResponseEntity processJsonRawData( } @Operation(summary = "Get processed data ids from last n hours (default 24h)") - @GetMapping(path = "/lunatic-json/processed/ids") + @GetMapping(path = "/responses/raw/lunatic-json/processed/ids") @PreAuthorize("hasRole('ADMIN')") public ResponseEntity>> getProcessedDataIdsSinceHours( @RequestParam("questionnaireId") String questionnaireId, @@ -212,7 +268,7 @@ public ResponseEntity>> getProcessedDataIdsSinceHours( } @Operation(summary = "Get lunatic JSON data from one campaign in Genesis Database, filtered by start and end dates") - @GetMapping(path = "/lunatic-json/{campaignId}") + @GetMapping(path = "/responses/raw/lunatic-json/{campaignId}") @PreAuthorize("hasRole('USER_BATCH_GENERIC')") public ResponseEntity> getRawResponsesFromJsonBody( @PathVariable String campaignId, @@ -228,31 +284,5 @@ public ResponseEntity> getRawResponsesFromJs return ResponseEntity.status(HttpStatus.OK).body(new PagedModel<>(rawResponses)); } - private void validate(List errors) throws GenesisException { - if (!errors.isEmpty()) { - String errorMessage = errors.stream() - .map(Error::getMessage) - .collect(Collectors.joining(System.lineSeparator() + " - ")); - - throw new GenesisException( - 400, - "Input data JSON is not valid: %n - %s".formatted(errorMessage) - ); - } - } - - private void checkRequiredIds(Map body) throws GenesisException { - for (String requiredKey : List.of( - PARTITION_ID, - "questionnaireModelId", - INTERROGATION_ID, - "surveyUnitId", - "mode" - )) { - if (body.get(requiredKey) == null) { - throw new GenesisException(400, "No %s found in body".formatted(requiredKey)); - } - } - } } diff --git a/src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java b/src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java index 5fea2f4e..f8d575ba 100644 --- a/src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java +++ b/src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java @@ -41,7 +41,14 @@ import org.springframework.lang.Nullable; import org.springframework.security.access.prepost.PreAuthorize; import org.springframework.stereotype.Controller; -import org.springframework.web.bind.annotation.*; +import org.springframework.web.bind.annotation.DeleteMapping; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.PutMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; import org.xml.sax.SAXException; import javax.xml.parsers.ParserConfigurationException; @@ -57,7 +64,7 @@ @RequestMapping(path = "/responses" ) @Controller -@Tag(name = "Response services", description = "A **response** is considered the entire set of data associated with an interrogation (survey unit x questionnaireId). \n\n These data may have different state (collected, edited, external, ...) ") +@Tag(name = "Response services", description = "A **response** is considered the entire set of data associated with an interrogation (survey unit x collecton Intrument Id [ex questionnaire]). \n\n These data may have different state (collected, edited, external, ...) ") @Slf4j public class ResponseController implements CommonApiResponse { @@ -91,6 +98,7 @@ public ResponseController(SurveyUnitApiPort surveyUnitService, } //SAVE + @Deprecated(since = "2026-01-01") @Operation(summary = "Save one file of responses to Genesis Database, passing its path as a parameter") @PutMapping(path = "/lunatic-xml/save-one") @PreAuthorize("hasRole('ADMIN')") @@ -98,7 +106,7 @@ public ResponseEntity saveResponsesFromXmlFile(@RequestParam("pathLunati @RequestParam(value = "pathSpecFile") String metadataFilePath, @RequestParam(value = "mode") Mode modeSpecified )throws Exception { - log.info("Try to read Xml file : {}", xmlFile); + log.info("Try to read one Xml file : {}", xmlFile); Path filepath = Paths.get(xmlFile); if (getFileSizeInMB(filepath) <= Constants.MAX_FILE_SIZE_UNTIL_SEQUENTIAL) { @@ -107,6 +115,7 @@ public ResponseEntity saveResponsesFromXmlFile(@RequestParam("pathLunati return processXmlFileSequentially(filepath, modeSpecified, metadataFilePath); } + @Deprecated(since = "2026-01-01") @Operation(summary = "Save multiple files to Genesis Database from the campaign root folder") @PutMapping(path = "/lunatic-xml/save-folder") @PreAuthorize("hasRole('ADMIN')") @@ -139,6 +148,7 @@ public ResponseEntity saveResponsesFromXmlCampaignFolder(@RequestParam(" } //SAVE ALL + @Deprecated(since = "2026-01-01") @Operation(summary = "Save all files to Genesis Database (differential data folder only), regardless of the campaign") @PutMapping(path = "/lunatic-xml/save-all-campaigns") @PreAuthorize("hasRole('SCHEDULER')") @@ -175,27 +185,41 @@ public ResponseEntity saveResponsesFromAllCampaignFolders(){ //DELETE - @Operation(summary = "Delete all responses associated with a questionnaire") - @DeleteMapping(path = "/delete/by-questionnaire") + @Operation(summary = "Delete all responses associated with a collection instrument (formerly questionnaire)") + @DeleteMapping(path = "/delete/{collectionInstrumentId}") @PreAuthorize("hasRole('ADMIN')") - public ResponseEntity deleteAllResponsesByQuestionnaire(@RequestParam("questionnaireId") String questionnaireId) { - log.info("Try to delete all responses of questionnaire : {}", questionnaireId); - Long ndDocuments = surveyUnitService.deleteByQuestionnaireId(questionnaireId); + public ResponseEntity deleteAllResponsesByCollectionInstrument(@PathVariable("collectionInstrumentId") String collectionInstrumentId) { + log.info("Try to delete all responses of collection instrument : {}", collectionInstrumentId); + Long ndDocuments = surveyUnitService.deleteByCollectionInstrumentId(collectionInstrumentId); log.info("{} responses deleted", ndDocuments); return ResponseEntity.ok(String.format("%d responses deleted", ndDocuments)); } //GET - @Operation(summary = "Retrieve responses for an interrogation, using interrogationId and questionnaireId from Genesis Database") - @GetMapping(path = "/by-ue-and-questionnaire") + @Operation(summary = "Retrieve responses for an interrogation, using interrogationId and collectionInstrumentId (formerly questionnaireId) from Genesis Database") + @GetMapping(path = "/by-interrogation-and-collection-instrument") @PreAuthorize("hasRole('ADMIN')") - public ResponseEntity> findResponsesByInterrogationAndQuestionnaire(@RequestParam("interrogationId") String interrogationId, - @RequestParam("questionnaireId") String questionnaireId) { - List responses = surveyUnitService.findByIdsInterrogationAndQuestionnaire(interrogationId, questionnaireId); + public ResponseEntity> findResponsesByInterrogationAndCollectionInstrument( + @RequestParam("interrogationId") String interrogationId, + @RequestParam("collectionInstrumentId") String collectionInstrumentId) + { + List responses = surveyUnitService.findByIdsInterrogationAndCollectionInstrument(interrogationId, collectionInstrumentId); return ResponseEntity.ok(responses); } - @Operation(summary = "Retrieve responses for an interrogation, using interrogationId and questionnaireId from Genesis Database with the latest value for each available state of every variable") + /** + * @deprecated + * This endpoint is deprecated because the parameter `questionnaireId` has been renamed + * to `collectionInstrumentId` in the Information System (modeled in the modelefiliere library). + * A new endpoint using the updated parameter names will be provided to remain compliant with + * the current data model. This endpoint will be removed once all dependent APIs have adopted + * the new naming convention. + * + * Use the new endpoint with `collectionInstrumentId` for future implementations. + */ + @Deprecated(forRemoval = true, since= "2026-01-01") + @Operation(summary = "Retrieve responses for an interrogation, using interrogationId and questionnaireId from Genesis Database with the latest value for each available state of every variable", + description = "use /by-interrogation-and-collection-instrument/latest-states instead") @GetMapping(path = "/by-ue-and-questionnaire/latest-states", produces = MediaType.APPLICATION_JSON_VALUE) @PreAuthorize("hasAnyRole('USER_PLATINE','SCHEDULER')") @@ -210,27 +234,46 @@ public ResponseEntity findResponsesByInterrogationAndQuestionnaireLatest return ResponseEntity.status(403).body(new ApiError("Review is disabled for that partition")); } - SurveyUnitDto response = surveyUnitService.findLatestValuesByStateByIdAndByQuestionnaireId(interrogationId, questionnaireId); + SurveyUnitDto response = surveyUnitService.findLatestValuesByStateByIdAndByCollectionInstrumentId(interrogationId, questionnaireId); SurveyUnitQualityToolDto responseQualityToolDto = DataTransformer.transformSurveyUnitDto(response); return ResponseEntity.ok(responseQualityToolDto); } - @Operation(summary = "Retrieve responses for an interrogation, using interrogationId and questionnaireId from Genesis Database. It returns only the latest value of each variable regardless of the state.") - @GetMapping(path = "/by-ue-and-questionnaire/latest") + @Operation(summary = "Retrieve the latest available values for each variable state for a given interrogation and collection instrument (formerly questionnaire).") + @GetMapping(path = "/by-interrogation-and-collection-instrument/latest-states", + produces = MediaType.APPLICATION_JSON_VALUE) + @PreAuthorize("hasAnyRole('USER_PLATINE','SCHEDULER')") + public ResponseEntity findResponsesByInterrogationAndCollectionInstrumentLatestStates( + @RequestParam("interrogationId") String interrogationId, + @RequestParam("collectionInstrumentId") String collectionInstrumentId) throws GenesisException { + //Check context + DataProcessingContextModel dataProcessingContextModel = contextService.getContext(interrogationId); + + if(dataProcessingContextModel == null || !dataProcessingContextModel.isWithReview()){ + return ResponseEntity.status(403).body(new ApiError("Review is disabled for that partition")); + } + + SurveyUnitDto response = surveyUnitService.findLatestValuesByStateByIdAndByCollectionInstrumentId(interrogationId, collectionInstrumentId); + SurveyUnitQualityToolDto responseQualityToolDto = DataTransformer.transformSurveyUnitDto(response); + return ResponseEntity.ok(responseQualityToolDto); + } + + @Operation(summary = "Retrieve responses for an interrogation, using interrogationId and collectionInstrumentId from Genesis Database. It returns only the latest value of each variable regardless of the state.") + @GetMapping(path = "/by-interrogation-and-collection-instrument/latest") @PreAuthorize("hasRole('ADMIN')") - public ResponseEntity> getLatestByInterrogation(@RequestParam("interrogationId") String interrogationId, - @RequestParam("questionnaireId") String questionnaireId) { - List responses = surveyUnitService.findLatestByIdAndByQuestionnaireId(interrogationId, questionnaireId); + public ResponseEntity> getLatestByInterrogationAndCollectionInstrument(@RequestParam("interrogationId") String interrogationId, + @RequestParam("collectionInstrumentId") String collectionInstrumentId) { + List responses = surveyUnitService.findLatestByIdAndByCollectionInstrumentId(interrogationId, collectionInstrumentId); return ResponseEntity.ok(responses); } - @Operation(summary = "Retrieve responses for an interrogation, using interrogationId and questionnaireId from Genesis Database. For a given mode, it returns only the latest value of each variable regardless of the state. The result is one object by mode in the output") - @GetMapping(path = "/simplified/by-ue-questionnaire-and-mode/latest") + @Operation(summary = "Retrieve responses for an interrogation, using interrogationId and collectionInstrumentId from Genesis Database. For a given mode, it returns only the latest value of each variable regardless of the state. The result is one object by mode in the output") + @GetMapping(path = "/simplified/by-interrogation-collection-instrument-and-mode/latest") @PreAuthorize("hasRole('USER_KRAFTWERK')") public ResponseEntity getLatestByInterrogationOneObject(@RequestParam("interrogationId") String interrogationId, - @RequestParam("questionnaireId") String questionnaireId, + @RequestParam("collectionInstrumentId") String collectionInstrumentId, @RequestParam("mode") Mode mode) { - List responses = surveyUnitService.findLatestByIdAndByQuestionnaireId(interrogationId, questionnaireId); + List responses = surveyUnitService.findLatestByIdAndByCollectionInstrumentId(interrogationId, collectionInstrumentId); List outputVariables = new ArrayList<>(); List outputExternalVariables = new ArrayList<>(); responses.stream().filter(rep -> rep.getMode().equals(mode)).forEach(response -> { @@ -238,27 +281,29 @@ public ResponseEntity getLatestByInterrogationOneObject(@R outputExternalVariables.addAll(response.getExternalVariables()); }); return ResponseEntity.ok(SurveyUnitSimplified.builder() - .questionnaireId(responses.getFirst().getQuestionnaireId()) + .collectionInstrumentId(responses.getFirst().getCollectionInstrumentId()) .campaignId(responses.getFirst().getCampaignId()) .interrogationId(responses.getFirst().getInterrogationId()) - .surveyUnitId(responses.getFirst().getIdUE()) + .usualSurveyUnitId(responses.getFirst().getUsualSurveyUnitId()) .variablesUpdate(outputVariables) .externalVariables(outputExternalVariables) .build()); } - @Operation(summary = "Retrieve all responses for a questionnaire and a list of UE", - description = "Return the latest state for each variable for the given ids and a given questionnaire.
" + + @Operation(summary = "Retrieve all responses for a collection instrument and a list of interrogations", + description = "Return the latest state for each variable for the given interrogationIds and a given collection instrument (formerly questionnaire).
" + "For a given id, the endpoint returns a document by collection mode (if there is more than one).") - @PostMapping(path = "/simplified/by-list-interrogation-and-questionnaire/latest") + @PostMapping(path = "/simplified/by-list-interrogation-and-collection-instrument/latest") @PreAuthorize("hasRole('USER_KRAFTWERK')") - public ResponseEntity> getLatestForInterrogationList(@RequestParam("questionnaireId") String questionnaireId, - @RequestBody List interrogationIds) { + public ResponseEntity> getLatestForInterrogationListAndCollectionInstrument( + @RequestParam("collectionInstrumentId") String collectionInstrumentId, + @RequestBody List interrogationIds) + { List results = new ArrayList<>(); - List modes = surveyUnitService.findModesByQuestionnaireId(questionnaireId); + List modes = surveyUnitService.findModesByCollectionInstrumentId(collectionInstrumentId); interrogationIds.forEach(interrogationId -> { - List responses = surveyUnitService.findLatestByIdAndByQuestionnaireId(interrogationId.getInterrogationId(), questionnaireId); + List responses = surveyUnitService.findLatestByIdAndByCollectionInstrumentId(interrogationId.getInterrogationId(), collectionInstrumentId); modes.forEach(mode -> { List outputVariables = new ArrayList<>(); List outputExternalVariables = new ArrayList<>(); @@ -268,10 +313,10 @@ public ResponseEntity> getLatestForInterrogationList( }); if (!outputVariables.isEmpty() || !outputExternalVariables.isEmpty()) { results.add(SurveyUnitSimplified.builder() - .questionnaireId(responses.getFirst().getQuestionnaireId()) + .collectionInstrumentId(responses.getFirst().getCollectionInstrumentId()) .campaignId(responses.getFirst().getCampaignId()) .interrogationId(responses.getFirst().getInterrogationId()) - .surveyUnitId(responses.getFirst().getIdUE()) + .usualSurveyUnitId(responses.getFirst().getUsualSurveyUnitId()) .mode(mode) .variablesUpdate(outputVariables) .externalVariables(outputExternalVariables) @@ -300,7 +345,7 @@ public ResponseEntity> getLatestForInterrogationListV //!!!WARNING!!! : FOR PERFORMANCES PURPOSES, WE DONT'MAKE REQUESTS ON INDIVIDUAL ELEMENTS ANYMORE, BUT ON A SUBLIST OF THE INPUTLIST final int SUBBLOCK_SIZE = 100; int offset = 0; - List interrogationIdsSubList = null; + List interrogationIdsSubList; for(String mode : modes) { @@ -348,7 +393,7 @@ private SurveyUnitSimplified fusionWithLastUpdated(List respons Mode modeWrapped = Mode.getEnumFromModeName(mode); simplifiedResponse = SurveyUnitSimplified.builder() - .questionnaireId(responsesForSingleInterrId.getFirst().getQuestionnaireId()) + .collectionInstrumentId(responsesForSingleInterrId.getFirst().getCollectionInstrumentId()) .campaignId(responsesForSingleInterrId.getFirst().getCampaignId()) .interrogationId(responsesForSingleInterrId.getFirst().getInterrogationId()) .mode(modeWrapped) diff --git a/src/main/java/fr/insee/genesis/controller/utils/ExtendedJsonNormalizer.java b/src/main/java/fr/insee/genesis/controller/utils/ExtendedJsonNormalizer.java new file mode 100644 index 00000000..6bb1c4bd --- /dev/null +++ b/src/main/java/fr/insee/genesis/controller/utils/ExtendedJsonNormalizer.java @@ -0,0 +1,51 @@ +package fr.insee.genesis.controller.utils; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fasterxml.jackson.databind.node.TextNode; +public class ExtendedJsonNormalizer { + + public static final String $_DATE = "$date"; + + private ExtendedJsonNormalizer() {} + + /** + * Recursively converts Mongo Extended JSON objects into simple types expected by the schema: + * - {"$date": "..."} -> TextNode("...") + * Leaves all other values untouched. Returns a structural copy of the node (does not mutate the original). + */ + + public static JsonNode normalize(JsonNode node) { + if (node == null || node.isNull() || node.isMissingNode()) return node; + + if (node.isObject()) { + ObjectNode obj = (ObjectNode) node; + + if (obj.size() == 1) { + if (obj.has($_DATE) && obj.get($_DATE).isTextual()) { + return TextNode.valueOf(obj.get($_DATE).asText()); + } +// if (obj.has("$oid") && obj.get("$oid").isTextual()) { +// return TextNode.valueOf(obj.get("$oid").asText()); +// } + } + + ObjectNode copy = obj.objectNode(); + obj.fields().forEachRemaining(e -> + copy.set(e.getKey(), normalize(e.getValue())) + ); + return copy; + } + + if (node.isArray()) { + ArrayNode src = (ArrayNode) node; + ArrayNode dst = src.arrayNode(); + for (JsonNode child : src) { + dst.add(normalize(child)); + } + return dst; + } + return node; + } +} diff --git a/src/main/java/fr/insee/genesis/controller/utils/JsonSchemaValidator.java b/src/main/java/fr/insee/genesis/controller/utils/JsonSchemaValidator.java new file mode 100644 index 00000000..9a033f49 --- /dev/null +++ b/src/main/java/fr/insee/genesis/controller/utils/JsonSchemaValidator.java @@ -0,0 +1,64 @@ +package fr.insee.genesis.controller.utils; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.networknt.schema.JsonSchema; +import com.networknt.schema.JsonSchemaFactory; +import com.networknt.schema.SpecVersion; +import com.networknt.schema.ValidationMessage; +import fr.insee.genesis.exceptions.SchemaValidationException; +import lombok.extern.slf4j.Slf4j; + +import java.io.IOException; +import java.io.InputStream; + +@Slf4j +public class JsonSchemaValidator { + + private JsonSchemaValidator() {} + + public static T readAndValidate(JsonNode root, + JsonSchema schema, + Class targetType, + ObjectMapper mapper) + throws SchemaValidationException, JsonProcessingException { + ensureValid(root, schema); + return mapper.treeToValue(root, targetType); + } + + public static T readAndValidateFromClasspath(JsonNode root, + String schemaResourcePath, + Class targetType, + ObjectMapper mapper) + throws SchemaValidationException, IOException { + JsonSchema schema = loadSchemaFromClasspath(schemaResourcePath, mapper); + return readAndValidate(root, schema, targetType, mapper); + } + + public static JsonSchema loadSchemaFromClasspath(String resourcePath, ObjectMapper mapper) throws IOException { + if (resourcePath == null || resourcePath.isBlank()) { + throw new IOException("Schema resource path is null or blank"); + } + String cp = resourcePath.startsWith("/") ? resourcePath.substring(1) : resourcePath; + try (InputStream in = Thread.currentThread().getContextClassLoader().getResourceAsStream(cp)) { + if (in == null){ throw new IOException("Schema not found on classpath: " + resourcePath);} + JsonNode schemaNode = mapper.readTree(in); + return JsonSchemaFactory.getInstance(SpecVersion.VersionFlag.V202012).getSchema(schemaNode); + } + } + + private static void ensureValid(JsonNode root, JsonSchema schema) throws SchemaValidationException { + log.info("Schéma Apply : {}", schema.getSchemaNode().get("title")); + java.util.Set errors = schema.validate(root); + if (!errors.isEmpty()) { + String formatted = errors.stream() + .sorted(java.util.Comparator.comparing(ValidationMessage::getEvaluationPath)) + .map(ValidationMessage::getMessage) + .collect(java.util.stream.Collectors.joining("\n")); + throw new SchemaValidationException( + "Uploaded JSON is not correct according to the json-schema:\n" + formatted, errors); + } + log.info("Schema-compliant JSON : {}", schema.getSchemaNode().get("title")); + } +} diff --git a/src/main/java/fr/insee/genesis/controller/utils/SchemaType.java b/src/main/java/fr/insee/genesis/controller/utils/SchemaType.java new file mode 100644 index 00000000..16171879 --- /dev/null +++ b/src/main/java/fr/insee/genesis/controller/utils/SchemaType.java @@ -0,0 +1,26 @@ +package fr.insee.genesis.controller.utils; + +import lombok.Getter; + +@Getter +public enum SchemaType { + + // PROCESS_MESSAGE(Names.PROCESS_MESSAGE), + INTERROGATION(Names.INTERROGATION), + RAW_RESPONSE(Names.RAW_RESPONSE); + + public static class Names { +// public static final String PROCESS_MESSAGE = "/modele-filiere-spec/Command.json"; + public static final String INTERROGATION = "/modele-filiere-spec/Interrogation.json"; + public static final String RAW_RESPONSE = "/modele-filiere-spec/RawResponse.json"; + + private Names() { + + } + } + private final String schemaFileName; + + SchemaType(String schemaFileName) { + this.schemaFileName = schemaFileName; + } +} diff --git a/src/main/java/fr/insee/genesis/domain/model/context/DataProcessingContextModel.java b/src/main/java/fr/insee/genesis/domain/model/context/DataProcessingContextModel.java index 10de6526..d0a14746 100644 --- a/src/main/java/fr/insee/genesis/domain/model/context/DataProcessingContextModel.java +++ b/src/main/java/fr/insee/genesis/domain/model/context/DataProcessingContextModel.java @@ -20,8 +20,11 @@ public class DataProcessingContextModel { @Id private ObjectId id; //Used to remove warning + @Deprecated(forRemoval = true) private String partitionId; + private String collectionInstrumentId; //QuestionnaireId + private LocalDateTime lastExecution; List kraftwerkExecutionScheduleList; @@ -31,6 +34,7 @@ public class DataProcessingContextModel { public ScheduleDto toScheduleDto(){ return ScheduleDto.builder() .surveyName(partitionId) + .collectionInstrumentId(collectionInstrumentId) .lastExecution(lastExecution) .kraftwerkExecutionScheduleList(kraftwerkExecutionScheduleList) .build(); diff --git a/src/main/java/fr/insee/genesis/domain/model/contextualvariable/ContextualExternalVariableModel.java b/src/main/java/fr/insee/genesis/domain/model/contextualvariable/ContextualExternalVariableModel.java index 8be579b3..fe45ebd6 100644 --- a/src/main/java/fr/insee/genesis/domain/model/contextualvariable/ContextualExternalVariableModel.java +++ b/src/main/java/fr/insee/genesis/domain/model/contextualvariable/ContextualExternalVariableModel.java @@ -9,7 +9,7 @@ @Data public class ContextualExternalVariableModel { String id; - String questionnaireId; + String collectionInstrumentId; String interrogationId; Map variables; } diff --git a/src/main/java/fr/insee/genesis/domain/model/contextualvariable/ContextualPreviousVariableModel.java b/src/main/java/fr/insee/genesis/domain/model/contextualvariable/ContextualPreviousVariableModel.java index 4ebba43c..030ad843 100644 --- a/src/main/java/fr/insee/genesis/domain/model/contextualvariable/ContextualPreviousVariableModel.java +++ b/src/main/java/fr/insee/genesis/domain/model/contextualvariable/ContextualPreviousVariableModel.java @@ -9,7 +9,7 @@ @Data public class ContextualPreviousVariableModel { String id; - String questionnaireId; + String collectionInstrumentId; String interrogationId; Map variables; String sourceState; diff --git a/src/main/java/fr/insee/genesis/domain/model/extraction/json/LastJsonExtractionModel.java b/src/main/java/fr/insee/genesis/domain/model/extraction/json/LastJsonExtractionModel.java index 861d9e6e..478f3e8b 100644 --- a/src/main/java/fr/insee/genesis/domain/model/extraction/json/LastJsonExtractionModel.java +++ b/src/main/java/fr/insee/genesis/domain/model/extraction/json/LastJsonExtractionModel.java @@ -16,7 +16,7 @@ public class LastJsonExtractionModel { @Id private String id; //Used to remove warning - String questionnaireModelId; + String collectionInstrumentId; Mode mode; LocalDateTime lastExtractionDate; } diff --git a/src/main/java/fr/insee/genesis/domain/model/lunaticmodel/LunaticModelModel.java b/src/main/java/fr/insee/genesis/domain/model/lunaticmodel/LunaticModelModel.java index 22d3f3d2..395bd342 100644 --- a/src/main/java/fr/insee/genesis/domain/model/lunaticmodel/LunaticModelModel.java +++ b/src/main/java/fr/insee/genesis/domain/model/lunaticmodel/LunaticModelModel.java @@ -7,7 +7,7 @@ @Builder public record LunaticModelModel ( - String questionnaireId, + String collectionInstrumentId, Map lunaticModel, LocalDateTime recordDate ){} diff --git a/src/main/java/fr/insee/genesis/domain/model/metadata/QuestionnaireMetadataModel.java b/src/main/java/fr/insee/genesis/domain/model/metadata/QuestionnaireMetadataModel.java index b924b2b2..d5291b21 100644 --- a/src/main/java/fr/insee/genesis/domain/model/metadata/QuestionnaireMetadataModel.java +++ b/src/main/java/fr/insee/genesis/domain/model/metadata/QuestionnaireMetadataModel.java @@ -2,9 +2,11 @@ import fr.insee.bpm.metadata.model.MetadataModel; import fr.insee.genesis.domain.model.surveyunit.Mode; +import lombok.Builder; +@Builder public record QuestionnaireMetadataModel ( - String questionnaireId, + String collectionInstrumentId, Mode mode, MetadataModel metadataModel ){} diff --git a/src/main/java/fr/insee/genesis/domain/model/surveyunit/Mode.java b/src/main/java/fr/insee/genesis/domain/model/surveyunit/Mode.java index cbf45b86..b3ce4303 100644 --- a/src/main/java/fr/insee/genesis/domain/model/surveyunit/Mode.java +++ b/src/main/java/fr/insee/genesis/domain/model/surveyunit/Mode.java @@ -1,13 +1,21 @@ package fr.insee.genesis.domain.model.surveyunit; +import com.fasterxml.jackson.annotation.JsonCreator; import io.swagger.v3.oas.annotations.media.Schema; import lombok.Getter; import org.springframework.lang.Nullable; +import java.util.HashMap; +import java.util.Map; + @Getter public enum Mode { - WEB("WEB", "WEB", "CAWI"),TEL("TEL", "ENQ", "CATI"), F2F("F2F", "ENQ", "CAPI"),OTHER("OTHER", "", ""),PAPER("PAPER", "", "PAPI"); + WEB("WEB", "WEB", "CAWI"), + TEL("TEL", "ENQ", "CATI"), + F2F("F2F", "ENQ", "CAPI"), + OTHER("OTHER", "", ""), + PAPER("PAPER", "", "PAPI"); @Nullable @Schema(nullable = true, type = "string", allowableValues = { "WEB", "TEL", "F2F", "PAPER", "OTHER" }) @@ -23,28 +31,42 @@ public enum Mode { this.jsonName = jsonName; } - public static Mode getEnumFromModeName(String modeName) { - if (modeName == null){ - return null; - } - for (Mode mode : Mode.values()) { - if (modeName.equals(mode.getModeName())) { - return mode; - } - } - return null; - } + // lookup Maps + private static final Map BY_MODE_NAME = new HashMap<>(); + private static final Map BY_JSON_NAME = new HashMap<>(); + private static final Map BY_ANY_NAME = new HashMap<>(); - public static Mode getEnumFromJsonName(String modeName) { - if (modeName == null){ - return null; - } - for (Mode mode : Mode.values()) { - if (modeName.equals(mode.getJsonName())) { - return mode; - } - } - return null; - } + static { + for (Mode mode : values()) { + if (mode.modeName != null && !mode.modeName.isBlank()) { + BY_MODE_NAME.put(mode.modeName.toUpperCase(), mode); + BY_ANY_NAME.put(mode.modeName.toUpperCase(), mode); + } + if (mode.jsonName != null && !mode.jsonName.isBlank()) { + BY_JSON_NAME.put(mode.jsonName.toUpperCase(), mode); + BY_ANY_NAME.put(mode.jsonName.toUpperCase(), mode); + } + } + } + + @JsonCreator + public static Mode fromString(String value) { + if (value == null){ return null;} + + Mode mode = BY_ANY_NAME.get(value.trim().toUpperCase()); + if (mode != null) { return mode; } + + throw new IllegalArgumentException("Invalid Mode: " + value); + } + + public static Mode getEnumFromModeName(@Nullable String modeName) { + if (modeName == null) { return null; } + return BY_MODE_NAME.get(modeName.trim().toUpperCase()); + } + + public static Mode getEnumFromJsonName(@Nullable String jsonName) { + if (jsonName == null) { return null; } + return BY_JSON_NAME.get(jsonName.trim().toUpperCase()); + } } diff --git a/src/main/java/fr/insee/genesis/domain/model/surveyunit/SurveyUnitModel.java b/src/main/java/fr/insee/genesis/domain/model/surveyunit/SurveyUnitModel.java index d1fc8bd5..212adb42 100644 --- a/src/main/java/fr/insee/genesis/domain/model/surveyunit/SurveyUnitModel.java +++ b/src/main/java/fr/insee/genesis/domain/model/surveyunit/SurveyUnitModel.java @@ -10,19 +10,33 @@ import java.util.List; import java.util.Objects; +/* + * This model class should be in equation with the model of our information system ("modèle filière) + * Its up to the adapter and mappers to deal with old fields + * + */ @Builder @Data @NoArgsConstructor @AllArgsConstructor public class SurveyUnitModel { - private String questionnaireId; + // New name of questionnaireId + private String collectionInstrumentId; + // To be removed + /** + * @deprecated We will not receive this identifier anymore + */ + @Deprecated(forRemoval = true, since = "2026-01-01") private String campaignId; private String interrogationId; - private String idUE; + // New name of idUE + private String usualSurveyUnitId; + private String technicalSurveyUnitId; + // Represents the major version of the "modèle filière" + private String majorModelVersion; private DataState state; private Mode mode; - private String contextualId; private Boolean isCapturedIndirectly; private LocalDateTime validationDate; @JsonFormat(shape = JsonFormat.Shape.STRING, pattern = "yyyy-MM-dd'T'hh:mm") diff --git a/src/main/java/fr/insee/genesis/domain/model/surveyunit/rawdata/RawResponse.java b/src/main/java/fr/insee/genesis/domain/model/surveyunit/rawdata/RawResponse.java new file mode 100644 index 00000000..7f50b76c --- /dev/null +++ b/src/main/java/fr/insee/genesis/domain/model/surveyunit/rawdata/RawResponse.java @@ -0,0 +1,18 @@ +package fr.insee.genesis.domain.model.surveyunit.rawdata; + +import fr.insee.genesis.domain.model.surveyunit.Mode; +import org.bson.types.ObjectId; + +import java.time.LocalDateTime; +import java.util.Map; + +public record RawResponse( + ObjectId id, + String interrogationId, + String collectionInstrumentId, + Mode mode, + Map payload, + LocalDateTime recordDate, + LocalDateTime processDate +) +{} diff --git a/src/main/java/fr/insee/genesis/domain/model/variabletype/VariableTypeModel.java b/src/main/java/fr/insee/genesis/domain/model/variabletype/VariableTypeModel.java deleted file mode 100644 index c0e79a4d..00000000 --- a/src/main/java/fr/insee/genesis/domain/model/variabletype/VariableTypeModel.java +++ /dev/null @@ -1,13 +0,0 @@ -package fr.insee.genesis.domain.model.variabletype; - -import fr.insee.bpm.metadata.model.VariablesMap; -import fr.insee.genesis.domain.model.surveyunit.Mode; -import lombok.Builder; - -@Builder -public record VariableTypeModel ( - String campaignId, - String questionnaireId, - Mode mode, - VariablesMap variablesMap -){} diff --git a/src/main/java/fr/insee/genesis/domain/ports/api/ContextualExternalVariableApiPort.java b/src/main/java/fr/insee/genesis/domain/ports/api/ContextualExternalVariableApiPort.java index 8c085834..ac889d39 100644 --- a/src/main/java/fr/insee/genesis/domain/ports/api/ContextualExternalVariableApiPort.java +++ b/src/main/java/fr/insee/genesis/domain/ports/api/ContextualExternalVariableApiPort.java @@ -4,7 +4,7 @@ import fr.insee.genesis.exceptions.GenesisException; public interface ContextualExternalVariableApiPort { - boolean readContextualExternalFile(String questionnaireId, String filePath) throws GenesisException; + boolean readContextualExternalFile(String collectionInstrumentId, String filePath) throws GenesisException; - ContextualExternalVariableModel findByQuestionnaireIdAndInterrogationId(String questionnaireId, String interrogationId); + ContextualExternalVariableModel findByCollectionInstrumentIdAndInterrogationId(String collectionInstrumentId, String interrogationId); } diff --git a/src/main/java/fr/insee/genesis/domain/ports/api/ContextualPreviousVariableApiPort.java b/src/main/java/fr/insee/genesis/domain/ports/api/ContextualPreviousVariableApiPort.java index 1580d103..dd6aa4df 100644 --- a/src/main/java/fr/insee/genesis/domain/ports/api/ContextualPreviousVariableApiPort.java +++ b/src/main/java/fr/insee/genesis/domain/ports/api/ContextualPreviousVariableApiPort.java @@ -4,7 +4,7 @@ import fr.insee.genesis.exceptions.GenesisException; public interface ContextualPreviousVariableApiPort { - boolean readContextualPreviousFile(String questionnaireId, String sourceState, String filePath) throws GenesisException; + boolean readContextualPreviousFile(String collectionInstrumentId, String sourceState, String filePath) throws GenesisException; - ContextualPreviousVariableModel findByQuestionnaireIdAndInterrogationId(String questionnaireId, String interrogationId); + ContextualPreviousVariableModel findByCollectionInstrumentIdAndInterrogationId(String collectionInstrumentId, String interrogationId); } diff --git a/src/main/java/fr/insee/genesis/domain/ports/api/ContextualVariableApiPort.java b/src/main/java/fr/insee/genesis/domain/ports/api/ContextualVariableApiPort.java index 3626bcf5..152321ee 100644 --- a/src/main/java/fr/insee/genesis/domain/ports/api/ContextualVariableApiPort.java +++ b/src/main/java/fr/insee/genesis/domain/ports/api/ContextualVariableApiPort.java @@ -5,6 +5,6 @@ import fr.insee.genesis.infrastructure.utils.FileUtils; public interface ContextualVariableApiPort { - ContextualVariableModel getContextualVariable(String questionnaireId, String interrogationId); - int saveContextualVariableFiles(String questionnaireId, FileUtils fileUtils,String contextualFolderPath) throws GenesisException; + ContextualVariableModel getContextualVariable(String collectionInstrumentId, String interrogationId); + int saveContextualVariableFiles(String collectionInstrumentId, FileUtils fileUtils, String contextualFolderPath) throws GenesisException; } \ No newline at end of file diff --git a/src/main/java/fr/insee/genesis/domain/ports/api/DataProcessingContextApiPort.java b/src/main/java/fr/insee/genesis/domain/ports/api/DataProcessingContextApiPort.java index 077cdc83..e38d4ac8 100644 --- a/src/main/java/fr/insee/genesis/domain/ports/api/DataProcessingContextApiPort.java +++ b/src/main/java/fr/insee/genesis/domain/ports/api/DataProcessingContextApiPort.java @@ -2,7 +2,6 @@ import fr.insee.genesis.controller.dto.ScheduleDto; import fr.insee.genesis.domain.model.context.DataProcessingContextModel; -import fr.insee.genesis.domain.model.context.schedule.KraftwerkExecutionSchedule; import fr.insee.genesis.domain.model.context.schedule.ServiceToCall; import fr.insee.genesis.domain.model.context.schedule.TrustParameters; import fr.insee.genesis.exceptions.GenesisException; @@ -12,7 +11,9 @@ public interface DataProcessingContextApiPort { void saveContext(String partitionId, Boolean withReview) throws GenesisException; + void saveContextByCollectionInstrumentId(String collectionInstrumentID, Boolean withReview) throws GenesisException; + @Deprecated(forRemoval = true) void saveKraftwerkExecutionSchedule(String partitionId, ServiceToCall serviceToCall, String frequency, @@ -20,18 +21,29 @@ void saveKraftwerkExecutionSchedule(String partitionId, LocalDateTime endDate, TrustParameters trustParameters) throws GenesisException; + void saveKraftwerkExecutionScheduleByCollectionInstrumentId(String collectionInstrumentId, + ServiceToCall serviceToCall, + String frequency, + LocalDateTime startDate, + LocalDateTime endDate, + TrustParameters trustParameters) throws GenesisException; + void updateLastExecutionDate(String surveyName, LocalDateTime newDate) throws GenesisException; + void updateLastExecutionDateByCollectionInstrumentId(String collectionInstrumentId, LocalDateTime newDate) throws GenesisException; void deleteSchedules(String surveyName) throws GenesisException; + void deleteSchedulesByCollectionInstrumentId(String collectionInstrumentId) throws GenesisException; List getAllSchedules(); - List deleteExpiredSchedules(String surveyScheduleName) throws GenesisException; + void deleteExpiredSchedules(String logFolder) throws GenesisException; long countSchedules(); DataProcessingContextModel getContext(String interrogationId) throws GenesisException; - DataProcessingContextModel getContextByCollectionInstrumentId(String partitionId) throws GenesisException; + + DataProcessingContextModel getContextByCollectionInstrumentId(String collectionInstrumentId); + List getPartitionIds(boolean withReview); /** @@ -39,5 +51,10 @@ void saveKraftwerkExecutionSchedule(String partitionId, * @param partitionId id of the partition * @return the review indicator stored in genesis */ + @Deprecated(forRemoval = true) boolean getReviewByPartitionId(String partitionId) throws GenesisException; + + boolean getReviewByCollectionInstrumentId(String collectionInstrumentId) throws GenesisException; + + } diff --git a/src/main/java/fr/insee/genesis/domain/ports/api/LastJsonExtractionApiPort.java b/src/main/java/fr/insee/genesis/domain/ports/api/LastJsonExtractionApiPort.java index 196f3b10..331e2b3d 100644 --- a/src/main/java/fr/insee/genesis/domain/ports/api/LastJsonExtractionApiPort.java +++ b/src/main/java/fr/insee/genesis/domain/ports/api/LastJsonExtractionApiPort.java @@ -7,5 +7,5 @@ public interface LastJsonExtractionApiPort { void recordDate(LastJsonExtractionModel extraction); - LastJsonExtractionModel getLastExtractionDate(String questionnaireModelId, Mode mode) throws GenesisException; + LastJsonExtractionModel getLastExtractionDate(String collectionInstrumentId, Mode mode) throws GenesisException; } diff --git a/src/main/java/fr/insee/genesis/domain/ports/api/LunaticModelApiPort.java b/src/main/java/fr/insee/genesis/domain/ports/api/LunaticModelApiPort.java index 6150ac77..bc85632e 100644 --- a/src/main/java/fr/insee/genesis/domain/ports/api/LunaticModelApiPort.java +++ b/src/main/java/fr/insee/genesis/domain/ports/api/LunaticModelApiPort.java @@ -6,7 +6,6 @@ import java.util.Map; public interface LunaticModelApiPort { - void save(String questionnaireId, Map lunaticModel); - - LunaticModelModel get(String questionnaireId) throws GenesisException; + void save(String collectionInstrumentId, Map lunaticModel); + LunaticModelModel get(String collectionInstrumentId) throws GenesisException; } diff --git a/src/main/java/fr/insee/genesis/domain/ports/api/QuestionnaireMetadataApiPort.java b/src/main/java/fr/insee/genesis/domain/ports/api/QuestionnaireMetadataApiPort.java index 5ccca0e8..33593726 100644 --- a/src/main/java/fr/insee/genesis/domain/ports/api/QuestionnaireMetadataApiPort.java +++ b/src/main/java/fr/insee/genesis/domain/ports/api/QuestionnaireMetadataApiPort.java @@ -9,10 +9,10 @@ import java.util.List; public interface QuestionnaireMetadataApiPort { - MetadataModel find(String questionnaireId, Mode mode) throws GenesisException; - MetadataModel loadAndSaveIfNotExists(String campaignName, String questionnaireId, Mode mode, FileUtils fileUtils, + MetadataModel find(String collectionInstrumentId, Mode mode) throws GenesisException; + MetadataModel loadAndSaveIfNotExists(String campaignName, String collectionInstrumentId, Mode mode, FileUtils fileUtils, List errors) throws GenesisException; - void remove(String questionnaireId, Mode mode); + void remove(String collectionInstrumentId, Mode mode); - void save(String questionnaireId, Mode mode, MetadataModel metadataModel); + void save(String collectionInstrumentId, Mode mode, MetadataModel metadataModel); } diff --git a/src/main/java/fr/insee/genesis/domain/ports/api/RawResponseApiPort.java b/src/main/java/fr/insee/genesis/domain/ports/api/RawResponseApiPort.java new file mode 100644 index 00000000..d17a1141 --- /dev/null +++ b/src/main/java/fr/insee/genesis/domain/ports/api/RawResponseApiPort.java @@ -0,0 +1,22 @@ +package fr.insee.genesis.domain.ports.api; + +import fr.insee.bpm.metadata.model.VariablesMap; +import fr.insee.genesis.domain.model.surveyunit.Mode; +import fr.insee.genesis.domain.model.surveyunit.SurveyUnitModel; +import fr.insee.genesis.domain.model.surveyunit.rawdata.DataProcessResult; +import fr.insee.genesis.domain.model.surveyunit.rawdata.RawResponse; +import fr.insee.genesis.exceptions.GenesisError; +import fr.insee.genesis.exceptions.GenesisException; + +import java.util.List; + +public interface RawResponseApiPort { + + List getRawResponses(String collectionInstrumentId, Mode mode, List interrogationIdList); + DataProcessResult processRawResponses(String collectionInstrumentId, List interrogationIdList, List errors) throws GenesisException; + DataProcessResult processRawResponses(String collectionInstrumentId) throws GenesisException; + List convertRawResponse(List rawResponses, VariablesMap variablesMap); + List getUnprocessedCollectionInstrumentIds(); + void updateProcessDates(List surveyUnitModels); + +} diff --git a/src/main/java/fr/insee/genesis/domain/ports/api/SurveyUnitApiPort.java b/src/main/java/fr/insee/genesis/domain/ports/api/SurveyUnitApiPort.java index 9465a321..7a65a943 100644 --- a/src/main/java/fr/insee/genesis/domain/ports/api/SurveyUnitApiPort.java +++ b/src/main/java/fr/insee/genesis/domain/ports/api/SurveyUnitApiPort.java @@ -2,10 +2,10 @@ import fr.insee.bpm.metadata.model.VariablesMap; import fr.insee.genesis.controller.dto.CampaignWithQuestionnaire; -import fr.insee.genesis.domain.model.surveyunit.InterrogationId; import fr.insee.genesis.controller.dto.QuestionnaireWithCampaign; import fr.insee.genesis.controller.dto.SurveyUnitDto; import fr.insee.genesis.controller.dto.SurveyUnitInputDto; +import fr.insee.genesis.domain.model.surveyunit.InterrogationId; import fr.insee.genesis.domain.model.surveyunit.Mode; import fr.insee.genesis.domain.model.surveyunit.SurveyUnitModel; import fr.insee.genesis.exceptions.GenesisException; @@ -13,26 +13,23 @@ import java.time.LocalDateTime; import java.util.List; import java.util.Set; -import java.util.stream.Stream; public interface SurveyUnitApiPort { void saveSurveyUnits(List suList); - List findByIdsInterrogationAndQuestionnaire(String interrogationId, String questionnaireId); + List findByIdsInterrogationAndCollectionInstrument(String interrogationId, String questionnaireId); List findByInterrogationId(String interrogationId); - Stream findByQuestionnaireId(String questionnaireId); - - List findLatestByIdAndByQuestionnaireId(String interrogationId, String questionnaireId); + List findLatestByIdAndByCollectionInstrumentId(String interrogationId, String collectionInstrumentId); //========= OPTIMISATIONS PERFS (START) ========== List> findLatestByIdAndByQuestionnaireIdAndModeOrdered(String questionnaireId, String mode, List interrogationIds); //========= OPTIMISATIONS PERFS (END) ========== - SurveyUnitDto findLatestValuesByStateByIdAndByQuestionnaireId(String interrogationId, String questionnaireId) throws GenesisException; + SurveyUnitDto findLatestValuesByStateByIdAndByCollectionInstrumentId(String interrogationId, String collectionInstrumentId) throws GenesisException; List findInterrogationIdsAndModesByQuestionnaireId(String questionnaireId); @@ -49,7 +46,7 @@ List findDistinctPageableInterrogationIdsByQuestionnaireId(Stri List findModesByQuestionnaireIdV2(String questionnaireId); //========= OPTIMISATIONS PERFS (END) ========== - List findModesByQuestionnaireId(String questionnaireId); + List findModesByCollectionInstrumentId(String collectionInstrumentId); List findModesByCampaignId(String campaignId); @@ -57,7 +54,7 @@ List findDistinctPageableInterrogationIdsByQuestionnaireId(Stri List findModesByCampaignIdV2(String campaignId); //========= OPTIMISATIONS PERFS (END) ========== - Long deleteByQuestionnaireId(String questionnaireId); + Long deleteByCollectionInstrumentId(String collectionInstrumentId); long countResponses(); diff --git a/src/main/java/fr/insee/genesis/domain/ports/api/VariableTypeApiPort.java b/src/main/java/fr/insee/genesis/domain/ports/api/VariableTypeApiPort.java deleted file mode 100644 index 17220bfe..00000000 --- a/src/main/java/fr/insee/genesis/domain/ports/api/VariableTypeApiPort.java +++ /dev/null @@ -1,8 +0,0 @@ -package fr.insee.genesis.domain.ports.api; - -import fr.insee.bpm.metadata.model.VariablesMap; -import fr.insee.genesis.domain.model.surveyunit.Mode; - -public interface VariableTypeApiPort { - void saveMetadatas(String campaignId, String questionnaireId, Mode mode, VariablesMap variablesMap); -} diff --git a/src/main/java/fr/insee/genesis/domain/ports/spi/ContextualExternalVariablePersistancePort.java b/src/main/java/fr/insee/genesis/domain/ports/spi/ContextualExternalVariablePersistancePort.java index 37ed1ef7..7b499021 100644 --- a/src/main/java/fr/insee/genesis/domain/ports/spi/ContextualExternalVariablePersistancePort.java +++ b/src/main/java/fr/insee/genesis/domain/ports/spi/ContextualExternalVariablePersistancePort.java @@ -5,10 +5,10 @@ import java.util.List; public interface ContextualExternalVariablePersistancePort { - void backup(String questionnaireId); - void deleteBackup(String questionnaireId); - void restoreBackup(String questionnaireId); + void backup(String collectionInstrumentId); + void deleteBackup(String collectionInstrumentId); + void restoreBackup(String collectionInstrumentId); void saveAll(List contextualPreviousVariableModelList); - void delete(String questionnaireId); - ContextualExternalVariableModel findByQuestionnaireIdAndInterrogationId(String questionnaireId, String interrogationId); + void delete(String collectionInstrumentId); + ContextualExternalVariableModel findByCollectionInstrumentIdAndInterrogationId(String collectionInstrumentId, String interrogationId); } \ No newline at end of file diff --git a/src/main/java/fr/insee/genesis/domain/ports/spi/ContextualPreviousVariablePersistancePort.java b/src/main/java/fr/insee/genesis/domain/ports/spi/ContextualPreviousVariablePersistancePort.java index d791e91b..57d92931 100644 --- a/src/main/java/fr/insee/genesis/domain/ports/spi/ContextualPreviousVariablePersistancePort.java +++ b/src/main/java/fr/insee/genesis/domain/ports/spi/ContextualPreviousVariablePersistancePort.java @@ -5,10 +5,10 @@ import java.util.List; public interface ContextualPreviousVariablePersistancePort { - void backup(String questionnaireId); - void deleteBackup(String questionnaireId); - void restoreBackup(String questionnaireId); + void backup(String collectionInstrumentId); + void deleteBackup(String collectionInstrumentId); + void restoreBackup(String collectionInstrumentId); void saveAll(List contextualPreviousVariableModelList); - void delete(String questionnaireId); - ContextualPreviousVariableModel findByQuestionnaireIdAndInterrogationId(String questionnaireId, String interrogationId); + void delete(String collectionInstrumentId); + ContextualPreviousVariableModel findByCollectionInstrumentIdAndInterrogationId(String collectionInstrumentId, String interrogationId); } \ No newline at end of file diff --git a/src/main/java/fr/insee/genesis/domain/ports/spi/DataProcessingContextPersistancePort.java b/src/main/java/fr/insee/genesis/domain/ports/spi/DataProcessingContextPersistancePort.java index eb579253..1b4b4702 100644 --- a/src/main/java/fr/insee/genesis/domain/ports/spi/DataProcessingContextPersistancePort.java +++ b/src/main/java/fr/insee/genesis/domain/ports/spi/DataProcessingContextPersistancePort.java @@ -4,6 +4,7 @@ import fr.insee.genesis.domain.model.context.schedule.KraftwerkExecutionSchedule; import fr.insee.genesis.infrastructure.document.context.DataProcessingContextDocument; +import java.io.IOException; import java.util.List; public interface DataProcessingContextPersistancePort { @@ -11,6 +12,10 @@ public interface DataProcessingContextPersistancePort { List findByPartitionIds(List partitionIds); + DataProcessingContextModel findByCollectionInstrumentId(String collectionInstrumentId); + + List findByCollectionInstrumentIds(List collectionInstrumentIds); + void save(DataProcessingContextDocument dataProcessingContextDocument); void saveAll(List dataProcessingContextDocuments); @@ -21,7 +26,7 @@ public interface DataProcessingContextPersistancePort { long count(); - List removeExpiredSchedules(DataProcessingContextModel dataProcessingContextModel); + List removeExpiredSchedules(DataProcessingContextModel dataProcessingContextModel) throws IOException; List findAllByReview(boolean withReview); } diff --git a/src/main/java/fr/insee/genesis/domain/ports/spi/LastJsonExtractionPersistencePort.java b/src/main/java/fr/insee/genesis/domain/ports/spi/LastJsonExtractionPersistencePort.java index 39265b50..53a58e0f 100644 --- a/src/main/java/fr/insee/genesis/domain/ports/spi/LastJsonExtractionPersistencePort.java +++ b/src/main/java/fr/insee/genesis/domain/ports/spi/LastJsonExtractionPersistencePort.java @@ -6,5 +6,5 @@ public interface LastJsonExtractionPersistencePort { void save(LastJsonExtractionModel extraction); - LastJsonExtractionModel getLastExecutionDate(String questionnaireModelId, Mode mode) throws GenesisException; + LastJsonExtractionModel getLastExecutionDate(String collectionInstrumentId, Mode mode) throws GenesisException; } diff --git a/src/main/java/fr/insee/genesis/domain/ports/spi/LunaticModelPersistancePort.java b/src/main/java/fr/insee/genesis/domain/ports/spi/LunaticModelPersistancePort.java index 81bcfd81..632991f2 100644 --- a/src/main/java/fr/insee/genesis/domain/ports/spi/LunaticModelPersistancePort.java +++ b/src/main/java/fr/insee/genesis/domain/ports/spi/LunaticModelPersistancePort.java @@ -8,5 +8,5 @@ public interface LunaticModelPersistancePort { void save(LunaticModelModel lunaticModelModel); - List find(String questionnaireId); + List find(String collectionInstrumentId); } diff --git a/src/main/java/fr/insee/genesis/domain/ports/spi/QuestionnaireMetadataPersistancePort.java b/src/main/java/fr/insee/genesis/domain/ports/spi/QuestionnaireMetadataPersistencePort.java similarity index 57% rename from src/main/java/fr/insee/genesis/domain/ports/spi/QuestionnaireMetadataPersistancePort.java rename to src/main/java/fr/insee/genesis/domain/ports/spi/QuestionnaireMetadataPersistencePort.java index b33977d2..39588695 100644 --- a/src/main/java/fr/insee/genesis/domain/ports/spi/QuestionnaireMetadataPersistancePort.java +++ b/src/main/java/fr/insee/genesis/domain/ports/spi/QuestionnaireMetadataPersistencePort.java @@ -5,9 +5,9 @@ import java.util.List; -public interface QuestionnaireMetadataPersistancePort { - List find(String questionnaireId, Mode mode); +public interface QuestionnaireMetadataPersistencePort { + List find(String collectionInstrumentId, Mode mode); void save(QuestionnaireMetadataModel questionnaireMetadataModel); - void remove(String questionnaireId, Mode mode); + void remove(String collectionInstrumentId, Mode mode); } diff --git a/src/main/java/fr/insee/genesis/domain/ports/spi/RawResponsePersistencePort.java b/src/main/java/fr/insee/genesis/domain/ports/spi/RawResponsePersistencePort.java new file mode 100644 index 00000000..640bd9b3 --- /dev/null +++ b/src/main/java/fr/insee/genesis/domain/ports/spi/RawResponsePersistencePort.java @@ -0,0 +1,15 @@ +package fr.insee.genesis.domain.ports.spi; + +import fr.insee.genesis.domain.model.surveyunit.Mode; +import fr.insee.genesis.domain.model.surveyunit.rawdata.RawResponse; + +import java.util.List; +import java.util.Set; + +public interface RawResponsePersistencePort { + + List findRawResponses(String collectionInstrumentId, Mode mode, List interrogationIdList); + void updateProcessDates(String collectionInstrumentId, Set interrogationIds); + List getUnprocessedCollectionIds(); + Set findUnprocessedInterrogationIdsByCollectionInstrumentId(String collectionInstrumentId); + } diff --git a/src/main/java/fr/insee/genesis/domain/ports/spi/SurveyUnitPersistencePort.java b/src/main/java/fr/insee/genesis/domain/ports/spi/SurveyUnitPersistencePort.java index ae1a31b2..21986d55 100644 --- a/src/main/java/fr/insee/genesis/domain/ports/spi/SurveyUnitPersistencePort.java +++ b/src/main/java/fr/insee/genesis/domain/ports/spi/SurveyUnitPersistencePort.java @@ -26,7 +26,7 @@ public interface SurveyUnitPersistencePort { Stream findByQuestionnaireId(String questionnaireId); - List findInterrogationIdsByQuestionnaireId(String questionnaireId); + List findInterrogationIdsByCollectionInstrumentId(String collectionInstrumentId); List findInterrogationIdsByQuestionnaireIdAndDateAfter(String questionnaireId, LocalDateTime since); @@ -42,7 +42,7 @@ public interface SurveyUnitPersistencePort { List findInterrogationIdsByCampaignId(String campaignId); - Long deleteByQuestionnaireId(String questionnaireId); + Long deleteByCollectionInstrumentId(String collectionInstrumentId); long count(); diff --git a/src/main/java/fr/insee/genesis/domain/ports/spi/VariableTypePersistancePort.java b/src/main/java/fr/insee/genesis/domain/ports/spi/VariableTypePersistancePort.java deleted file mode 100644 index 3c2e3639..00000000 --- a/src/main/java/fr/insee/genesis/domain/ports/spi/VariableTypePersistancePort.java +++ /dev/null @@ -1,12 +0,0 @@ -package fr.insee.genesis.domain.ports.spi; - -import fr.insee.genesis.domain.model.surveyunit.Mode; -import fr.insee.genesis.domain.model.variabletype.VariableTypeModel; -import fr.insee.genesis.infrastructure.document.variabletype.VariableTypeDocument; -import org.springframework.cache.annotation.Cacheable; - -public interface VariableTypePersistancePort { - void save(VariableTypeModel variableTypeModel); - @Cacheable("variableTypes") - VariableTypeDocument find(String campaignId, String questionnaireId, Mode mode); -} diff --git a/src/main/java/fr/insee/genesis/domain/service/context/DataProcessingContextService.java b/src/main/java/fr/insee/genesis/domain/service/context/DataProcessingContextService.java index a0c80a8c..4f19be63 100644 --- a/src/main/java/fr/insee/genesis/domain/service/context/DataProcessingContextService.java +++ b/src/main/java/fr/insee/genesis/domain/service/context/DataProcessingContextService.java @@ -1,5 +1,8 @@ package fr.insee.genesis.domain.service.context; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; +import fr.insee.genesis.Constants; import fr.insee.genesis.controller.dto.ScheduleDto; import fr.insee.genesis.domain.model.context.DataProcessingContextModel; import fr.insee.genesis.domain.model.context.schedule.KraftwerkExecutionSchedule; @@ -16,9 +19,12 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; import java.time.LocalDateTime; import java.util.ArrayList; -import java.util.Arrays; import java.util.HashSet; import java.util.List; import java.util.Set; @@ -54,6 +60,21 @@ public void saveContext(String partitionId, Boolean withReview) throws GenesisEx dataProcessingContextPersistancePort.save(DataProcessingContextMapper.INSTANCE.modelToDocument(dataProcessingContextModel)); } + @Override + public void saveContextByCollectionInstrumentId(String collectionInstrumentId, Boolean withReview) throws GenesisException { + DataProcessingContextModel dataProcessingContextModel = dataProcessingContextPersistancePort.findByCollectionInstrumentId(collectionInstrumentId); + if(dataProcessingContextModel == null){ + //Create if not exist + dataProcessingContextModel = DataProcessingContextModel.builder() + .collectionInstrumentId(collectionInstrumentId) + .kraftwerkExecutionScheduleList(new ArrayList<>()) + .build(); + } + dataProcessingContextModel.setWithReview(withReview); + + dataProcessingContextPersistancePort.save(DataProcessingContextMapper.INSTANCE.modelToDocument(dataProcessingContextModel)); + } + @Override public void saveKraftwerkExecutionSchedule(String partitionId, ServiceToCall serviceToCall, @@ -83,6 +104,30 @@ public void saveKraftwerkExecutionSchedule(String partitionId, dataProcessingContextPersistancePort.save(DataProcessingContextMapper.INSTANCE.modelToDocument(dataProcessingContextModel)); } + @Override + public void saveKraftwerkExecutionScheduleByCollectionInstrumentId(String collectionInstrumentId, ServiceToCall serviceToCall, String frequency, LocalDateTime startDate, LocalDateTime endDate, TrustParameters trustParameters) throws GenesisException { + DataProcessingContextModel dataProcessingContextModel = dataProcessingContextPersistancePort.findByCollectionInstrumentId(collectionInstrumentId); + if(dataProcessingContextModel == null){ + //Create if not exist + dataProcessingContextModel = DataProcessingContextModel.builder() + .collectionInstrumentId(collectionInstrumentId) + .withReview(false) + .kraftwerkExecutionScheduleList(new ArrayList<>()) + .build(); + } + + dataProcessingContextModel.getKraftwerkExecutionScheduleList().add( + new KraftwerkExecutionSchedule(frequency, + serviceToCall, + startDate, + endDate, + trustParameters + ) + ); + dataProcessingContextPersistancePort.save(DataProcessingContextMapper.INSTANCE.modelToDocument(dataProcessingContextModel)); + } + + @Deprecated(forRemoval = true) @Override public void updateLastExecutionDate(String partitionId, LocalDateTime newDate) throws GenesisException { DataProcessingContextModel dataProcessingContextModel = @@ -96,6 +141,16 @@ public void updateLastExecutionDate(String partitionId, LocalDateTime newDate) t dataProcessingContextPersistancePort.save(DataProcessingContextMapper.INSTANCE.modelToDocument(dataProcessingContextModel)); } + @Override + public void updateLastExecutionDateByCollectionInstrumentId(String collectionInstrumentId, LocalDateTime newDate) throws GenesisException { + DataProcessingContextModel dataProcessingContextModel = dataProcessingContextPersistancePort.findByCollectionInstrumentId(collectionInstrumentId); + if (dataProcessingContextModel == null) { + throw new GenesisException(404, NOT_FOUND_MESSAGE); + } + dataProcessingContextModel.setLastExecution(newDate); + dataProcessingContextPersistancePort.save(DataProcessingContextMapper.INSTANCE.modelToDocument(dataProcessingContextModel)); + } + @Override public void deleteSchedules(String partitionId) throws GenesisException { DataProcessingContextModel dataProcessingContextModel = @@ -109,6 +164,17 @@ public void deleteSchedules(String partitionId) throws GenesisException { dataProcessingContextPersistancePort.save(DataProcessingContextMapper.INSTANCE.modelToDocument(dataProcessingContextModel)); } + @Override + public void deleteSchedulesByCollectionInstrumentId(String collectionInstrumentId) throws GenesisException { + DataProcessingContextModel dataProcessingContextModel = + dataProcessingContextPersistancePort.findByCollectionInstrumentId(collectionInstrumentId); + if (dataProcessingContextModel == null) { + throw new GenesisException(404, NOT_FOUND_MESSAGE); + } + dataProcessingContextModel.setKraftwerkExecutionScheduleList(new ArrayList<>()); + dataProcessingContextPersistancePort.save(DataProcessingContextMapper.INSTANCE.modelToDocument(dataProcessingContextModel)); + } + @Override public List getAllSchedules() { List scheduleDtos = new ArrayList<>(); @@ -124,17 +190,38 @@ public List getAllSchedules() { } @Override - public List deleteExpiredSchedules(String partitionId) throws GenesisException { - DataProcessingContextModel dataProcessingContextModel = - DataProcessingContextMapper.INSTANCE.documentToModel( - dataProcessingContextPersistancePort.findByPartitionId(partitionId) - ); - if (dataProcessingContextModel == null) { - throw new GenesisException(404, NOT_FOUND_MESSAGE); + public void deleteExpiredSchedules(String logFolder) throws GenesisException { + List dataProcessingContextModels = + DataProcessingContextMapper.INSTANCE.listDocumentToListModel(dataProcessingContextPersistancePort.findAll()); + for(DataProcessingContextModel context : dataProcessingContextModels){ + try { + List deletedKraftwerkExecutionSchedules = dataProcessingContextPersistancePort.removeExpiredSchedules(context); + //Save in JSON log + if(!deletedKraftwerkExecutionSchedules.isEmpty()) { + String scheduleName = context.getCollectionInstrumentId()==null ? + context.getPartitionId() : context.getCollectionInstrumentId(); + Path jsonLogPath = Path.of(logFolder, Constants.SCHEDULE_ARCHIVE_FOLDER_NAME, + scheduleName + ".json"); + ObjectMapper objectMapper = new ObjectMapper().findAndRegisterModules(); + objectMapper.registerModule(new JavaTimeModule()); + String jsonToWrite = objectMapper.writeValueAsString(deletedKraftwerkExecutionSchedules); + if(Files.exists(jsonLogPath)){ + //Remove last ] and append survey + StringBuilder content = new StringBuilder(Files.readString(jsonLogPath)); + content.setCharAt(content.length()-1, ','); + content.append(jsonToWrite, 1, jsonToWrite.length()-1); + content.append(']'); + Files.write(jsonLogPath, content.toString().getBytes(), StandardOpenOption.TRUNCATE_EXISTING); + }else { + Files.createDirectories(jsonLogPath.getParent()); + Files.write(jsonLogPath, jsonToWrite.getBytes()); + } + } + } catch (IOException e) { + String name = context.getCollectionInstrumentId()!=null?context.getCollectionInstrumentId() :context.getPartitionId(); + throw new GenesisException(500,String.format("An error occured trying to delete expired schedules for %s",name)); + } } - return new ArrayList<>( - dataProcessingContextPersistancePort.removeExpiredSchedules(dataProcessingContextModel) - ); } @Override @@ -149,28 +236,45 @@ public DataProcessingContextModel getContext(String interrogationId) throws Gene throw new GenesisException(404,"No interrogation in database with id %s".formatted(interrogationId)); } Set partitionIds = new HashSet<>(); - surveyUnitModels.forEach( - surveyUnitModel -> partitionIds.add(surveyUnitModel.getCampaignId()) - ); - if(partitionIds.isEmpty()){ + Set campaignIds = new HashSet<>(); + Set collectionInstrumentIds = new HashSet<>(); + + for (SurveyUnitModel su : surveyUnitModels){ + if (su.getCampaignId()!=null){ + campaignIds.add(su.getCampaignId()); + } + if (su.getCollectionInstrumentId()!=null){ + collectionInstrumentIds.add(su.getCollectionInstrumentId()); + } + } + if(campaignIds.size() > 1 || collectionInstrumentIds.size()>1){ + throw new GenesisException(500,"Multiple partitions for interrogation %s".formatted(interrogationId)); + } + + if(campaignIds.isEmpty() && collectionInstrumentIds.isEmpty()){ return null; } - if(partitionIds.size() > 1){ - throw new GenesisException(500,"Multiple partitions for interrogation %s %n%s".formatted( - interrogationId, - Arrays.toString(partitionIds.toArray()) - )); + + + DataProcessingContextModel contextModel = new DataProcessingContextModel(); + if (!campaignIds.isEmpty()){ + contextModel = DataProcessingContextMapper.INSTANCE.documentToModel( + dataProcessingContextPersistancePort.findByPartitionId(campaignIds.stream().toList().getFirst()) + ); } - return DataProcessingContextMapper.INSTANCE.documentToModel( - dataProcessingContextPersistancePort.findByPartitionId(partitionIds.stream().toList().getFirst()) - ); + if (contextModel.getPartitionId()==null && !collectionInstrumentIds.isEmpty()) { + contextModel = DataProcessingContextMapper.INSTANCE.documentToModel( + dataProcessingContextPersistancePort.findByPartitionId(collectionInstrumentIds.stream().toList().getFirst())); + } + + return contextModel; } @Override - public DataProcessingContextModel getContextByCollectionInstrumentId(String partitionId){ + public DataProcessingContextModel getContextByCollectionInstrumentId(String collectionInstrumentId){ return DataProcessingContextMapper.INSTANCE.documentToModel( - dataProcessingContextPersistancePort.findByPartitionId(partitionId) + dataProcessingContextPersistancePort.findByPartitionId(collectionInstrumentId) ); } @@ -186,6 +290,7 @@ public List getPartitionIds(boolean withReview){ return partitionIds; } + @Deprecated(forRemoval = true) @Override public boolean getReviewByPartitionId(String partitionId) throws GenesisException { DataProcessingContextDocument dataProcessingContextDocument = @@ -197,4 +302,14 @@ public boolean getReviewByPartitionId(String partitionId) throws GenesisExceptio dataProcessingContextPersistancePort.findByPartitionId(partitionId) ).isWithReview(); } + + @Override + public boolean getReviewByCollectionInstrumentId(String collectionInstrumentId) throws GenesisException { + DataProcessingContextModel dataProcessingContextModel = + dataProcessingContextPersistancePort.findByCollectionInstrumentId(collectionInstrumentId); + if(dataProcessingContextModel == null){ + throw new GenesisException(404, "Data processing context not found"); + } + return dataProcessingContextModel.isWithReview(); + } } diff --git a/src/main/java/fr/insee/genesis/domain/service/contextualvariable/ContextualVariableJsonService.java b/src/main/java/fr/insee/genesis/domain/service/contextualvariable/ContextualVariableJsonService.java index 92fe8d42..987a0d5d 100644 --- a/src/main/java/fr/insee/genesis/domain/service/contextualvariable/ContextualVariableJsonService.java +++ b/src/main/java/fr/insee/genesis/domain/service/contextualvariable/ContextualVariableJsonService.java @@ -40,7 +40,7 @@ public ContextualVariableJsonService(ContextualPreviousVariableApiPort contextua } @Override - public ContextualVariableModel getContextualVariable(String questionnaireId, String interrogationId) { + public ContextualVariableModel getContextualVariable(String collectionInstrumentId, String interrogationId) { ContextualVariableModel contextualVariableModel = ContextualVariableModel.builder() .interrogationId(interrogationId) .contextualPrevious(new ArrayList<>()) @@ -48,8 +48,8 @@ public ContextualVariableModel getContextualVariable(String questionnaireId, Str .build(); ContextualPreviousVariableModel contextualPreviousVariableModel = - contextualPreviousVariableApiPort.findByQuestionnaireIdAndInterrogationId( - questionnaireId, + contextualPreviousVariableApiPort.findByCollectionInstrumentIdAndInterrogationId( + collectionInstrumentId, interrogationId ); @@ -60,8 +60,8 @@ public ContextualVariableModel getContextualVariable(String questionnaireId, Str } ContextualExternalVariableModel contextualExternalVariableModel = - contextualExternalVariableApiPort.findByQuestionnaireIdAndInterrogationId( - questionnaireId, + contextualExternalVariableApiPort.findByCollectionInstrumentIdAndInterrogationId( + collectionInstrumentId, interrogationId ); @@ -75,19 +75,21 @@ public ContextualVariableModel getContextualVariable(String questionnaireId, Str } @Override - public int saveContextualVariableFiles(String questionnaireId, FileUtils fileUtils,String contextualFolderPath) throws GenesisException { + public int saveContextualVariableFiles(String collectionInstrumentId, FileUtils fileUtils, String contextualFolderPath) throws GenesisException { int fileCount = 0; for (Mode mode : Mode.values()) { - try (Stream filePaths = Files.list(Path.of(contextualFolderPath))) { + try (Stream filePaths = Files.list(Path.of(fileUtils.getDataFolder(collectionInstrumentId, + mode.getFolder() + , null)))) { Iterator it = filePaths .filter(path -> path.toString().endsWith(".json")) .iterator(); while (it.hasNext()) { Path jsonFilePath = it.next(); - if (processContextualVariableFile(questionnaireId, jsonFilePath)) { + if (processContextualVariableFile(collectionInstrumentId, jsonFilePath)) { //If the file is indeed a contextual variables file and had been processed - moveFile(questionnaireId, mode, fileUtils, jsonFilePath.toString()); + moveFile(collectionInstrumentId, mode, fileUtils, jsonFilePath.toString()); fileCount++; } } @@ -100,9 +102,9 @@ public int saveContextualVariableFiles(String questionnaireId, FileUtils fileUti return fileCount; } - private static void moveFile(String questionnaireId, Mode mode, FileUtils fileUtils, String filePath) throws GenesisException { + private static void moveFile(String collectionInstrumentId, Mode mode, FileUtils fileUtils, String filePath) throws GenesisException { try { - fileUtils.moveFiles(Path.of(filePath), fileUtils.getDoneFolder(questionnaireId, mode.getFolder())); + fileUtils.moveFiles(Path.of(filePath), fileUtils.getDoneFolder(collectionInstrumentId, mode.getFolder())); } catch (IOException e) { throw new GenesisException(500, "Error while moving file to done : %s".formatted(e.toString())); } @@ -145,13 +147,13 @@ private VariableQualityToolDto extractValue(Object variable, String variableName /** * @return true if any contextual variable part found in file, false otherwise */ - private boolean processContextualVariableFile(String questionnaireId, Path jsonFilePath) throws GenesisException { + private boolean processContextualVariableFile(String collectionInstrumentId, Path jsonFilePath) throws GenesisException { return contextualPreviousVariableApiPort.readContextualPreviousFile( - questionnaireId.toUpperCase(), + collectionInstrumentId.toUpperCase(), null, jsonFilePath.toString() ) || contextualExternalVariableApiPort.readContextualExternalFile( - questionnaireId.toUpperCase(), + collectionInstrumentId.toUpperCase(), jsonFilePath.toString() ); } diff --git a/src/main/java/fr/insee/genesis/domain/service/contextualvariable/external/ContextualExternalVariableJsonService.java b/src/main/java/fr/insee/genesis/domain/service/contextualvariable/external/ContextualExternalVariableJsonService.java index cd1f1d3e..f2c835a0 100644 --- a/src/main/java/fr/insee/genesis/domain/service/contextualvariable/external/ContextualExternalVariableJsonService.java +++ b/src/main/java/fr/insee/genesis/domain/service/contextualvariable/external/ContextualExternalVariableJsonService.java @@ -34,10 +34,10 @@ public ContextualExternalVariableJsonService(ContextualExternalVariablePersistan } @Override - public boolean readContextualExternalFile(String questionnaireId, String filePath) throws GenesisException { + public boolean readContextualExternalFile(String collectionInstrumentId, String filePath) throws GenesisException { try(FileInputStream inputStream = new FileInputStream(filePath)){ JsonFactory jsonFactory = new JsonFactory(); - moveCollectionToBackup(questionnaireId); + moveCollectionToBackup(collectionInstrumentId); try(JsonParser jsonParser = jsonFactory.createParser(inputStream)){ List toSave = new ArrayList<>(); goToContextualExternalToken(jsonParser); @@ -51,7 +51,7 @@ public boolean readContextualExternalFile(String questionnaireId, String filePat while (jsonParser.currentToken() != JsonToken.END_ARRAY) { ContextualExternalVariableModel contextualExternalVariableModel = readNextContextualExternal( jsonParser, - questionnaireId + collectionInstrumentId ); checkModel(contextualExternalVariableModel, jsonParser, savedInterrogationIds); @@ -67,21 +67,21 @@ public boolean readContextualExternalFile(String questionnaireId, String filePat contextualExternalVariablePersistancePort.saveAll(toSave); savedCount += toSave.size(); log.info("Reached end of contextual external file, saved %d interrogations".formatted(savedCount)); - contextualExternalVariablePersistancePort.deleteBackup(questionnaireId); + contextualExternalVariablePersistancePort.deleteBackup(collectionInstrumentId); return true; } }catch (JsonParseException jpe){ - contextualExternalVariablePersistancePort.restoreBackup(questionnaireId); + contextualExternalVariablePersistancePort.restoreBackup(collectionInstrumentId); throw new GenesisException(400, "JSON Parsing exception : %s".formatted(jpe.toString())); }catch (IOException ioe){ - contextualExternalVariablePersistancePort.restoreBackup(questionnaireId); + contextualExternalVariablePersistancePort.restoreBackup(collectionInstrumentId); throw new GenesisException(500, ioe.toString()); } } @Override - public ContextualExternalVariableModel findByQuestionnaireIdAndInterrogationId(String questionnaireId, String interrogationId) { - return contextualExternalVariablePersistancePort.findByQuestionnaireIdAndInterrogationId(questionnaireId, interrogationId); + public ContextualExternalVariableModel findByCollectionInstrumentIdAndInterrogationId(String collectionInstrumentId, String interrogationId) { + return contextualExternalVariablePersistancePort.findByCollectionInstrumentIdAndInterrogationId(collectionInstrumentId, interrogationId); } private static void goToContextualExternalToken(JsonParser jsonParser) throws IOException{ @@ -99,9 +99,9 @@ private static void goToContextualExternalToken(JsonParser jsonParser) throws IO } } - private void moveCollectionToBackup(String questionnaireId) { - contextualExternalVariablePersistancePort.backup(questionnaireId); - contextualExternalVariablePersistancePort.delete(questionnaireId); + private void moveCollectionToBackup(String collectionInstrumentId) { + contextualExternalVariablePersistancePort.backup(collectionInstrumentId); + contextualExternalVariablePersistancePort.delete(collectionInstrumentId); } private long saveBlock(List toSave, long savedCount) { @@ -125,13 +125,13 @@ private static void checkModel(ContextualExternalVariableModel contextualExterna } private ContextualExternalVariableModel readNextContextualExternal(JsonParser jsonParser, - String questionnaireId + String collectionInstrumentId ) throws IOException { if(jsonParser.currentToken() != JsonToken.START_OBJECT){ throw new JsonParseException("Expected { on line %d, got token %s".formatted(jsonParser.currentLocation().getLineNr(), jsonParser.currentToken())); } ContextualExternalVariableModel contextualExternalVariableModel = ContextualExternalVariableModel.builder() - .questionnaireId(questionnaireId) + .collectionInstrumentId(collectionInstrumentId) .variables(new HashMap<>()) .build(); jsonParser.nextToken(); diff --git a/src/main/java/fr/insee/genesis/domain/service/contextualvariable/previous/ContextualPreviousVariableJsonService.java b/src/main/java/fr/insee/genesis/domain/service/contextualvariable/previous/ContextualPreviousVariableJsonService.java index 1af94d4c..bc6af354 100644 --- a/src/main/java/fr/insee/genesis/domain/service/contextualvariable/previous/ContextualPreviousVariableJsonService.java +++ b/src/main/java/fr/insee/genesis/domain/service/contextualvariable/previous/ContextualPreviousVariableJsonService.java @@ -34,12 +34,12 @@ public ContextualPreviousVariableJsonService(ContextualPreviousVariablePersistan } @Override - public boolean readContextualPreviousFile(String questionnaireId, + public boolean readContextualPreviousFile(String collectionInstrumentId, String sourceState, String filePath) throws GenesisException { try(FileInputStream inputStream = new FileInputStream(filePath)){ checkSourceStateLength(sourceState); - moveCollectionToBackup(questionnaireId); + moveCollectionToBackup(collectionInstrumentId); JsonFactory jsonFactory = new JsonFactory(); try (JsonParser jsonParser = jsonFactory.createParser(inputStream)) { @@ -55,7 +55,7 @@ public boolean readContextualPreviousFile(String questionnaireId, while (jsonParser.currentToken() != JsonToken.END_ARRAY) { ContextualPreviousVariableModel contextualPreviousVariableModel = readNextContextualPrevious( jsonParser, - questionnaireId, + collectionInstrumentId, sourceState ); @@ -71,22 +71,22 @@ public boolean readContextualPreviousFile(String questionnaireId, } savedCount = saveBlock(toSave, savedCount); log.info("Reached end of contextual previous file, saved %d interrogations".formatted(savedCount)); - contextualPreviousVariablePersistancePort.deleteBackup(questionnaireId); + contextualPreviousVariablePersistancePort.deleteBackup(collectionInstrumentId); return true; } }catch (JsonParseException jpe){ - contextualPreviousVariablePersistancePort.restoreBackup(questionnaireId); + contextualPreviousVariablePersistancePort.restoreBackup(collectionInstrumentId); throw new GenesisException(400, "JSON Parsing exception : %s".formatted(jpe.toString())); }catch (IOException ioe){ - contextualPreviousVariablePersistancePort.restoreBackup(questionnaireId); + contextualPreviousVariablePersistancePort.restoreBackup(collectionInstrumentId); throw new GenesisException(500, ioe.toString()); } } @Override - public ContextualPreviousVariableModel findByQuestionnaireIdAndInterrogationId(String questionnaireId, String interrogationId) { - return contextualPreviousVariablePersistancePort.findByQuestionnaireIdAndInterrogationId( - questionnaireId, + public ContextualPreviousVariableModel findByCollectionInstrumentIdAndInterrogationId(String collectionInstrumentId, String interrogationId) { + return contextualPreviousVariablePersistancePort.findByCollectionInstrumentIdAndInterrogationId( + collectionInstrumentId, interrogationId ); } @@ -98,9 +98,9 @@ private long saveBlock(List toSave, long savedC return savedCount; } - private void moveCollectionToBackup(String questionnaireId) { - contextualPreviousVariablePersistancePort.backup(questionnaireId); - contextualPreviousVariablePersistancePort.delete(questionnaireId); + private void moveCollectionToBackup(String collectionInstrumentId) { + contextualPreviousVariablePersistancePort.backup(collectionInstrumentId); + contextualPreviousVariablePersistancePort.delete(collectionInstrumentId); } private static void checkSourceStateLength(String sourceState) throws GenesisException { @@ -125,14 +125,14 @@ private static void goToEditedPreviousToken(JsonParser jsonParser) throws IOExce } private ContextualPreviousVariableModel readNextContextualPrevious(JsonParser jsonParser, - String questionnaireId, + String collectionInstrumentId, String sourceState ) throws IOException { if(jsonParser.currentToken() != JsonToken.START_OBJECT){ throw new JsonParseException("Expected { on line %d, got token %s".formatted(jsonParser.currentLocation().getLineNr(), jsonParser.currentToken())); } ContextualPreviousVariableModel contextualPreviousVariableModel = ContextualPreviousVariableModel.builder() - .questionnaireId(questionnaireId) + .collectionInstrumentId(collectionInstrumentId) .sourceState(sourceState) .variables(new HashMap<>()) .build(); diff --git a/src/main/java/fr/insee/genesis/domain/service/extraction/LastJsonExtractionService.java b/src/main/java/fr/insee/genesis/domain/service/extraction/LastJsonExtractionService.java index dca95323..7a4d9fb0 100644 --- a/src/main/java/fr/insee/genesis/domain/service/extraction/LastJsonExtractionService.java +++ b/src/main/java/fr/insee/genesis/domain/service/extraction/LastJsonExtractionService.java @@ -25,15 +25,15 @@ public LastJsonExtractionService(LastJsonExtractionPersistencePort extractionPer @Override public void recordDate(LastJsonExtractionModel extraction) { // Create a unique ID based on the questionnaire and the mode. - extraction.setId(String.format("%s_%s",extraction.getQuestionnaireModelId(),extraction.getMode())); + extraction.setId(String.format("%s_%s",extraction.getCollectionInstrumentId(),extraction.getMode())); // save() does an insert if the id doesn't exist, otherwise an update extractionPersistencePort.save(extraction); } @Override - public LastJsonExtractionModel getLastExtractionDate(String questionnaireModelId, Mode mode) throws GenesisException { - return extractionPersistencePort.getLastExecutionDate(questionnaireModelId,mode); + public LastJsonExtractionModel getLastExtractionDate(String collectionInstrumentId, Mode mode) throws GenesisException { + return extractionPersistencePort.getLastExecutionDate(collectionInstrumentId,mode); } diff --git a/src/main/java/fr/insee/genesis/domain/service/lunaticmodel/LunaticModelService.java b/src/main/java/fr/insee/genesis/domain/service/lunaticmodel/LunaticModelService.java index 5325e1dc..f4f46983 100644 --- a/src/main/java/fr/insee/genesis/domain/service/lunaticmodel/LunaticModelService.java +++ b/src/main/java/fr/insee/genesis/domain/service/lunaticmodel/LunaticModelService.java @@ -23,10 +23,10 @@ public LunaticModelService(LunaticModelPersistancePort lunaticModelPersistancePo } @Override - public void save(String questionnaireId, Map lunaticModel) { + public void save(String collectionInstrumentId, Map lunaticModel) { lunaticModelPersistancePort.save( LunaticModelModel.builder() - .questionnaireId(questionnaireId) + .collectionInstrumentId(collectionInstrumentId) .lunaticModel(lunaticModel) .recordDate(LocalDateTime.now()) .build() @@ -34,10 +34,10 @@ public void save(String questionnaireId, Map lunaticModel) { } @Override - public LunaticModelModel get(String questionnaireId) throws GenesisException { - if(lunaticModelPersistancePort.find(questionnaireId).isEmpty()){ + public LunaticModelModel get(String collectionInstrumentId) throws GenesisException { + if(lunaticModelPersistancePort.find(collectionInstrumentId).isEmpty()){ throw new GenesisException(404,"Questionnaire not found"); } - return LunaticModelMapper.INSTANCE.documentToModel(lunaticModelPersistancePort.find(questionnaireId).getFirst()); + return LunaticModelMapper.INSTANCE.documentToModel(lunaticModelPersistancePort.find(collectionInstrumentId).getFirst()); } } diff --git a/src/main/java/fr/insee/genesis/domain/service/metadata/QuestionnaireMetadataService.java b/src/main/java/fr/insee/genesis/domain/service/metadata/QuestionnaireMetadataService.java index 5d780199..0b1ee957 100644 --- a/src/main/java/fr/insee/genesis/domain/service/metadata/QuestionnaireMetadataService.java +++ b/src/main/java/fr/insee/genesis/domain/service/metadata/QuestionnaireMetadataService.java @@ -10,7 +10,7 @@ import fr.insee.genesis.domain.model.metadata.QuestionnaireMetadataModel; import fr.insee.genesis.domain.model.surveyunit.Mode; import fr.insee.genesis.domain.ports.api.QuestionnaireMetadataApiPort; -import fr.insee.genesis.domain.ports.spi.QuestionnaireMetadataPersistancePort; +import fr.insee.genesis.domain.ports.spi.QuestionnaireMetadataPersistencePort; import fr.insee.genesis.exceptions.GenesisError; import fr.insee.genesis.exceptions.GenesisException; import fr.insee.genesis.infrastructure.utils.FileUtils; @@ -31,33 +31,28 @@ public class QuestionnaireMetadataService implements QuestionnaireMetadataApiPor private static final String DDI_FILE_PATTERN = "ddi[\\w,\\s-]+\\.xml"; private static final String LUNATIC_FILE_PATTERN = "lunatic[\\w,\\s-]+\\.json"; - QuestionnaireMetadataPersistancePort questionnaireMetadataPersistancePort; + QuestionnaireMetadataPersistencePort questionnaireMetadataPersistencePort; @Override - public MetadataModel find(String questionnaireId, Mode mode) throws GenesisException { + public MetadataModel find(String collectionInstrumentId, Mode mode) throws GenesisException { List questionnaireMetadataModels = - questionnaireMetadataPersistancePort.find(questionnaireId, mode); + questionnaireMetadataPersistencePort.find(collectionInstrumentId, mode); if(questionnaireMetadataModels.isEmpty()){ - throw new GenesisException(404, "Questionnaire metadata not found"); + throw new GenesisException(404, "Collection instrument metadata not found"); } return questionnaireMetadataModels.getFirst().metadataModel(); } - public MetadataModel loadAndSaveIfNotExists( - String campaignName, - String questionnaireId, - Mode mode, - FileUtils fileUtils, - List errors - ) throws GenesisException { - - List storedMetadatas = - questionnaireMetadataPersistancePort.find(questionnaireId.toUpperCase(), mode); - - if (!storedMetadatas.isEmpty() - && storedMetadatas.getFirst().metadataModel() != null) { - return storedMetadatas.getFirst().metadataModel(); + @Override + public MetadataModel loadAndSaveIfNotExists(String campaignName, String collectionInstrumentId, Mode mode, FileUtils fileUtils, + List errors) throws GenesisException { + List questionnaireMetadataModels = + questionnaireMetadataPersistencePort.find(collectionInstrumentId.toUpperCase(), mode); + if(questionnaireMetadataModels.isEmpty() || questionnaireMetadataModels.getFirst().metadataModel() == null){ + MetadataModel metadataModel = readMetadatas(campaignName, mode.getModeName(), fileUtils, errors); + saveMetadata(collectionInstrumentId.toUpperCase(), mode, metadataModel); + return metadataModel; } MetadataModel metadataModel = @@ -68,10 +63,10 @@ public MetadataModel loadAndSaveIfNotExists( return metadataModel; } - private void saveMetadata(String questionnaireId, Mode mode, MetadataModel metadataModel) { - questionnaireMetadataPersistancePort.save( + private void saveMetadata(String collectionInstrumentId, Mode mode, MetadataModel metadataModel) { + questionnaireMetadataPersistencePort.save( new QuestionnaireMetadataModel( - questionnaireId, + collectionInstrumentId, mode, metadataModel ) @@ -154,14 +149,14 @@ private MetadataModel parseMetadata(String metadataFilePath, boolean withDDI) { } @Override - public void remove(String questionnaireId, Mode mode) { - questionnaireMetadataPersistancePort.remove(questionnaireId, mode); + public void remove(String collectionInstrumentId, Mode mode) { + questionnaireMetadataPersistencePort.remove(collectionInstrumentId, mode); } @Override - public void save(String questionnaireId, Mode mode, MetadataModel metadataModel) { - questionnaireMetadataPersistancePort.save(new QuestionnaireMetadataModel( - questionnaireId, + public void save(String collectionInstrumentId, Mode mode, MetadataModel metadataModel) { + questionnaireMetadataPersistencePort.save(new QuestionnaireMetadataModel( + collectionInstrumentId, mode, metadataModel )); diff --git a/src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java b/src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java index 2134f67a..efb17ff4 100644 --- a/src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java +++ b/src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java @@ -251,7 +251,7 @@ private Map> getProcessedIdsMap(List survey Map> processedInterrogationIdsPerQuestionnaire = new HashMap<>(); surveyUnitModels.forEach(model -> processedInterrogationIdsPerQuestionnaire - .computeIfAbsent(model.getQuestionnaireId(), k -> new HashSet<>()) + .computeIfAbsent(model.getCollectionInstrumentId(), k -> new HashSet<>()) .add(model.getInterrogationId()) ); return processedInterrogationIdsPerQuestionnaire; @@ -267,17 +267,15 @@ public List convertRawData(List rawDat RawDataModelType rawDataModelType = getRawDataModelType(rawData); //Get optional fields - String contextualId = getContextualId(rawData); Boolean isCapturedIndirectly = getIsCapturedIndirectly(rawData); LocalDateTime validationDate = getValidationDate(rawData); SurveyUnitModel surveyUnitModel = SurveyUnitModel.builder() .campaignId(rawData.campaignId()) - .questionnaireId(rawData.questionnaireId()) + .collectionInstrumentId(rawData.questionnaireId()) .mode(rawData.mode()) .interrogationId(rawData.interrogationId()) - .idUE(rawData.idUE()) - .contextualId(contextualId) + .usualSurveyUnitId(rawData.idUE()) .validationDate(validationDate) .isCapturedIndirectly(isCapturedIndirectly) .state(dataState) @@ -338,15 +336,6 @@ private static Boolean getIsCapturedIndirectly(LunaticJsonRawDataModel rawData) } - private static String getContextualId(LunaticJsonRawDataModel rawData) { - try{ - return rawData.data().get("contextualId") == null ? null : rawData.data().get("contextualId").toString(); - }catch(Exception e){ - log.warn("Exception when parsing contextual id : {}}",e.toString()); - return null; - } - } - @Override public List getUnprocessedDataIds() { List dtos = new ArrayList<>(); diff --git a/src/main/java/fr/insee/genesis/domain/service/rawdata/RawResponseService.java b/src/main/java/fr/insee/genesis/domain/service/rawdata/RawResponseService.java new file mode 100644 index 00000000..6f343237 --- /dev/null +++ b/src/main/java/fr/insee/genesis/domain/service/rawdata/RawResponseService.java @@ -0,0 +1,447 @@ +package fr.insee.genesis.domain.service.rawdata; + +import fr.insee.bpm.metadata.model.VariablesMap; +import fr.insee.genesis.Constants; +import fr.insee.genesis.configuration.Config; +import fr.insee.genesis.controller.utils.ControllerUtils; +import fr.insee.genesis.domain.model.context.DataProcessingContextModel; +import fr.insee.genesis.domain.model.surveyunit.DataState; +import fr.insee.genesis.domain.model.surveyunit.Mode; +import fr.insee.genesis.domain.model.surveyunit.SurveyUnitModel; +import fr.insee.genesis.domain.model.surveyunit.VariableModel; +import fr.insee.genesis.domain.model.surveyunit.rawdata.DataProcessResult; +import fr.insee.genesis.domain.model.surveyunit.rawdata.RawResponse; +import fr.insee.genesis.domain.ports.api.RawResponseApiPort; +import fr.insee.genesis.domain.ports.spi.RawResponsePersistencePort; +import fr.insee.genesis.domain.ports.spi.SurveyUnitQualityToolPort; +import fr.insee.genesis.domain.service.context.DataProcessingContextService; +import fr.insee.genesis.domain.service.metadata.QuestionnaireMetadataService; +import fr.insee.genesis.domain.service.surveyunit.SurveyUnitQualityService; +import fr.insee.genesis.domain.service.surveyunit.SurveyUnitService; +import fr.insee.genesis.domain.utils.GroupUtils; +import fr.insee.genesis.domain.utils.JsonUtils; +import fr.insee.genesis.exceptions.GenesisError; +import fr.insee.genesis.exceptions.GenesisException; +import fr.insee.genesis.infrastructure.utils.FileUtils; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.http.ResponseEntity; +import org.springframework.stereotype.Service; + +import java.io.IOException; +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +@Service +@Slf4j +public class RawResponseService implements RawResponseApiPort { + + private final ControllerUtils controllerUtils; + private final QuestionnaireMetadataService metadataService; + private final SurveyUnitService surveyUnitService; + private final SurveyUnitQualityService surveyUnitQualityService; + private final SurveyUnitQualityToolPort surveyUnitQualityToolPort; + private final DataProcessingContextService dataProcessingContextService; + private final FileUtils fileUtils; + private final Config config; + + @Qualifier("rawResponseMongoAdapter") + private final RawResponsePersistencePort rawResponsePersistencePort; + + public RawResponseService(ControllerUtils controllerUtils, QuestionnaireMetadataService metadataService, SurveyUnitService surveyUnitService, SurveyUnitQualityService surveyUnitQualityService, SurveyUnitQualityToolPort surveyUnitQualityToolPort, DataProcessingContextService dataProcessingContextService, FileUtils fileUtils, Config config, RawResponsePersistencePort rawResponsePersistencePort) { + this.controllerUtils = controllerUtils; + this.metadataService = metadataService; + this.surveyUnitService = surveyUnitService; + this.surveyUnitQualityService = surveyUnitQualityService; + this.surveyUnitQualityToolPort = surveyUnitQualityToolPort; + this.dataProcessingContextService = dataProcessingContextService; + this.fileUtils = fileUtils; + this.config = config; + this.rawResponsePersistencePort = rawResponsePersistencePort; + } + + @Override + public List getRawResponses(String collectionInstrumentId, Mode mode, List interrogationIdList) { + return rawResponsePersistencePort.findRawResponses(collectionInstrumentId,mode,interrogationIdList); + } + + @Override + public DataProcessResult processRawResponses(String collectionInstrumentId, List interrogationIdList, List errors) throws GenesisException { + int dataCount=0; + int formattedDataCount=0; + DataProcessingContextModel dataProcessingContext = + dataProcessingContextService.getContextByCollectionInstrumentId(collectionInstrumentId); + List modesList = controllerUtils.getModesList(collectionInstrumentId, null); + for (Mode mode : modesList) { + //Load and save metadata into database, throw exception if none + VariablesMap variablesMap = getVariablesMap(collectionInstrumentId,mode,errors); + int totalBatchs = Math.ceilDiv(interrogationIdList.size() , config.getRawDataProcessingBatchSize()); + int batchNumber = 1; + List interrogationIdListForMode = new ArrayList<>(interrogationIdList); + while(!interrogationIdListForMode.isEmpty()){ + log.info("Processing raw data batch {}/{}", batchNumber, totalBatchs); + int maxIndex = Math.min(interrogationIdListForMode.size(), config.getRawDataProcessingBatchSize()); + List interrogationIdToProcess = interrogationIdListForMode.subList(0, maxIndex); + + List rawResponses = getRawResponses(collectionInstrumentId, mode, interrogationIdToProcess); + + List surveyUnitModels = convertRawResponse( + rawResponses, + variablesMap + ); + + //Save converted data + surveyUnitQualityService.verifySurveyUnits(surveyUnitModels, variablesMap); + surveyUnitService.saveSurveyUnits(surveyUnitModels); + + //Update process dates + updateProcessDates(surveyUnitModels); + + //Increment data count + dataCount += surveyUnitModels.size(); + formattedDataCount += surveyUnitModels.stream() + .filter(surveyUnitModel -> surveyUnitModel.getState().equals(DataState.FORMATTED)) + .toList() + .size(); + + //Send processed ids grouped by questionnaire (if review activated) + if(dataProcessingContext != null && dataProcessingContext.isWithReview()) { + sendProcessedIdsToQualityTool(surveyUnitModels); + } + + //Remove processed ids from list + interrogationIdListForMode = interrogationIdListForMode.subList(maxIndex, interrogationIdListForMode.size()); + + batchNumber++; + } + } + return new DataProcessResult(dataCount, formattedDataCount, errors); + } + + @Override + public DataProcessResult processRawResponses(String collectionInstrumentId) throws GenesisException { + int dataCount=0; + int formattedDataCount=0; + DataProcessingContextModel dataProcessingContext = + dataProcessingContextService.getContextByCollectionInstrumentId(collectionInstrumentId); + List errors = new ArrayList<>(); + + List modesList = controllerUtils.getModesList(collectionInstrumentId, null); + for (Mode mode : modesList) { + //Load and save metadata into database, throw exception if none + VariablesMap variablesMap = getVariablesMap(collectionInstrumentId,mode,errors); + Set interrogationIds = + rawResponsePersistencePort.findUnprocessedInterrogationIdsByCollectionInstrumentId(collectionInstrumentId); + + int totalBatchs = Math.ceilDiv(interrogationIds.size() , config.getRawDataProcessingBatchSize()); + int batchNumber = 1; + List interrogationIdListForMode = new ArrayList<>(interrogationIds); + while(!interrogationIdListForMode.isEmpty()){ + log.info("Processing raw data batch {}/{}", batchNumber, totalBatchs); + int maxIndex = Math.min(interrogationIdListForMode.size(), config.getRawDataProcessingBatchSize()); + + List surveyUnitModels = getConvertedSurveyUnits( + collectionInstrumentId, + mode, + interrogationIdListForMode, + maxIndex, + variablesMap); + + //Save converted data + surveyUnitQualityService.verifySurveyUnits(surveyUnitModels, variablesMap); + surveyUnitService.saveSurveyUnits(surveyUnitModels); + + //Update process dates + updateProcessDates(surveyUnitModels); + + //Increment data count + dataCount += surveyUnitModels.size(); + formattedDataCount += surveyUnitModels.stream() + .filter(surveyUnitModel -> surveyUnitModel.getState().equals(DataState.FORMATTED)) + .toList() + .size(); + + //Send processed ids grouped by questionnaire (if review activated) + if(dataProcessingContext != null && dataProcessingContext.isWithReview()) { + sendProcessedIdsToQualityTool(surveyUnitModels); + } + + //Remove processed ids from list + interrogationIdListForMode = interrogationIdListForMode.subList(maxIndex, interrogationIdListForMode.size()); + batchNumber++; + } + } + return new DataProcessResult(dataCount, formattedDataCount, errors); + } + + private List getConvertedSurveyUnits(String collectionInstrumentId, Mode mode, List interrogationIdListForMode, int maxIndex, VariablesMap variablesMap) { + List interrogationIdToProcess = interrogationIdListForMode.subList(0, maxIndex); + List rawResponses = getRawResponses(collectionInstrumentId, mode, interrogationIdToProcess); + return convertRawResponse( + rawResponses, + variablesMap + ); + } + + private VariablesMap getVariablesMap(String collectionInstrumentId, Mode mode, List errors) throws GenesisException { + VariablesMap variablesMap = metadataService.loadAndSaveIfNotExists(collectionInstrumentId, collectionInstrumentId, mode, fileUtils, + errors).getVariables(); + if (variablesMap == null) { + throw new GenesisException(400, + "Error during metadata parsing for mode %s :%n%s" + .formatted(mode, errors.getLast().getMessage()) + ); + } + return variablesMap; + } + + @Override + public List convertRawResponse(List rawResponses, VariablesMap variablesMap) { + //Convert to genesis model + List surveyUnitModels = new ArrayList<>(); + //For each possible data state (we receive COLLECTED or EDITED) + for(DataState dataState : List.of(DataState.COLLECTED,DataState.EDITED)){ + for (RawResponse rawResponse : rawResponses) { + //Get optional fields + Boolean isCapturedIndirectly = getIsCapturedIndirectly(rawResponse); + LocalDateTime validationDate = getValidationDate(rawResponse); + String usualSurveyUnitId = getStringFieldInPayload(rawResponse,"usualSurveyUnitId"); + String majorModelVersion = getStringFieldInPayload(rawResponse, "majorModelVersion"); + + SurveyUnitModel surveyUnitModel = SurveyUnitModel.builder() + .collectionInstrumentId(rawResponse.collectionInstrumentId()) + .majorModelVersion(majorModelVersion) + .mode(rawResponse.mode()) + .interrogationId(rawResponse.interrogationId()) + .usualSurveyUnitId(usualSurveyUnitId) + .validationDate(validationDate) + .isCapturedIndirectly(isCapturedIndirectly) + .state(dataState) + .fileDate(rawResponse.recordDate()) + .recordDate(LocalDateTime.now()) + .collectedVariables(new ArrayList<>()) + .externalVariables(new ArrayList<>()) + .build(); + + //Data collected variables conversion + convertRawDataCollectedVariables(rawResponse, surveyUnitModel, dataState, variablesMap); + + //External variables conversion into COLLECTED document + if(dataState == DataState.COLLECTED){ + convertRawDataExternalVariables(rawResponse, surveyUnitModel, variablesMap); + } + + boolean hasNoVariable = surveyUnitModel.getCollectedVariables().isEmpty() + && surveyUnitModel.getExternalVariables().isEmpty(); + + if(hasNoVariable){ + if(surveyUnitModel.getState() == DataState.COLLECTED){ + log.warn("No collected or external variable for interrogation {}, raw data is ignored.", rawResponse.interrogationId()); + } + continue;// don't add suModel + } + surveyUnitModels.add(surveyUnitModel); + } + } + return surveyUnitModels; + } + + @Override + public List getUnprocessedCollectionInstrumentIds() { + return rawResponsePersistencePort.getUnprocessedCollectionIds(); + } + + @Override + public void updateProcessDates(List surveyUnitModels) { + Set collectionInstrumentIds = new HashSet<>(); + for (SurveyUnitModel surveyUnitModel : surveyUnitModels) { + collectionInstrumentIds.add(surveyUnitModel.getCollectionInstrumentId()); + } + + for (String collectionInstrumentId : collectionInstrumentIds) { + Set interrogationIds = surveyUnitModels.stream() + .filter(su -> su.getCollectionInstrumentId().equals(collectionInstrumentId)) + .map(SurveyUnitModel::getInterrogationId) + .collect(Collectors.toSet()); + rawResponsePersistencePort.updateProcessDates(collectionInstrumentId, interrogationIds); + } + } + + private Map> getProcessedIdsMap(List surveyUnitModels) { + Map> processedInterrogationIdsPerQuestionnaire = new HashMap<>(); + surveyUnitModels.forEach(model -> + processedInterrogationIdsPerQuestionnaire + .computeIfAbsent(model.getCollectionInstrumentId(), k -> new HashSet<>()) + .add(model.getInterrogationId()) + ); + return processedInterrogationIdsPerQuestionnaire; + } + + private void sendProcessedIdsToQualityTool(List surveyUnitModels) { + try { + Map> processedIdsMap = getProcessedIdsMap(surveyUnitModels); + ResponseEntity response = + surveyUnitQualityToolPort.sendProcessedIds(processedIdsMap); + + if (response.getStatusCode().is2xxSuccessful()) { + log.info("Successfully sent {} ids to quality tool", processedIdsMap.size()); + }else{ + log.warn("Survey unit quality tool responded non-2xx code {} and body {}", + response.getStatusCode(), response.getBody()); + } + }catch (IOException e){ + log.error("Error during Perret call request building : {}", e.toString()); + } + } + + private static Boolean getIsCapturedIndirectly(RawResponse rawResponse) { + try{ + return rawResponse.payload().get("isCapturedIndirectly") == null ? null : + Boolean.parseBoolean(rawResponse.payload().get("isCapturedIndirectly").toString()); + }catch(Exception e){ + log.warn("Exception when parsing isCapturedIndirectly : {}",e.toString()); + return Boolean.FALSE; + } + } + + private static LocalDateTime getValidationDate(RawResponse rawResponse) { + try{ + return rawResponse.payload().get("validationDate") == null ? null : + LocalDateTime.parse(rawResponse.payload().get("validationDate").toString()); + }catch(Exception e){ + log.warn("Exception when parsing validation date : {}",e.toString()); + return null; + } + } + + private static String getStringFieldInPayload(RawResponse rawResponse, String field) { + try{ + return rawResponse.payload().get(field).toString(); + }catch(Exception e){ + log.warn("Exception when parsing {} : {}",field, e.toString()); + return null; + } + } + + @SuppressWarnings("unchecked") + private void convertRawDataCollectedVariables( + RawResponse rawResponse, + SurveyUnitModel dstSurveyUnitModel, + DataState dataState, + VariablesMap variablesMap + ) { + Map dataMap = rawResponse.payload(); + dataMap = (Map) dataMap.get("data"); + + dataMap = (Map)dataMap.get("COLLECTED"); + + + Map collectedMap = JsonUtils.asMap(dataMap); + if (collectedMap == null || collectedMap.isEmpty()){ + if(dataState.equals(DataState.COLLECTED)) { + log.warn("No collected data for interrogation {}", rawResponse.interrogationId()); + } + return; + } + convertToCollectedVar(dstSurveyUnitModel, dataState, variablesMap, collectedMap); + } + + private static void convertToCollectedVar( + SurveyUnitModel dstSurveyUnitModel, + DataState dataState, + VariablesMap variablesMap, + Map collectedMap + ) { + final String stateKey = dataState.toString(); + final var dest = dstSurveyUnitModel.getCollectedVariables(); + + for (Map.Entry collectedVariable : collectedMap.entrySet()) { + // Map for this variable (COLLECTED/EDITED -> value) + Map states = JsonUtils.asMap(collectedVariable.getValue()); + + if (states != null && states.containsKey(stateKey)) { + Object value = states.get(stateKey); + + // liste ? + if (value instanceof List) { + // on garde exactement ta signature existante + convertListVar(value, collectedVariable, variablesMap, dest); + } + + // scalaire non null ? + if (value != null && !(value instanceof List)) { + // idem: on garde convertOneVar(entry, String, ...) + convertOneVar(collectedVariable, String.valueOf(value), variablesMap, 1, dest); + } + } + } + } + + private static void convertListVar(Object valuesForState, Map.Entry collectedVariable, VariablesMap variablesMap, List dstSurveyUnitModel) { + List values = JsonUtils.asStringList(valuesForState); + if (!values.isEmpty()) { + int iteration = 1; + for (String value : values) { + convertOneVar(collectedVariable, value, variablesMap, iteration, dstSurveyUnitModel); + iteration++; + } + } + } + + private static void convertOneVar(Map.Entry externalVariableEntry, String valueObject, VariablesMap variablesMap, int iteration, List dstSurveyUnitModel) { + VariableModel externalVariableModel = VariableModel.builder() + .varId(externalVariableEntry.getKey()) + .value(valueObject) + .scope(getIdLoop(variablesMap, externalVariableEntry.getKey())) + .iteration(iteration) + .parentId(GroupUtils.getParentGroupName(externalVariableEntry.getKey(), variablesMap)) + .build(); + dstSurveyUnitModel.add(externalVariableModel); + } + + private static String getIdLoop(VariablesMap variablesMap, String variableName) { + if (variablesMap.getVariable(variableName) == null) { + log.warn("Variable {} not present in metadata, assigning to {}", variableName, Constants.ROOT_GROUP_NAME); + return Constants.ROOT_GROUP_NAME; + } + return variablesMap.getVariable(variableName).getGroupName(); + } + + @SuppressWarnings("unchecked") + private static void convertRawDataExternalVariables( + RawResponse rawResponse, + SurveyUnitModel dstSurveyUnitModel, + VariablesMap variablesMap + ) { + Map dataMap = rawResponse.payload(); + dataMap = (Map) dataMap.get("data"); + + + dataMap = (Map)dataMap.get("EXTERNAL"); + Map externalMap = JsonUtils.asMap(dataMap); + if (externalMap != null && !externalMap.isEmpty()){ + convertToExternalVar(dstSurveyUnitModel, variablesMap, externalMap); + } + } + + private static void convertToExternalVar(SurveyUnitModel dstSurveyUnitModel, VariablesMap variablesMap, Map externalMap) { + for(Map.Entry externalVariableEntry : externalMap.entrySet()){ + Object valueObject = externalVariableEntry.getValue(); + if (valueObject instanceof List){ + //Array of values + convertListVar(valueObject, externalVariableEntry, variablesMap, dstSurveyUnitModel.getExternalVariables()); + continue; + } + //Value + if (valueObject != null) { + convertOneVar(externalVariableEntry, valueObject.toString(), variablesMap, 1, dstSurveyUnitModel.getExternalVariables()); + } + } + } +} diff --git a/src/main/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitService.java b/src/main/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitService.java index 91b35e0c..89a082a5 100644 --- a/src/main/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitService.java +++ b/src/main/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitService.java @@ -34,7 +34,6 @@ import java.util.Map; import java.util.Set; import java.util.stream.Collectors; -import java.util.stream.Stream; @Service @Slf4j @@ -60,8 +59,8 @@ public void saveSurveyUnits(List surveyUnitModels) { } @Override - public List findByIdsInterrogationAndQuestionnaire(String interrogationId, String questionnaireId) { - return surveyUnitPersistencePort.findByIds(interrogationId, questionnaireId); + public List findByIdsInterrogationAndCollectionInstrument(String interrogationId, String collectionInstrumentId) { + return surveyUnitPersistencePort.findByIds(interrogationId, collectionInstrumentId); } @Override @@ -69,23 +68,18 @@ public List findByInterrogationId(String interrogationId) { return surveyUnitPersistencePort.findByInterrogationId(interrogationId); } - @Override - public Stream findByQuestionnaireId(String questionnaireId) { - return surveyUnitPersistencePort.findByQuestionnaireId(questionnaireId); - } - /** * In this method we want to get the latest update for each variable of a survey unit * But we need to separate the updates by mode * So we will calculate the latest state for a given collection mode - * @param interrogationId : Survey unit id - * @param questionnaireId : Questionnaire id + * @param interrogationId : Interrogation id + * @param collectionInstrumentId : Collection instrument id * @return the latest update for each variable of a survey unit */ @Override - public List findLatestByIdAndByQuestionnaireId(String interrogationId, String questionnaireId) { + public List findLatestByIdAndByCollectionInstrumentId(String interrogationId, String collectionInstrumentId) { List latestUpdatesbyVariables = new ArrayList<>(); - List surveyUnitModels = surveyUnitPersistencePort.findByIds(interrogationId, questionnaireId); + List surveyUnitModels = surveyUnitPersistencePort.findByIds(interrogationId, collectionInstrumentId); List modes = getDistinctsModes(surveyUnitModels); modes.forEach(mode ->{ List suByMode = surveyUnitModels.stream() @@ -229,8 +223,10 @@ private List extractLatestUpdates(List allResp @Override - public SurveyUnitDto findLatestValuesByStateByIdAndByQuestionnaireId(String interrogationId, - String questionnaireId) throws GenesisException { + public SurveyUnitDto findLatestValuesByStateByIdAndByCollectionInstrumentId( + String interrogationId, + String collectionInstrumentId) throws GenesisException + { SurveyUnitDto surveyUnitDto = SurveyUnitDto.builder() .interrogationId(interrogationId) .collectedVariables(new ArrayList<>()) @@ -240,7 +236,7 @@ public SurveyUnitDto findLatestValuesByStateByIdAndByQuestionnaireId(String inte //Extract variables Map collectedVariableMap = new HashMap<>(); Map externalVariableMap = new HashMap<>(); - List surveyUnitModels = surveyUnitPersistencePort.findByIds(interrogationId, questionnaireId); + List surveyUnitModels = surveyUnitPersistencePort.findByIds(interrogationId, collectionInstrumentId); List modes = getDistinctsModes(surveyUnitModels); for (Mode mode : modes) { List suByMode = surveyUnitModels.stream() @@ -253,7 +249,7 @@ public SurveyUnitDto findLatestValuesByStateByIdAndByQuestionnaireId(String inte if (variablesMap == null) { variablesMap = metadataService.loadAndSaveIfNotExists( surveyUnitModel.getCampaignId(), - surveyUnitModel.getQuestionnaireId(), + surveyUnitModel.getCollectionInstrumentId(), surveyUnitModel.getMode(), fileUtils, new ArrayList<>()).getVariables(); @@ -269,7 +265,7 @@ public SurveyUnitDto findLatestValuesByStateByIdAndByQuestionnaireId(String inte @Override public List findDistinctInterrogationIdsByQuestionnaireId(String questionnaireId) { - List surveyUnitModels = surveyUnitPersistencePort.findInterrogationIdsByQuestionnaireId(questionnaireId); + List surveyUnitModels = surveyUnitPersistencePort.findInterrogationIdsByCollectionInstrumentId(questionnaireId); List suIds = new ArrayList<>(); surveyUnitModels.forEach(surveyUnitModel -> suIds.add(new InterrogationId(surveyUnitModel.getInterrogationId()))); return suIds.stream().distinct().toList(); @@ -329,13 +325,13 @@ public long countInterrogationIdsByQuestionnaireId(String questionnaireId) { @Override public List findInterrogationIdsAndModesByQuestionnaireId(String questionnaireId) { - List surveyUnitModels = surveyUnitPersistencePort.findInterrogationIdsByQuestionnaireId(questionnaireId); + List surveyUnitModels = surveyUnitPersistencePort.findInterrogationIdsByCollectionInstrumentId(questionnaireId); return surveyUnitModels.stream().distinct().toList(); } @Override - public List findModesByQuestionnaireId(String questionnaireId) { - List surveyUnitModels = surveyUnitPersistencePort.findInterrogationIdsByQuestionnaireId(questionnaireId); + public List findModesByCollectionInstrumentId(String collectionInstrumentId) { + List surveyUnitModels = surveyUnitPersistencePort.findInterrogationIdsByCollectionInstrumentId(collectionInstrumentId); List sources = new ArrayList<>(); surveyUnitModels.forEach(surveyUnitModel -> sources.add(surveyUnitModel.getMode())); return sources.stream().distinct().toList(); @@ -368,8 +364,8 @@ public List findModesByCampaignIdV2(String campaignId) { //========= OPTIMISATIONS PERFS (END) ========== @Override - public Long deleteByQuestionnaireId(String questionnaireId) { - return surveyUnitPersistencePort.deleteByQuestionnaireId(questionnaireId); + public Long deleteByCollectionInstrumentId(String collectionInstrumentId) { + return surveyUnitPersistencePort.deleteByCollectionInstrumentId(collectionInstrumentId); } @Override @@ -453,7 +449,7 @@ public List parseEditedVariables( SurveyUnitModel surveyUnitModel = SurveyUnitModel.builder() .campaignId(surveyUnitInputDto.getCampaignId()) .mode(surveyUnitInputDto.getMode()) - .questionnaireId(surveyUnitInputDto.getQuestionnaireId().toUpperCase()) + .collectionInstrumentId(surveyUnitInputDto.getQuestionnaireId().toUpperCase()) .interrogationId(surveyUnitInputDto.getInterrogationId()) .state(state) .recordDate(LocalDateTime.now()) @@ -496,7 +492,7 @@ public String findQuestionnaireIdByInterrogationId(String interrogationId) throw } Set questionnaireIds = new HashSet<>(); for(SurveyUnitModel surveyUnitModel : surveyUnitModels){ - questionnaireIds.add(surveyUnitModel.getQuestionnaireId()); + questionnaireIds.add(surveyUnitModel.getCollectionInstrumentId()); } if(questionnaireIds.size() > 1){ throw new GenesisException(207,String.format("Multiple questionnaires for %s :%n%s", @@ -510,7 +506,7 @@ public String findQuestionnaireIdByInterrogationId(String interrogationId) throw @Override public Set findCampaignIdsFrom(SurveyUnitInputDto dto) { - List responses = findByIdsInterrogationAndQuestionnaire( + List responses = findByIdsInterrogationAndCollectionInstrument( dto.getInterrogationId(), dto.getQuestionnaireId() ); diff --git a/src/main/java/fr/insee/genesis/domain/service/variabletype/VariableTypeService.java b/src/main/java/fr/insee/genesis/domain/service/variabletype/VariableTypeService.java deleted file mode 100644 index 52ed64f9..00000000 --- a/src/main/java/fr/insee/genesis/domain/service/variabletype/VariableTypeService.java +++ /dev/null @@ -1,39 +0,0 @@ -package fr.insee.genesis.domain.service.variabletype; - -import fr.insee.bpm.metadata.model.Variable; -import fr.insee.bpm.metadata.model.VariablesMap; -import fr.insee.genesis.domain.model.surveyunit.Mode; -import fr.insee.genesis.domain.model.variabletype.VariableTypeModel; -import fr.insee.genesis.domain.ports.api.VariableTypeApiPort; -import fr.insee.genesis.domain.ports.spi.VariableTypePersistancePort; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.stereotype.Service; - -@Service -public class VariableTypeService implements VariableTypeApiPort { - @Qualifier("variableTypeMongoAdapter") - private final VariableTypePersistancePort variableTypePersistancePort; - - @Autowired - public VariableTypeService(VariableTypePersistancePort variableTypePersistancePort) { - this.variableTypePersistancePort = variableTypePersistancePort; - } - - @Override - public void saveMetadatas(String campaignId, String questionnaireId, Mode mode, VariablesMap variablesMap) { - VariableTypeModel variableTypeModel = VariableTypeModel.builder() - .campaignId(campaignId) - .questionnaireId(questionnaireId) - .mode(mode) - .variablesMap(new VariablesMap()) - .build(); - - for(String variableName : variablesMap.getVariables().keySet()){ - Variable bpmVariable = variablesMap.getVariable(variableName); - variableTypeModel.variablesMap().putVariable(bpmVariable); - } - - variableTypePersistancePort.save(variableTypeModel); - } -} diff --git a/src/main/java/fr/insee/genesis/domain/utils/DataVerifier.java b/src/main/java/fr/insee/genesis/domain/utils/DataVerifier.java index 55a7abfd..45a30d7e 100644 --- a/src/main/java/fr/insee/genesis/domain/utils/DataVerifier.java +++ b/src/main/java/fr/insee/genesis/domain/utils/DataVerifier.java @@ -73,10 +73,10 @@ private static SurveyUnitModel createFormattedSurveyUnitModel( ) { SurveyUnitModel sampleSurveyUnitModel = surveyUnitModelsList.stream().filter(element -> element.getInterrogationId().equals(interrogationId)).toList().getFirst(); SurveyUnitModel newFormattedSurveyUnitModel = SurveyUnitModel.builder() - .questionnaireId(sampleSurveyUnitModel.getQuestionnaireId()) + .collectionInstrumentId(sampleSurveyUnitModel.getCollectionInstrumentId()) .campaignId(sampleSurveyUnitModel.getCampaignId()) .interrogationId(interrogationId) - .idUE(sampleSurveyUnitModel.getIdUE()) + .usualSurveyUnitId(sampleSurveyUnitModel.getUsualSurveyUnitId()) .state(DataState.FORMATTED) .mode(sampleSurveyUnitModel.getMode()) .recordDate(LocalDateTime.now().plusSeconds(1)) // Add 1 second to avoid same recordDate as COLLECTED diff --git a/src/main/java/fr/insee/genesis/exceptions/SchemaValidationException.java b/src/main/java/fr/insee/genesis/exceptions/SchemaValidationException.java new file mode 100644 index 00000000..37313109 --- /dev/null +++ b/src/main/java/fr/insee/genesis/exceptions/SchemaValidationException.java @@ -0,0 +1,16 @@ +package fr.insee.genesis.exceptions; + +import com.networknt.schema.ValidationMessage; +import lombok.AllArgsConstructor; +import lombok.Getter; + +import java.util.Set; + +@Getter +@AllArgsConstructor +public class SchemaValidationException extends Exception{ + + private final String message; + + private Set errors; +} diff --git a/src/main/java/fr/insee/genesis/infrastructure/adapter/ContextualExternalVariableMongoAdapter.java b/src/main/java/fr/insee/genesis/infrastructure/adapter/ContextualExternalVariableMongoAdapter.java index 80ca813e..6211c1b0 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/adapter/ContextualExternalVariableMongoAdapter.java +++ b/src/main/java/fr/insee/genesis/infrastructure/adapter/ContextualExternalVariableMongoAdapter.java @@ -14,6 +14,7 @@ import org.springframework.data.mongodb.core.query.Criteria; import org.springframework.stereotype.Service; +import java.util.ArrayList; import java.util.List; @Slf4j @@ -29,12 +30,17 @@ public ContextualExternalVariableMongoAdapter(ContextualExternalVariableMongoDBR } @Override - public void backup(String questionnaireId) { - deleteBackup(questionnaireId); - MatchOperation match = Aggregation.match(Criteria.where("questionnaireId").is(questionnaireId)); + public void backup(String collectionInstrumentId) { + deleteBackup(collectionInstrumentId); + MatchOperation match = Aggregation.match( + new Criteria().orOperator( + Criteria.where("questionnaireId").is(collectionInstrumentId), + Criteria.where("collectionInstrumentId").is(collectionInstrumentId) + ) + ); MergeOperation merge = Aggregation .merge() - .intoCollection(getFormattedCollection(questionnaireId)) + .intoCollection(getFormattedCollection(collectionInstrumentId)) .whenMatched(MergeOperation.WhenDocumentsMatch.replaceDocument()) .whenDocumentsDontMatch(MergeOperation.WhenDocumentsDontMatch.insertNewDocument()) .build(); @@ -56,8 +62,8 @@ public void deleteBackup(String questionnaireId) { } @Override - public void restoreBackup(String questionnaireId) { - delete(questionnaireId); + public void restoreBackup(String collectionInstrumentId) { + delete(collectionInstrumentId); MergeOperation merge = Aggregation .merge() .intoCollection("editedExternalResponses") @@ -67,7 +73,7 @@ public void restoreBackup(String questionnaireId) { Aggregation aggregation = Aggregation.newAggregation(merge); - mongoTemplate.aggregate(aggregation, getFormattedCollection(questionnaireId), + mongoTemplate.aggregate(aggregation, getFormattedCollection(collectionInstrumentId), ContextualExternalVariableDocument.class); } @@ -79,20 +85,24 @@ public void saveAll(List contextualExternalVari } @Override - public void delete(String questionnaireId) { - repository.deleteByQuestionnaireId(questionnaireId); + public void delete(String collectionInstrumentId) { + repository.deleteByCollectionInstrumentId(collectionInstrumentId); + // For older documents + repository.deleteByQuestionnaireId(collectionInstrumentId); } @Override - public ContextualExternalVariableModel findByQuestionnaireIdAndInterrogationId(String questionnaireId, String interrogationId) { - List contextualExternalVariableDocumentList = - repository.findByQuestionnaireIdAndInterrogationId(questionnaireId, interrogationId); - if(contextualExternalVariableDocumentList.isEmpty()){ + public ContextualExternalVariableModel findByCollectionInstrumentIdAndInterrogationId(String collectionInstrumentId, String interrogationId) { + List results = new ArrayList<>(); + results.addAll(repository.findByQuestionnaireIdAndInterrogationId(collectionInstrumentId, interrogationId)); + // For older documents + results.addAll(repository.findByCollectionInstrumentIdAndInterrogationId(collectionInstrumentId, interrogationId)); + if(results.isEmpty()){ return null; } - if(contextualExternalVariableDocumentList.size() > 1){ - log.warn("More than 1 contextual external response document for questionnaire {}, interrogation {}", questionnaireId, interrogationId); + if(results.size() > 1){ + log.warn("More than 1 contextual external response document for collection instrument {}, interrogation {}", collectionInstrumentId, interrogationId); } - return ContextualExternalVariableDocumentMapper.INSTANCE.documentToModel(contextualExternalVariableDocumentList.getFirst()); + return ContextualExternalVariableDocumentMapper.INSTANCE.documentToModel(results.getFirst()); } } diff --git a/src/main/java/fr/insee/genesis/infrastructure/adapter/ContextualPreviousVariableMongoAdapter.java b/src/main/java/fr/insee/genesis/infrastructure/adapter/ContextualPreviousVariableMongoAdapter.java index d8cf34f2..8a46393e 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/adapter/ContextualPreviousVariableMongoAdapter.java +++ b/src/main/java/fr/insee/genesis/infrastructure/adapter/ContextualPreviousVariableMongoAdapter.java @@ -14,6 +14,7 @@ import org.springframework.data.mongodb.core.query.Criteria; import org.springframework.stereotype.Service; +import java.util.ArrayList; import java.util.List; @Slf4j @@ -29,12 +30,17 @@ public ContextualPreviousVariableMongoAdapter(ContextualPreviousVariableMongoDBR } @Override - public void backup(String questionnaireId) { - deleteBackup(questionnaireId); - MatchOperation match = Aggregation.match(Criteria.where("questionnaireId").is(questionnaireId)); + public void backup(String collectionInstrumentId) { + deleteBackup(collectionInstrumentId); + MatchOperation match = Aggregation.match( + new Criteria().orOperator( + Criteria.where("questionnaireId").is(collectionInstrumentId), + Criteria.where("collectionInstrumentId").is(collectionInstrumentId) + ) + ); MergeOperation merge = Aggregation .merge() - .intoCollection(getFormattedCollection(questionnaireId)) + .intoCollection(getFormattedCollection(collectionInstrumentId)) .whenMatched(MergeOperation.WhenDocumentsMatch.replaceDocument()) .whenDocumentsDontMatch(MergeOperation.WhenDocumentsDontMatch.insertNewDocument()) .build(); @@ -44,20 +50,20 @@ public void backup(String questionnaireId) { mongoTemplate.aggregate(aggregation, "editedPreviousResponses", ContextualPreviousVariableDocument.class); } - private static String getFormattedCollection(String questionnaireId) { - return "editedPreviousResponses_%s_backup".formatted(questionnaireId); + private static String getFormattedCollection(String collectionInstrumentId) { + return "editedPreviousResponses_%s_backup".formatted(collectionInstrumentId); } @Override - public void deleteBackup(String questionnaireId) { - if (mongoTemplate.collectionExists(getFormattedCollection(questionnaireId))){ - mongoTemplate.dropCollection(getFormattedCollection(questionnaireId)); + public void deleteBackup(String collectionInstrumentId) { + if (mongoTemplate.collectionExists(getFormattedCollection(collectionInstrumentId))){ + mongoTemplate.dropCollection(getFormattedCollection(collectionInstrumentId)); } } @Override - public void restoreBackup(String questionnaireId) { - delete(questionnaireId); + public void restoreBackup(String collectionInstrumentId) { + delete(collectionInstrumentId); MergeOperation merge = Aggregation .merge() .intoCollection("editedPreviousResponses") @@ -67,7 +73,7 @@ public void restoreBackup(String questionnaireId) { Aggregation aggregation = Aggregation.newAggregation(merge); - mongoTemplate.aggregate(aggregation, getFormattedCollection(questionnaireId), + mongoTemplate.aggregate(aggregation, getFormattedCollection(collectionInstrumentId), ContextualPreviousVariableDocument.class); } @@ -79,20 +85,22 @@ public void saveAll(List contextualPreviousVari } @Override - public void delete(String questionnaireId) { - repository.deleteByQuestionnaireId(questionnaireId); + public void delete(String collectionInstrumentId) { + repository.deleteByCollectionInstrumentId(collectionInstrumentId); + repository.deleteByQuestionnaireId(collectionInstrumentId); } @Override - public ContextualPreviousVariableModel findByQuestionnaireIdAndInterrogationId(String questionnaireId, String interrogationId) { - List contextualPreviousVariableDocumentList = - repository.findByQuestionnaireIdAndInterrogationId(questionnaireId, interrogationId); - if(contextualPreviousVariableDocumentList.isEmpty()){ + public ContextualPreviousVariableModel findByCollectionInstrumentIdAndInterrogationId(String collectionInstrumentId, String interrogationId) { + List results = new ArrayList<>(); + results.addAll(repository.findByQuestionnaireIdAndInterrogationId(collectionInstrumentId, interrogationId)); + results.addAll(repository.findByCollectionInstrumentIdAndInterrogationId(collectionInstrumentId, interrogationId)); + if(results.isEmpty()){ return null; } - if(contextualPreviousVariableDocumentList.size() > 1){ - log.warn("More than 1 contextual previous response document for questionnaire {}, interrogation {}", questionnaireId, interrogationId); + if(results.size() > 1){ + log.warn("More than 1 contextual previous response document for collection instrument {}, interrogation {}", collectionInstrumentId, interrogationId); } - return ContextualPreviousVariableDocumentMapper.INSTANCE.documentToModel(contextualPreviousVariableDocumentList.getFirst()); + return ContextualPreviousVariableDocumentMapper.INSTANCE.documentToModel(results.getFirst()); } } diff --git a/src/main/java/fr/insee/genesis/infrastructure/adapter/DataProcessingContextMongoAdapter.java b/src/main/java/fr/insee/genesis/infrastructure/adapter/DataProcessingContextMongoAdapter.java index f104965a..f2aec1af 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/adapter/DataProcessingContextMongoAdapter.java +++ b/src/main/java/fr/insee/genesis/infrastructure/adapter/DataProcessingContextMongoAdapter.java @@ -7,6 +7,7 @@ import fr.insee.genesis.infrastructure.document.context.DataProcessingContextDocument; import fr.insee.genesis.infrastructure.mappers.DataProcessingContextMapper; import fr.insee.genesis.infrastructure.repository.DataProcessingContextMongoDBRepository; +import fr.insee.genesis.infrastructure.utils.FileUtils; import fr.insee.genesis.infrastructure.utils.context.ContextDedupUtils; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; @@ -17,6 +18,7 @@ import org.springframework.data.mongodb.core.query.Update; import org.springframework.stereotype.Service; +import java.io.IOException; import java.time.LocalDateTime; import java.util.ArrayList; import java.util.List; @@ -27,11 +29,14 @@ public class DataProcessingContextMongoAdapter implements DataProcessingContextPersistancePort { private final DataProcessingContextMongoDBRepository dataProcessingContextMongoDBRepository; private final MongoTemplate mongoTemplate; + private final FileUtils fileUtils; + @Autowired - public DataProcessingContextMongoAdapter(DataProcessingContextMongoDBRepository dataProcessingContextMongoDBRepository, MongoTemplate mongoTemplate) { + public DataProcessingContextMongoAdapter(DataProcessingContextMongoDBRepository dataProcessingContextMongoDBRepository, MongoTemplate mongoTemplate, FileUtils fileUtils) { this.dataProcessingContextMongoDBRepository = dataProcessingContextMongoDBRepository; this.mongoTemplate = mongoTemplate; + this.fileUtils = fileUtils; } @@ -49,6 +54,20 @@ public List findByPartitionIds(List partitio return DataProcessingContextMapper.INSTANCE.listDocumentToListModel(ContextDedupUtils.deduplicateContexts(existingDocuments)); } + @Override + public DataProcessingContextModel findByCollectionInstrumentId(String collectionInstrumentId) { + List existingDocuments = + dataProcessingContextMongoDBRepository.findByCollectionInstrumentIdList(List.of(collectionInstrumentId)); + return DataProcessingContextMapper.INSTANCE.documentToModel(existingDocuments.isEmpty()?null:existingDocuments.getFirst()); + } + + @Override + public List findByCollectionInstrumentIds(List collectionInstrumentIds) { + List existingDocuments = + dataProcessingContextMongoDBRepository.findByCollectionInstrumentIdList(collectionInstrumentIds); + return DataProcessingContextMapper.INSTANCE.listDocumentToListModel(existingDocuments); + } + @Override public void save(DataProcessingContextDocument dataProcessingContextDocument) { dataProcessingContextMongoDBRepository.save(dataProcessingContextDocument); @@ -66,7 +85,7 @@ public void deleteBypartitionId(String partitionId) { @Override public List findAll() { - return ContextDedupUtils.deduplicateContexts(dataProcessingContextMongoDBRepository.findAll()); + return dataProcessingContextMongoDBRepository.findAll(); } @Override @@ -75,7 +94,7 @@ public long count() { } @Override - public List removeExpiredSchedules(DataProcessingContextModel dataProcessingContextModel) { + public List removeExpiredSchedules(DataProcessingContextModel dataProcessingContextModel) throws IOException { List deletedKraftwerkExecutionSchedules = new ArrayList<>(); for (KraftwerkExecutionSchedule kraftwerkExecutionScheduleToRemove : dataProcessingContextModel.getKraftwerkExecutionScheduleList().stream().filter( @@ -84,6 +103,13 @@ public List removeExpiredSchedules(DataProcessingCon deletedKraftwerkExecutionSchedules.add(kraftwerkExecutionScheduleToRemove); Query query = Query.query(Criteria.where("scheduleEndDate").is(kraftwerkExecutionScheduleToRemove.getScheduleEndDate())); + // If collectionInstrumentId is present we use it, if not we use partitionId + if (dataProcessingContextModel.getCollectionInstrumentId()!=null){ + mongoTemplate.updateMulti(Query.query(Criteria.where("collectionInstrumentId").is(dataProcessingContextModel.getCollectionInstrumentId())), new Update().pull( + "kraftwerkExecutionScheduleList", query), + Constants.MONGODB_SCHEDULE_COLLECTION_NAME); + continue; + } mongoTemplate.updateMulti(Query.query(Criteria.where("surveyName").is(dataProcessingContextModel.getPartitionId())), new Update().pull( "kraftwerkExecutionScheduleList", query), Constants.MONGODB_SCHEDULE_COLLECTION_NAME); @@ -95,7 +121,7 @@ public List removeExpiredSchedules(DataProcessingCon @Override public List findAllByReview(boolean withReview) { - return ContextDedupUtils.deduplicateContexts(dataProcessingContextMongoDBRepository.findAll()) + return dataProcessingContextMongoDBRepository.findAll() .stream().filter(doc -> doc.isWithReview() == withReview ).toList(); diff --git a/src/main/java/fr/insee/genesis/infrastructure/adapter/LastJsonExtractionMongoAdapter.java b/src/main/java/fr/insee/genesis/infrastructure/adapter/LastJsonExtractionMongoAdapter.java index 89635f6b..0cbc0923 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/adapter/LastJsonExtractionMongoAdapter.java +++ b/src/main/java/fr/insee/genesis/infrastructure/adapter/LastJsonExtractionMongoAdapter.java @@ -32,13 +32,13 @@ public void save(LastJsonExtractionModel extraction) { } @Override - public LastJsonExtractionModel getLastExecutionDate(String questionnaireModelId, Mode mode) throws GenesisException { - String id = String.format("%s_%s",questionnaireModelId, mode); + public LastJsonExtractionModel getLastExecutionDate(String collectionInstrumentId, Mode mode) throws GenesisException { + String id = String.format("%s_%s",collectionInstrumentId, mode); Optional extraction = extractionRepository.findById(id); if (extraction.isPresent()) { return LastJsonExtractionDocumentMapper.INSTANCE.documentToModel(extraction.get()); } else { - String message = String.format("No extraction date found for questionnaire %s and mode %s",questionnaireModelId,mode==null?null:mode.getModeName()); + String message = String.format("No extraction date found for collection instrument %s and mode %s",collectionInstrumentId,mode==null?null:mode.getModeName()); throw new GenesisException(404,message); } } diff --git a/src/main/java/fr/insee/genesis/infrastructure/adapter/LunaticJsonRawDataMongoAdapter.java b/src/main/java/fr/insee/genesis/infrastructure/adapter/LunaticJsonRawDataMongoAdapter.java index 9dc510bf..78ca2568 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/adapter/LunaticJsonRawDataMongoAdapter.java +++ b/src/main/java/fr/insee/genesis/infrastructure/adapter/LunaticJsonRawDataMongoAdapter.java @@ -67,7 +67,7 @@ public void updateProcessDates(String campaignId, Set interrogationIds) @Override public Set findDistinctQuestionnaireIds() { Set questionnaireIds = new HashSet<>(); - for(String questionnaireId : mongoTemplate.getCollection(Constants.MONGODB_RESPONSE_RAW_COLLECTION_NAME).distinct( + for(String questionnaireId : mongoTemplate.getCollection(Constants.MONGODB_RAW_RESPONSES_COLLECTION_NAME).distinct( "questionnaireId", String.class)){ questionnaireIds.add(questionnaireId); diff --git a/src/main/java/fr/insee/genesis/infrastructure/adapter/LunaticModelMongoAdapter.java b/src/main/java/fr/insee/genesis/infrastructure/adapter/LunaticModelMongoAdapter.java index 44f3cc79..20324413 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/adapter/LunaticModelMongoAdapter.java +++ b/src/main/java/fr/insee/genesis/infrastructure/adapter/LunaticModelMongoAdapter.java @@ -7,13 +7,13 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.data.mongodb.core.MongoTemplate; +import org.springframework.data.mongodb.core.query.Criteria; import org.springframework.data.mongodb.core.query.Update; import org.springframework.stereotype.Service; +import java.util.ArrayList; import java.util.List; -import static org.springframework.data.mongodb.core.query.Criteria.where; - @Slf4j @Service @Qualifier("lunaticModelMongoAdapter") @@ -28,14 +28,26 @@ public LunaticModelMongoAdapter(LunaticModelMongoDBRepository repository, MongoT @Override public void save(LunaticModelModel lunaticModelModel) { + Criteria criteria = new Criteria().orOperator( + Criteria.where("questionnaireId").is(lunaticModelModel.collectionInstrumentId()), + Criteria.where("collectionInstrumentId").is(lunaticModelModel.collectionInstrumentId()) + ); mongoTemplate.update(LunaticModelDocument.class) - .matching(where("questionnaireId").is(lunaticModelModel.questionnaireId())) - .apply(new Update().set("lunaticModel", lunaticModelModel.lunaticModel())) + .matching(criteria) + .apply(new Update() + .set("lunaticModel", lunaticModelModel.lunaticModel()) + .setOnInsert("collectionInstrumentId", lunaticModelModel.collectionInstrumentId())) .upsert(); } @Override - public List find(String questionnaireId) { - return repository.findByQuestionnaireId(questionnaireId); + public List find(String collectionInstrumentId) { + List results = new ArrayList<>(); + results.addAll(repository.findByCollectionInstrumentId(collectionInstrumentId)); + results.addAll(repository.findByQuestionnaireId(collectionInstrumentId)); + return results.stream() + .distinct() + .toList(); } + } diff --git a/src/main/java/fr/insee/genesis/infrastructure/adapter/QuestionnaireMetadataMongoAdapter.java b/src/main/java/fr/insee/genesis/infrastructure/adapter/QuestionnaireMetadataMongoAdapter.java index 3317c481..4eed576c 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/adapter/QuestionnaireMetadataMongoAdapter.java +++ b/src/main/java/fr/insee/genesis/infrastructure/adapter/QuestionnaireMetadataMongoAdapter.java @@ -2,7 +2,8 @@ import fr.insee.genesis.domain.model.metadata.QuestionnaireMetadataModel; import fr.insee.genesis.domain.model.surveyunit.Mode; -import fr.insee.genesis.domain.ports.spi.QuestionnaireMetadataPersistancePort; +import fr.insee.genesis.domain.ports.spi.QuestionnaireMetadataPersistencePort; +import fr.insee.genesis.infrastructure.document.metadata.QuestionnaireMetadataDocument; import fr.insee.genesis.infrastructure.mappers.QuestionnaireMetadataDocumentMapper; import fr.insee.genesis.infrastructure.repository.QuestionnaireMetadataMongoDBRepository; import lombok.AllArgsConstructor; @@ -11,36 +12,37 @@ import org.springframework.cache.annotation.Cacheable; import org.springframework.stereotype.Service; +import java.util.ArrayList; import java.util.List; @Service @AllArgsConstructor @Qualifier("questionnaireMetadataMongoAdapter") -public class QuestionnaireMetadataMongoAdapter implements QuestionnaireMetadataPersistancePort { +public class QuestionnaireMetadataMongoAdapter implements QuestionnaireMetadataPersistencePort { private final QuestionnaireMetadataMongoDBRepository questionnaireMetadataMongoDBRepository; @Override - @Cacheable(value = "metadatas", key = "#questionnaireId + '-' + #mode") - public List find(String questionnaireId, Mode mode) { - return QuestionnaireMetadataDocumentMapper.INSTANCE.listDocumentToListModel( - questionnaireMetadataMongoDBRepository.findByQuestionnaireIdAndMode( - questionnaireId, mode - ) - ); + @Cacheable(value = "metadatas", key = "#collectionInstrumentId + '-' + #mode") + public List find(String collectionInstrumentId, Mode mode) { + List results = new ArrayList<>(); + results.addAll(questionnaireMetadataMongoDBRepository.findByQuestionnaireIdAndMode(collectionInstrumentId, mode)); + results.addAll(questionnaireMetadataMongoDBRepository.findByCollectionInstrumentIdAndMode(collectionInstrumentId, mode)); + return QuestionnaireMetadataDocumentMapper.INSTANCE.listDocumentToListModel(results); } @Override - @CacheEvict(value = "metadatas", key = "#questionnaireMetadataModel.questionnaireId() + '-' + #questionnaireMetadataModel.mode()") + @CacheEvict(value = "metadatas", key = "#questionnaireMetadataModel.collectionInstrumentId() + '-' + #questionnaireMetadataModel.mode()") public void save(QuestionnaireMetadataModel questionnaireMetadataModel) { - remove(questionnaireMetadataModel.questionnaireId(), questionnaireMetadataModel.mode()); + remove(questionnaireMetadataModel.collectionInstrumentId(), questionnaireMetadataModel.mode()); questionnaireMetadataMongoDBRepository.save( QuestionnaireMetadataDocumentMapper.INSTANCE.modelToDocument(questionnaireMetadataModel) ); } @Override - @CacheEvict(value = "metadatas", key = "#questionnaireId + '-' + #mode") - public void remove(String questionnaireId, Mode mode) { - questionnaireMetadataMongoDBRepository.deleteByQuestionnaireIdAndMode(questionnaireId, mode); + @CacheEvict(value = "metadatas", key = "#collectionInstrumentId + '-' + #mode") + public void remove(String collectionInstrumentId, Mode mode) { + questionnaireMetadataMongoDBRepository.deleteByQuestionnaireIdAndMode(collectionInstrumentId, mode); + questionnaireMetadataMongoDBRepository.deleteByCollectionInstrumentIdIdAndMode(collectionInstrumentId, mode); } } diff --git a/src/main/java/fr/insee/genesis/infrastructure/adapter/RawResponseMongoAdapter.java b/src/main/java/fr/insee/genesis/infrastructure/adapter/RawResponseMongoAdapter.java new file mode 100644 index 00000000..77f2d29d --- /dev/null +++ b/src/main/java/fr/insee/genesis/infrastructure/adapter/RawResponseMongoAdapter.java @@ -0,0 +1,61 @@ +package fr.insee.genesis.infrastructure.adapter; + +import fr.insee.genesis.Constants; +import fr.insee.genesis.domain.model.surveyunit.Mode; +import fr.insee.genesis.domain.model.surveyunit.rawdata.RawResponse; +import fr.insee.genesis.domain.ports.spi.RawResponsePersistencePort; +import fr.insee.genesis.infrastructure.document.rawdata.RawResponseDocument; +import fr.insee.genesis.infrastructure.mappers.RawResponseDocumentMapper; +import fr.insee.genesis.infrastructure.repository.RawResponseRepository; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.data.mongodb.core.MongoTemplate; +import org.springframework.data.mongodb.core.query.Criteria; +import org.springframework.data.mongodb.core.query.Query; +import org.springframework.data.mongodb.core.query.Update; +import org.springframework.stereotype.Service; + +import java.time.LocalDateTime; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +@Service +@Slf4j +@Qualifier("rawResponseMongoAdapter") +public class RawResponseMongoAdapter implements RawResponsePersistencePort { + + private final RawResponseRepository repository; + private final MongoTemplate mongoTemplate; + + public RawResponseMongoAdapter(RawResponseRepository repository, MongoTemplate mongoTemplate) { + this.repository = repository; + this.mongoTemplate = mongoTemplate; + } + + @Override + public List findRawResponses(String collectionInstrumentId, Mode mode, List interrogationIdList) { + List rawDataDocs = repository.findByCollectionInstrumentIdAndModeAndInterrogationIdList(collectionInstrumentId, mode.getJsonName(), interrogationIdList); + return RawResponseDocumentMapper.INSTANCE.listDocumentToListModel(rawDataDocs); + } + + @Override + public void updateProcessDates(String collectionInstrumentId, Set interrogationIds) { + mongoTemplate.updateMulti( + Query.query(Criteria.where("collectionInstrumentId").is(collectionInstrumentId).and("interrogationId").in(interrogationIds)) + , new Update().set("processDate", LocalDateTime.now()) + , Constants.MONGODB_RAW_RESPONSES_COLLECTION_NAME + ); + } + + @Override + public List getUnprocessedCollectionIds() { + return repository.findDistinctCollectionInstrumentIdByProcessDateIsNull(); + } + + @Override + public Set findUnprocessedInterrogationIdsByCollectionInstrumentId(String collectionInstrumentId) { + // We remove duplicate ids + return new HashSet<>(repository.findInterrogationIdByCollectionInstrumentIdAndProcessDateIsNull(collectionInstrumentId)); + } +} diff --git a/src/main/java/fr/insee/genesis/infrastructure/adapter/SurveyUnitMongoAdapter.java b/src/main/java/fr/insee/genesis/infrastructure/adapter/SurveyUnitMongoAdapter.java index 3bbbe509..79de453a 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/adapter/SurveyUnitMongoAdapter.java +++ b/src/main/java/fr/insee/genesis/infrastructure/adapter/SurveyUnitMongoAdapter.java @@ -3,6 +3,7 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import com.mongodb.client.MongoCollection; import fr.insee.genesis.Constants; import fr.insee.genesis.domain.model.surveyunit.SurveyUnitModel; import fr.insee.genesis.domain.ports.spi.SurveyUnitPersistencePort; @@ -11,6 +12,7 @@ import fr.insee.genesis.infrastructure.repository.SurveyUnitMongoDBRepository; import jakarta.validation.constraints.NotNull; import lombok.extern.slf4j.Slf4j; +import org.bson.Document; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.data.mongodb.core.MongoTemplate; @@ -46,9 +48,12 @@ public void saveAll(List surveyUnitModels) { } @Override - public List findByIds(String interrogationId, String questionnaireId) { - List surveyUnits = mongoRepository.findByInterrogationIdAndQuestionnaireId(interrogationId, questionnaireId); - return surveyUnits.isEmpty() ? Collections.emptyList() : SurveyUnitDocumentMapper.INSTANCE.listDocumentToListModel(surveyUnits); + public List findByIds(String interrogationId, String collectionInstrumentId) { + List results = new ArrayList<>(); + results.addAll(mongoRepository.findByInterrogationIdAndCollectionInstrumentId(interrogationId, collectionInstrumentId)); + // To ensure compatibility with older documents (with questionnaireId instead of collectionInstrumentId) + results.addAll(mongoRepository.findByInterrogationIdAndQuestionnaireId(interrogationId, collectionInstrumentId)); + return results.isEmpty() ? Collections.emptyList() : SurveyUnitDocumentMapper.INSTANCE.listDocumentToListModel(results); } @@ -87,8 +92,11 @@ public Stream findByQuestionnaireId(String questionnaireId) { } @Override - public Long deleteByQuestionnaireId(String questionnaireId) { - return mongoRepository.deleteByQuestionnaireId(questionnaireId); + public Long deleteByCollectionInstrumentId(String collectionInstrumentId) { + Long countDeleted = mongoRepository.deleteByCollectionInstrumentId(collectionInstrumentId); + // If the responses are in the old format (previous to modele filiere) + countDeleted += mongoRepository.deleteByQuestionnaireId(collectionInstrumentId); + return countDeleted; } @Override @@ -144,9 +152,11 @@ public Set findDistinctCampaignIds() { } @Override - public List findInterrogationIdsByQuestionnaireId(String questionnaireId) { - List surveyUnits = mongoRepository.findInterrogationIdsByQuestionnaireId(questionnaireId); - return surveyUnits.isEmpty() ? Collections.emptyList() : SurveyUnitDocumentMapper.INSTANCE.listDocumentToListModel(surveyUnits); + public List findInterrogationIdsByCollectionInstrumentId(String collectionInstrumentId) { + List results = new ArrayList<>(); + results.addAll(mongoRepository.findInterrogationIdsByCollectionInstrumentId(collectionInstrumentId)); + results.addAll(mongoRepository.findInterrogationIdsByQuestionnaireId(collectionInstrumentId)); + return results.isEmpty() ? Collections.emptyList() : SurveyUnitDocumentMapper.INSTANCE.listDocumentToListModel(results); } @Override @@ -195,11 +205,11 @@ public long countByCampaignId(String campaignId){ @Override public Set findDistinctQuestionnaireIds() { Set questionnaireIds = new HashSet<>(); - for(String questionnaireId : mongoTemplate.getCollection(Constants.MONGODB_RESPONSE_COLLECTION_NAME).distinct( - QUESTIONNAIRE_ID, - String.class)){ - questionnaireIds.add(questionnaireId); - } + // Id selection is executed by mongoDB + MongoCollection collection = mongoTemplate.getCollection(Constants.MONGODB_RESPONSE_COLLECTION_NAME); + collection.distinct(QUESTIONNAIRE_ID, String.class).into(questionnaireIds); + collection.distinct("collectionInstrumentId", String.class).into(questionnaireIds); + questionnaireIds.remove(null); return questionnaireIds; } diff --git a/src/main/java/fr/insee/genesis/infrastructure/adapter/VariableTypeMongoAdapter.java b/src/main/java/fr/insee/genesis/infrastructure/adapter/VariableTypeMongoAdapter.java deleted file mode 100644 index 9b12c07c..00000000 --- a/src/main/java/fr/insee/genesis/infrastructure/adapter/VariableTypeMongoAdapter.java +++ /dev/null @@ -1,44 +0,0 @@ -package fr.insee.genesis.infrastructure.adapter; - -import fr.insee.genesis.domain.model.surveyunit.Mode; -import fr.insee.genesis.domain.model.variabletype.VariableTypeModel; -import fr.insee.genesis.domain.ports.spi.VariableTypePersistancePort; -import fr.insee.genesis.infrastructure.document.variabletype.VariableTypeDocument; -import fr.insee.genesis.infrastructure.mappers.VariableTypeDocumentMapper; -import fr.insee.genesis.infrastructure.repository.VariableTypeMongoDBRepository; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.cache.annotation.Cacheable; -import org.springframework.data.mongodb.core.FindAndReplaceOptions; -import org.springframework.data.mongodb.core.MongoTemplate; -import org.springframework.data.mongodb.core.query.Criteria; -import org.springframework.data.mongodb.core.query.Query; -import org.springframework.stereotype.Service; - -@Service -@Qualifier("variableTypeMongoAdapter") -public class VariableTypeMongoAdapter implements VariableTypePersistancePort { - private final VariableTypeMongoDBRepository variableTypeMongoDBRepository; - private final MongoTemplate mongoTemplate; - - @Autowired - public VariableTypeMongoAdapter(VariableTypeMongoDBRepository variableTypeMongoDBRepository, MongoTemplate mongoTemplate) { - this.variableTypeMongoDBRepository = variableTypeMongoDBRepository; - this.mongoTemplate = mongoTemplate; - } - - @Override - public void save(VariableTypeModel variableTypeModel) { - mongoTemplate.update(VariableTypeDocument.class) - .matching(Query.query(Criteria.where("campaignId").is(variableTypeModel.campaignId()))) - .replaceWith(VariableTypeDocumentMapper.INSTANCE.modelToDocument(variableTypeModel)) - .withOptions(FindAndReplaceOptions.options().upsert()) - .findAndReplace(); - } - - @Override - @Cacheable("variableTypes") - public VariableTypeDocument find(String campaignId, String questionnaireId, Mode mode) { - return variableTypeMongoDBRepository.findFirstByCampaignIdQuestionnaireIdMode(campaignId, questionnaireId, mode); - } -} diff --git a/src/main/java/fr/insee/genesis/infrastructure/document/context/DataProcessingContextDocument.java b/src/main/java/fr/insee/genesis/infrastructure/document/context/DataProcessingContextDocument.java index c2fe4c2b..079886f7 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/document/context/DataProcessingContextDocument.java +++ b/src/main/java/fr/insee/genesis/infrastructure/document/context/DataProcessingContextDocument.java @@ -14,18 +14,13 @@ @Data @Document(collection = Constants.MONGODB_CONTEXT_COLLECTION_NAME) public class DataProcessingContextDocument{ - public DataProcessingContextDocument(String partitionId, - List kraftwerkExecutionScheduleList, - boolean withReview) { - this.partitionId = partitionId; - this.kraftwerkExecutionScheduleList = kraftwerkExecutionScheduleList; - this.withReview = withReview; - } @Id private ObjectId id; @Indexed private String partitionId; //ex Survey Name, campaignId + @Indexed + private String collectionInstrumentId; // QuestionnaireId private LocalDateTime lastExecution; private List kraftwerkExecutionScheduleList; private boolean withReview; diff --git a/src/main/java/fr/insee/genesis/infrastructure/document/contextualexternal/ContextualExternalVariableDocument.java b/src/main/java/fr/insee/genesis/infrastructure/document/contextualexternal/ContextualExternalVariableDocument.java index 1a15caa5..4518ead6 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/document/contextualexternal/ContextualExternalVariableDocument.java +++ b/src/main/java/fr/insee/genesis/infrastructure/document/contextualexternal/ContextualExternalVariableDocument.java @@ -1,5 +1,6 @@ package fr.insee.genesis.infrastructure.document.contextualexternal; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import fr.insee.genesis.Constants; import lombok.Data; import org.springframework.data.annotation.Id; @@ -12,11 +13,20 @@ @Data @Document(collection = Constants.MONGODB_CONTEXTUAL_EXTERNAL_COLLECTION_NAME) @CompoundIndex(name = "questionnaireId_1_interrogationId_1", def = "{'questionnaireId': 1, 'interrogationId': 1}") +@CompoundIndex(name = "collectionInstrumentId_1_interrogationId_1", def = "{'collectionInstrumentId': 1, 'interrogationId': 1}") +@JsonIgnoreProperties(ignoreUnknown = true) public class ContextualExternalVariableDocument { @Id private String id; + + /** + * @deprecated it will be replaced by collectionInstrumentId + */ + @Deprecated(forRemoval = true, since = "2026-01-01") @Indexed String questionnaireId; + @Indexed + String collectionInstrumentId; String interrogationId; Map variables; } diff --git a/src/main/java/fr/insee/genesis/infrastructure/document/contextualprevious/ContextualPreviousVariableDocument.java b/src/main/java/fr/insee/genesis/infrastructure/document/contextualprevious/ContextualPreviousVariableDocument.java index f3ef57a9..63b924ac 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/document/contextualprevious/ContextualPreviousVariableDocument.java +++ b/src/main/java/fr/insee/genesis/infrastructure/document/contextualprevious/ContextualPreviousVariableDocument.java @@ -1,5 +1,6 @@ package fr.insee.genesis.infrastructure.document.contextualprevious; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import fr.insee.genesis.Constants; import lombok.Data; import org.springframework.data.annotation.Id; @@ -12,11 +13,19 @@ @Data @Document(collection = Constants.MONGODB_CONTEXTUAL_PREVIOUS_COLLECTION_NAME) @CompoundIndex(name = "questionnaireId_1_interrogationId_1", def = "{'questionnaireId': 1, 'interrogationId': 1}") +@CompoundIndex(name = "collectionInstrumentId_1_interrogationId_1", def = "{'collectionInstrumentId': 1, 'interrogationId': 1}") +@JsonIgnoreProperties(ignoreUnknown = true) public class ContextualPreviousVariableDocument { @Id private String id; + /** + * @deprecated it will be replaced by collectionInstrumentId + */ + @Deprecated(forRemoval = true, since = "2026-01-01") @Indexed String questionnaireId; + @Indexed + String collectionInstrumentId; String interrogationId; Map variables; String sourceState; diff --git a/src/main/java/fr/insee/genesis/infrastructure/document/extraction/json/LastJsonExtractionDocument.java b/src/main/java/fr/insee/genesis/infrastructure/document/extraction/json/LastJsonExtractionDocument.java index 0e942f27..137af63c 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/document/extraction/json/LastJsonExtractionDocument.java +++ b/src/main/java/fr/insee/genesis/infrastructure/document/extraction/json/LastJsonExtractionDocument.java @@ -14,7 +14,7 @@ public class LastJsonExtractionDocument { @Id private String id; - private String questionnaireModelId; + private String collectionInstrumentId; private Mode mode; private LocalDateTime lastExtractionDate; diff --git a/src/main/java/fr/insee/genesis/infrastructure/document/lunaticmodel/LunaticModelDocument.java b/src/main/java/fr/insee/genesis/infrastructure/document/lunaticmodel/LunaticModelDocument.java index a07d3f91..b884d245 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/document/lunaticmodel/LunaticModelDocument.java +++ b/src/main/java/fr/insee/genesis/infrastructure/document/lunaticmodel/LunaticModelDocument.java @@ -7,11 +7,20 @@ import java.time.LocalDateTime; import java.util.Map; +/* It is not possible to respect these rules in a record (compilation error). +I choose to keep the record here (this choice can be challenged)*/ +@SuppressWarnings({ + "java:S1123", // deprecated annotation usage + "java:S6355" // Add 'since' and/or 'forRemoval' arguments to the @Deprecated annotation +}) @Builder @Document(collection = "lunaticmodels") public record LunaticModelDocument ( + @Deprecated (since = "2026-01-01", forRemoval = true) @Indexed String questionnaireId, + @Indexed + String collectionInstrumentId, Map lunaticModel, LocalDateTime recordDate ){} diff --git a/src/main/java/fr/insee/genesis/infrastructure/document/metadata/QuestionnaireMetadataDocument.java b/src/main/java/fr/insee/genesis/infrastructure/document/metadata/QuestionnaireMetadataDocument.java index 5f99ef28..8be83a6c 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/document/metadata/QuestionnaireMetadataDocument.java +++ b/src/main/java/fr/insee/genesis/infrastructure/document/metadata/QuestionnaireMetadataDocument.java @@ -6,11 +6,22 @@ import org.springframework.data.mongodb.core.index.Indexed; import org.springframework.data.mongodb.core.mapping.Document; +/* It is not possible to respect these rules in a record (compilation error). +I choose to keep the record here (this choice can be challenged)*/ +@SuppressWarnings({ + "java:S1123", // deprecated annotation usage + "java:S6355" // Add 'since' and/or 'forRemoval' arguments to the @Deprecated annotation +}) @CompoundIndex(name = "questionnaireId_1_mode_1", def = "{'questionnaireId': 1, 'mode': 1}") +@CompoundIndex(name = "collectionInstrumentId_1_mode_1", def = "{'collectionInstrumentId': 1, 'mode': 1}") @Document(collection = "questionnaireMetadatas") public record QuestionnaireMetadataDocument( + + @Deprecated (since = "2026-01-01", forRemoval = true) @Indexed String questionnaireId, + @Indexed + String collectionInstrumentId, Mode mode, MetadataModel metadataModel ){} diff --git a/src/main/java/fr/insee/genesis/infrastructure/document/rawdata/RawResponseDocument.java b/src/main/java/fr/insee/genesis/infrastructure/document/rawdata/RawResponseDocument.java new file mode 100644 index 00000000..e62e5a4c --- /dev/null +++ b/src/main/java/fr/insee/genesis/infrastructure/document/rawdata/RawResponseDocument.java @@ -0,0 +1,25 @@ +package fr.insee.genesis.infrastructure.document.rawdata; + +import lombok.Builder; +import org.bson.types.ObjectId; +import org.springframework.data.annotation.Id; +import org.springframework.data.mongodb.core.index.IndexDirection; +import org.springframework.data.mongodb.core.index.Indexed; +import org.springframework.data.mongodb.core.mapping.Document; + +import java.time.LocalDateTime; +import java.util.Map; + +@Builder +@Document(collection = "rawResponses") +public record RawResponseDocument ( + @Id + ObjectId id, + String interrogationId, + String collectionInstrumentId, + String mode, + Map payload, + LocalDateTime recordDate, + @Indexed(direction = IndexDirection.DESCENDING) + LocalDateTime processDate +){} diff --git a/src/main/java/fr/insee/genesis/infrastructure/document/surveyunit/SurveyUnitDocument.java b/src/main/java/fr/insee/genesis/infrastructure/document/surveyunit/SurveyUnitDocument.java index 1977c949..9b1e8ac5 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/document/surveyunit/SurveyUnitDocument.java +++ b/src/main/java/fr/insee/genesis/infrastructure/document/surveyunit/SurveyUnitDocument.java @@ -1,5 +1,6 @@ package fr.insee.genesis.infrastructure.document.surveyunit; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import fr.insee.genesis.Constants; import lombok.Data; import org.springframework.data.mongodb.core.index.CompoundIndex; @@ -15,12 +16,36 @@ @CompoundIndex(name = "questionnaireId_1_campaignId_1", def = "{'questionnaireId': 1, 'campaignId': 1}") @CompoundIndex(name = "questionnaireId_1_interrogationId_1", def = "{'questionnaireId': 1, 'interrogationId': 1}") @CompoundIndex(name = "interrogationId_1_questionnaireId_1", def = "{'interrogationId': 1, 'questionnaireId': 1}") +@CompoundIndex(name = "collectionInstrumentId_1_interrogationId_1", def = "{'collectionInstrumentId': 1, 'interrogationId': 1}") +@CompoundIndex(name = "interrogationId_1_collectionInstrumentId_1", def = "{'interrogationId': 1, 'collectionInstrumentId': 1}") +@JsonIgnoreProperties(ignoreUnknown = true) public class SurveyUnitDocument { + + /** + * @deprecated This piece of information will not be available anymore in the raw responses + */ + @Deprecated(forRemoval = true, since ="2026-01-01") private String campaignId; @Indexed private String interrogationId; + + /** + * @deprecated It will be replaced by usualSurveyUnitId + */ + @Deprecated(forRemoval = true, since ="2026-01-01") private String idUE; + + private String usualSurveyUnitId; + + /** + * @deprecated It will be replaced by collectionInstrumentId + */ + @Deprecated(forRemoval = true, since ="2026-01-01") private String questionnaireId; + + private String collectionInstrumentId; + + private String majorModelVersion; private String state; @Indexed private String mode; diff --git a/src/main/java/fr/insee/genesis/infrastructure/document/variabletype/VariableTypeDocument.java b/src/main/java/fr/insee/genesis/infrastructure/document/variabletype/VariableTypeDocument.java deleted file mode 100644 index 5fb3cc6a..00000000 --- a/src/main/java/fr/insee/genesis/infrastructure/document/variabletype/VariableTypeDocument.java +++ /dev/null @@ -1,16 +0,0 @@ -package fr.insee.genesis.infrastructure.document.variabletype; - -import fr.insee.bpm.metadata.model.VariablesMap; -import fr.insee.genesis.Constants; -import fr.insee.genesis.domain.model.surveyunit.Mode; -import lombok.Data; -import org.springframework.data.mongodb.core.mapping.Document; - -@Data -@Document(collection = Constants.MONGODB_VARIABLETYPE_COLLECTION_NAME) -public class VariableTypeDocument { - private String campaignId; - private String questionnaireId; - private Mode mode; - private VariablesMap variablesMap; -} diff --git a/src/main/java/fr/insee/genesis/infrastructure/mappers/ContextualExternalVariableDocumentMapper.java b/src/main/java/fr/insee/genesis/infrastructure/mappers/ContextualExternalVariableDocumentMapper.java index 5366caff..991b64fc 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/mappers/ContextualExternalVariableDocumentMapper.java +++ b/src/main/java/fr/insee/genesis/infrastructure/mappers/ContextualExternalVariableDocumentMapper.java @@ -2,7 +2,9 @@ import fr.insee.genesis.domain.model.contextualvariable.ContextualExternalVariableModel; import fr.insee.genesis.infrastructure.document.contextualexternal.ContextualExternalVariableDocument; +import org.mapstruct.AfterMapping; import org.mapstruct.Mapper; +import org.mapstruct.MappingTarget; import org.mapstruct.factory.Mappers; import java.util.List; @@ -18,4 +20,14 @@ public interface ContextualExternalVariableDocumentMapper { List listDocumentToListModel(List rawDataDocumentList); List listModelToListDocument(List rawDataModelList); + + @AfterMapping + default void fillModelAfterRead(ContextualExternalVariableDocument doc, + @MappingTarget ContextualExternalVariableModel model) { + + if (model.getCollectionInstrumentId() == null) { + model.setCollectionInstrumentId(doc.getQuestionnaireId()); + } + + } } diff --git a/src/main/java/fr/insee/genesis/infrastructure/mappers/ContextualPreviousVariableDocumentMapper.java b/src/main/java/fr/insee/genesis/infrastructure/mappers/ContextualPreviousVariableDocumentMapper.java index d4919fc6..ba72e779 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/mappers/ContextualPreviousVariableDocumentMapper.java +++ b/src/main/java/fr/insee/genesis/infrastructure/mappers/ContextualPreviousVariableDocumentMapper.java @@ -2,7 +2,9 @@ import fr.insee.genesis.domain.model.contextualvariable.ContextualPreviousVariableModel; import fr.insee.genesis.infrastructure.document.contextualprevious.ContextualPreviousVariableDocument; +import org.mapstruct.AfterMapping; import org.mapstruct.Mapper; +import org.mapstruct.MappingTarget; import org.mapstruct.factory.Mappers; import java.util.List; @@ -18,4 +20,14 @@ public interface ContextualPreviousVariableDocumentMapper { List listDocumentToListModel(List rawDataDocumentList); List listModelToListDocument(List rawDataModelList); + + @AfterMapping + default void fillModelAfterRead(ContextualPreviousVariableDocument doc, + @MappingTarget ContextualPreviousVariableModel model) { + + if (model.getCollectionInstrumentId() == null) { + model.setCollectionInstrumentId(doc.getQuestionnaireId()); + } + + } } diff --git a/src/main/java/fr/insee/genesis/infrastructure/mappers/LunaticModelMapper.java b/src/main/java/fr/insee/genesis/infrastructure/mappers/LunaticModelMapper.java index d5702cbd..ca65dac3 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/mappers/LunaticModelMapper.java +++ b/src/main/java/fr/insee/genesis/infrastructure/mappers/LunaticModelMapper.java @@ -1,7 +1,10 @@ package fr.insee.genesis.infrastructure.mappers; + import fr.insee.genesis.domain.model.lunaticmodel.LunaticModelModel; import fr.insee.genesis.infrastructure.document.lunaticmodel.LunaticModelDocument; +import org.mapstruct.AfterMapping; import org.mapstruct.Mapper; +import org.mapstruct.MappingTarget; import org.mapstruct.factory.Mappers; import java.util.List; @@ -17,4 +20,15 @@ public interface LunaticModelMapper { List listDocumentToListModel(List documentList); List listModelToListDocument(List modelList); + + @SuppressWarnings("removal") + @AfterMapping + default void fillModelAfterRead( + LunaticModelDocument doc, + @MappingTarget LunaticModelModel.LunaticModelModelBuilder builder + ) { + if (doc.collectionInstrumentId() == null && doc.questionnaireId()!=null) { + builder.collectionInstrumentId(doc.questionnaireId()); + } + } } diff --git a/src/main/java/fr/insee/genesis/infrastructure/mappers/QuestionnaireMetadataDocumentMapper.java b/src/main/java/fr/insee/genesis/infrastructure/mappers/QuestionnaireMetadataDocumentMapper.java index 8b951f3e..94ed2060 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/mappers/QuestionnaireMetadataDocumentMapper.java +++ b/src/main/java/fr/insee/genesis/infrastructure/mappers/QuestionnaireMetadataDocumentMapper.java @@ -2,7 +2,9 @@ import fr.insee.genesis.domain.model.metadata.QuestionnaireMetadataModel; import fr.insee.genesis.infrastructure.document.metadata.QuestionnaireMetadataDocument; +import org.mapstruct.AfterMapping; import org.mapstruct.Mapper; +import org.mapstruct.MappingTarget; import org.mapstruct.factory.Mappers; import java.util.List; @@ -18,4 +20,15 @@ public interface QuestionnaireMetadataDocumentMapper { List listDocumentToListModel(List documentList); List listModelToListDocument(List modelList); + + @SuppressWarnings("removal") + @AfterMapping + default void fillModelAfterRead( + QuestionnaireMetadataDocument doc, + @MappingTarget QuestionnaireMetadataModel.QuestionnaireMetadataModelBuilder builder + ) { + if (doc.collectionInstrumentId() == null && doc.questionnaireId()!=null) { + builder.collectionInstrumentId(doc.questionnaireId()); + } + } } diff --git a/src/main/java/fr/insee/genesis/infrastructure/mappers/RawResponseDocumentMapper.java b/src/main/java/fr/insee/genesis/infrastructure/mappers/RawResponseDocumentMapper.java new file mode 100644 index 00000000..96e5e073 --- /dev/null +++ b/src/main/java/fr/insee/genesis/infrastructure/mappers/RawResponseDocumentMapper.java @@ -0,0 +1,30 @@ +package fr.insee.genesis.infrastructure.mappers; + +import fr.insee.genesis.domain.model.surveyunit.Mode; +import fr.insee.genesis.domain.model.surveyunit.rawdata.RawResponse; +import fr.insee.genesis.infrastructure.document.rawdata.RawResponseDocument; +import org.mapstruct.Mapper; +import org.mapstruct.factory.Mappers; + +import java.util.List; + +@Mapper +public interface RawResponseDocumentMapper { + + RawResponseDocumentMapper INSTANCE = Mappers.getMapper(RawResponseDocumentMapper.class); + + RawResponse documentToModel(RawResponseDocument document); + RawResponseDocument modelToDocument(RawResponse model); + List listDocumentToListModel(List documentList); + List listModelToListDocument(List modelList); + + // --- Custom mapping: String -> Mode + default Mode stringToMode(String value) { + return Mode.fromString(value); // réutilise ton JsonCreator (parfait) + } + + // --- Custom mapping: Mode -> String + default String modeToString(Mode mode) { + return mode != null ? mode.getJsonName() : null; + } +} diff --git a/src/main/java/fr/insee/genesis/infrastructure/mappers/SurveyUnitDocumentMapper.java b/src/main/java/fr/insee/genesis/infrastructure/mappers/SurveyUnitDocumentMapper.java index fb2df28a..16fe8618 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/mappers/SurveyUnitDocumentMapper.java +++ b/src/main/java/fr/insee/genesis/infrastructure/mappers/SurveyUnitDocumentMapper.java @@ -2,7 +2,9 @@ import fr.insee.genesis.domain.model.surveyunit.SurveyUnitModel; import fr.insee.genesis.infrastructure.document.surveyunit.SurveyUnitDocument; +import org.mapstruct.AfterMapping; import org.mapstruct.Mapper; +import org.mapstruct.MappingTarget; import org.mapstruct.factory.Mappers; import java.util.List; @@ -19,4 +21,18 @@ public interface SurveyUnitDocumentMapper { List listModelToListDocument(List surveyUnitModels); + @AfterMapping + default void fillModelAfterRead(SurveyUnitDocument doc, + @MappingTarget SurveyUnitModel model) { + + if (model.getUsualSurveyUnitId() == null) { + model.setUsualSurveyUnitId(doc.getIdUE()); + } + + if (model.getCollectionInstrumentId() == null) { + model.setCollectionInstrumentId(doc.getQuestionnaireId()); + } + + } + } diff --git a/src/main/java/fr/insee/genesis/infrastructure/mappers/VariableTypeDocumentMapper.java b/src/main/java/fr/insee/genesis/infrastructure/mappers/VariableTypeDocumentMapper.java deleted file mode 100644 index ed8cdecb..00000000 --- a/src/main/java/fr/insee/genesis/infrastructure/mappers/VariableTypeDocumentMapper.java +++ /dev/null @@ -1,21 +0,0 @@ -package fr.insee.genesis.infrastructure.mappers; - -import fr.insee.genesis.domain.model.variabletype.VariableTypeModel; -import fr.insee.genesis.infrastructure.document.variabletype.VariableTypeDocument; -import org.mapstruct.Mapper; -import org.mapstruct.factory.Mappers; - -import java.util.List; - -@Mapper(componentModel = "spring") -public interface VariableTypeDocumentMapper { - VariableTypeDocumentMapper INSTANCE = Mappers.getMapper(VariableTypeDocumentMapper.class); - - VariableTypeModel documentToModel(VariableTypeDocument variableTypeDocument); - - VariableTypeDocument modelToDocument(VariableTypeModel variableTypeModel); - - List listDocumentToListModel(List variableTypeDocuments); - - List listModelToListDocument(List variableTypeModels); -} diff --git a/src/main/java/fr/insee/genesis/infrastructure/repository/ContextualExternalVariableMongoDBRepository.java b/src/main/java/fr/insee/genesis/infrastructure/repository/ContextualExternalVariableMongoDBRepository.java index e9bc22ec..9b264e19 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/repository/ContextualExternalVariableMongoDBRepository.java +++ b/src/main/java/fr/insee/genesis/infrastructure/repository/ContextualExternalVariableMongoDBRepository.java @@ -10,7 +10,11 @@ @Repository public interface ContextualExternalVariableMongoDBRepository extends MongoRepository { void deleteByQuestionnaireId(String questionnaireId); + void deleteByCollectionInstrumentId(String collectionInstrumentId); @Query(value = "{ 'questionnaireId' : ?0, 'interrogationId' : ?1 }") List findByQuestionnaireIdAndInterrogationId(String questionnaireId, String interrogationId); + + @Query(value = "{ 'collectionInstrumentId' : ?0, 'interrogationId' : ?1 }") + List findByCollectionInstrumentIdAndInterrogationId(String collectionInstrumentId, String interrogationId); } diff --git a/src/main/java/fr/insee/genesis/infrastructure/repository/ContextualPreviousVariableMongoDBRepository.java b/src/main/java/fr/insee/genesis/infrastructure/repository/ContextualPreviousVariableMongoDBRepository.java index f5604747..9adcecdf 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/repository/ContextualPreviousVariableMongoDBRepository.java +++ b/src/main/java/fr/insee/genesis/infrastructure/repository/ContextualPreviousVariableMongoDBRepository.java @@ -10,7 +10,10 @@ @Repository public interface ContextualPreviousVariableMongoDBRepository extends MongoRepository { void deleteByQuestionnaireId(String questionnaireId); + void deleteByCollectionInstrumentId(String collectionInstrumentId); @Query(value = "{ 'questionnaireId' : ?0, 'interrogationId' : ?1 }") List findByQuestionnaireIdAndInterrogationId(String questionnaireId, String interrogationId); + @Query(value = "{ 'collectionInstrumentId' : ?0, 'interrogationId' : ?1 }") + List findByCollectionInstrumentIdAndInterrogationId(String collectionInstrumentId, String interrogationId); } diff --git a/src/main/java/fr/insee/genesis/infrastructure/repository/DataProcessingContextMongoDBRepository.java b/src/main/java/fr/insee/genesis/infrastructure/repository/DataProcessingContextMongoDBRepository.java index 35eb37a8..0f7e2d72 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/repository/DataProcessingContextMongoDBRepository.java +++ b/src/main/java/fr/insee/genesis/infrastructure/repository/DataProcessingContextMongoDBRepository.java @@ -9,9 +9,18 @@ @Repository public interface DataProcessingContextMongoDBRepository extends MongoRepository { + @Deprecated(forRemoval = true) @Query(value = "{ 'partitionId' : {$in: ?0} }") List findByPartitionIdList(List partitionIds); + + @Query(value = "{ 'collectionInstrumentId' : {$in: ?0} }") + List findByCollectionInstrumentIdList(List collectionInstrumentIds); + + @Deprecated(forRemoval = true) @Query(value = "{ 'partitionId' : ?0 }", delete = true) void deleteByPartitionId(String partitionId); + + @Query(value = "{ 'collectionInstrumentId' : ?0 }", delete = true) + void deleteByCollectionInstrumentId(String partitionId); } diff --git a/src/main/java/fr/insee/genesis/infrastructure/repository/LunaticModelMongoDBRepository.java b/src/main/java/fr/insee/genesis/infrastructure/repository/LunaticModelMongoDBRepository.java index faa4faa8..01ab6d86 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/repository/LunaticModelMongoDBRepository.java +++ b/src/main/java/fr/insee/genesis/infrastructure/repository/LunaticModelMongoDBRepository.java @@ -12,4 +12,7 @@ public interface LunaticModelMongoDBRepository extends CrudRepository findByQuestionnaireId(String questionnaireId); + + @Query("{'collectionInstrumentId' : ?0}") + List findByCollectionInstrumentId(String collectionInstrumentId); } diff --git a/src/main/java/fr/insee/genesis/infrastructure/repository/QuestionnaireMetadataMongoDBRepository.java b/src/main/java/fr/insee/genesis/infrastructure/repository/QuestionnaireMetadataMongoDBRepository.java index 2d3484d1..3f09e88f 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/repository/QuestionnaireMetadataMongoDBRepository.java +++ b/src/main/java/fr/insee/genesis/infrastructure/repository/QuestionnaireMetadataMongoDBRepository.java @@ -11,6 +11,12 @@ public interface QuestionnaireMetadataMongoDBRepository extends CrudRepository findByQuestionnaireIdAndMode(String questionnaireId, Mode mode); + @Query("{'collectionInstrumentId' : ?0, 'mode' : ?1}") + List findByCollectionInstrumentIdAndMode(String collectionInstrumentId, Mode mode); + @Query(value = "{'questionnaireId' : ?0, 'mode' : ?1}", delete = true) void deleteByQuestionnaireIdAndMode(String questionnaireId, Mode mode); + + @Query(value = "{'collectionInstrumentId' : ?0, 'mode' : ?1}", delete = true) + void deleteByCollectionInstrumentIdIdAndMode(String collectionInstrumentId, Mode mode); } diff --git a/src/main/java/fr/insee/genesis/infrastructure/repository/RawResponseInputRepository.java b/src/main/java/fr/insee/genesis/infrastructure/repository/RawResponseInputRepository.java new file mode 100644 index 00000000..85a34ec5 --- /dev/null +++ b/src/main/java/fr/insee/genesis/infrastructure/repository/RawResponseInputRepository.java @@ -0,0 +1,42 @@ +package fr.insee.genesis.infrastructure.repository; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import fr.insee.modelefiliere.RawResponseDto; +import lombok.RequiredArgsConstructor; +import org.bson.Document; +import org.springframework.data.mongodb.core.MongoTemplate; +import org.springframework.stereotype.Repository; + +import java.time.Instant; +import java.util.Map; + +@Repository +@RequiredArgsConstructor +public class RawResponseInputRepository { + + private final MongoTemplate mongoTemplate; + private final ObjectMapper objectMapper; + + public void saveAsRawJson(RawResponseDto dto) { + // Convertir le DTO en JSON brut + String json = null; + + try { + json = objectMapper.writeValueAsString(dto); + // Parser en Document BSON (Mongo gère automatiquement les types) + Document payload = Document.parse(json); + + Map document = new java.util.HashMap<>(); + document.put("interrogationId", dto.getInterrogationId()); + document.put("collectionInstrumentId", dto.getCollectionInstrumentId()); + document.put("mode", dto.getMode()); + document.put("recordDate", Instant.now()); + document.put("payload", payload); + mongoTemplate.save(document, "rawResponses"); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + + } +} diff --git a/src/main/java/fr/insee/genesis/infrastructure/repository/RawResponseRepository.java b/src/main/java/fr/insee/genesis/infrastructure/repository/RawResponseRepository.java new file mode 100644 index 00000000..e09e9fd7 --- /dev/null +++ b/src/main/java/fr/insee/genesis/infrastructure/repository/RawResponseRepository.java @@ -0,0 +1,29 @@ +package fr.insee.genesis.infrastructure.repository; + +import fr.insee.genesis.infrastructure.document.rawdata.RawResponseDocument; +import org.springframework.data.mongodb.repository.Aggregation; +import org.springframework.data.mongodb.repository.MongoRepository; +import org.springframework.data.mongodb.repository.Query; +import org.springframework.stereotype.Repository; + +import java.util.List; + +@Repository +public interface RawResponseRepository extends MongoRepository { + + @Query(value = "{ 'collectionInstrumentId' : ?0, 'mode' : ?1, 'interrogationId': {$in: ?2} }") + List findByCollectionInstrumentIdAndModeAndInterrogationIdList(String questionnaireId, String mode, List interrogationIdList); + @Aggregation(pipeline = { + "{ $match: { processDate: null } }", + "{ $group: { _id: '$collectionInstrumentId' } }", + "{ $project: { _id: 0, collectionInstrumentId: '$_id' } }" + }) + List findDistinctCollectionInstrumentIdByProcessDateIsNull(); + + @Aggregation(pipeline = { + "{ $match: { collectionInstrumentId: ?0,processDate: null } }", + "{ $project: { _id: 0, interrogationId: '$interrogationId' } }" + }) + List findInterrogationIdByCollectionInstrumentIdAndProcessDateIsNull(String collectionInstrumentId); + +} diff --git a/src/main/java/fr/insee/genesis/infrastructure/repository/SurveyUnitMongoDBRepository.java b/src/main/java/fr/insee/genesis/infrastructure/repository/SurveyUnitMongoDBRepository.java index d1c5841d..63999e7c 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/repository/SurveyUnitMongoDBRepository.java +++ b/src/main/java/fr/insee/genesis/infrastructure/repository/SurveyUnitMongoDBRepository.java @@ -18,6 +18,7 @@ public interface SurveyUnitMongoDBRepository extends MongoRepository findByInterrogationId(String interrogationId); List findByInterrogationIdAndQuestionnaireId(String interrogationId, String questionnaireId); + List findByInterrogationIdAndCollectionInstrumentId(String interrogationId, String collectionInstrumentId); //========= OPTIMISATIONS PERFS (START) ========== /** @@ -30,6 +31,9 @@ public interface SurveyUnitMongoDBRepository extends MongoRepository findInterrogationIdsByQuestionnaireId(String questionnaireId); + @Query(value = "{ 'collectionInstrumentId' : ?0 }", fields = "{ 'interrogationId' : 1, 'mode' : 1 }") + List findInterrogationIdsByCollectionInstrumentId(String questionnaireId); + @Query(value = "{ 'questionnaireId' : ?0, 'recordDate': { $gte: ?1 } }", fields = "{ 'interrogationId' : 1, 'mode' : 1 }") List findInterrogationIdsByQuestionnaireIdAndDateAfter(String questionnaireId, LocalDateTime since); @@ -71,6 +75,7 @@ public interface SurveyUnitMongoDBRepository extends MongoRepository findInterrogationIdsByCampaignId(String campaignId); Long deleteByQuestionnaireId(String questionnaireId); + Long deleteByCollectionInstrumentId(String collectionInstrumentId); @Meta(cursorBatchSize = 20) Stream findByQuestionnaireId(String questionnaireId); diff --git a/src/main/java/fr/insee/genesis/infrastructure/repository/VariableTypeMongoDBRepository.java b/src/main/java/fr/insee/genesis/infrastructure/repository/VariableTypeMongoDBRepository.java deleted file mode 100644 index ecedd6d1..00000000 --- a/src/main/java/fr/insee/genesis/infrastructure/repository/VariableTypeMongoDBRepository.java +++ /dev/null @@ -1,14 +0,0 @@ -package fr.insee.genesis.infrastructure.repository; - -import fr.insee.genesis.domain.model.surveyunit.Mode; -import fr.insee.genesis.infrastructure.document.variabletype.VariableTypeDocument; -import org.springframework.data.mongodb.repository.MongoRepository; -import org.springframework.data.mongodb.repository.Query; -import org.springframework.stereotype.Repository; - -@Repository -public interface VariableTypeMongoDBRepository extends MongoRepository { - - @Query(value = "{ 'campaignId' : ?0 , 'questionnaireId' : ?1 , 'mode' : ?2 }") - VariableTypeDocument findFirstByCampaignIdQuestionnaireIdMode(String campaignId, String questionnaireId, Mode mode); -} diff --git a/src/test/java/cucumber/functional_tests/DataProcessingContextDefinitions.java b/src/test/java/cucumber/functional_tests/DataProcessingContextDefinitions.java index 90a488fc..35011b41 100644 --- a/src/test/java/cucumber/functional_tests/DataProcessingContextDefinitions.java +++ b/src/test/java/cucumber/functional_tests/DataProcessingContextDefinitions.java @@ -46,27 +46,29 @@ public void init() { @Given("We have a context in database with partition {string}") public void add_context_to_database(String partitionId) { - dataProcessingContextPersistancePortStub.getMongoStub().add( - new DataProcessingContextDocument(partitionId, new ArrayList<>(), false) - ); + DataProcessingContextDocument doc = new DataProcessingContextDocument(); + doc.setPartitionId(partitionId); + doc.setKraftwerkExecutionScheduleList(new ArrayList<>()); + doc.setWithReview(false); + dataProcessingContextPersistancePortStub.getMongoStub().add(doc); } @Given("We have a context in database with partition {string} and review indicator to {string}") public void add_context_with_review_indicator(String partitionId, String withReviewString) { - boolean withReview = Boolean.parseBoolean(withReviewString); - dataProcessingContextPersistancePortStub.getMongoStub().add( - new DataProcessingContextDocument(partitionId, new ArrayList<>(), withReview) - ); + DataProcessingContextDocument doc = new DataProcessingContextDocument(); + doc.setPartitionId(partitionId); + doc.setKraftwerkExecutionScheduleList(new ArrayList<>()); + doc.setWithReview(Boolean.parseBoolean(withReviewString)); + dataProcessingContextPersistancePortStub.getMongoStub().add(doc); } @Given("We have a context in database with partition {string} and {int} valid schedule\\(s)") public void add_context_with_schedule(String partitionId, int expectedScheduleNumber) { - DataProcessingContextDocument dataProcessingContextDocument = new DataProcessingContextDocument( - partitionId, - new ArrayList<>(), - false - ); + DataProcessingContextDocument dataProcessingContextDocument = new DataProcessingContextDocument(); + dataProcessingContextDocument.setPartitionId(partitionId); + dataProcessingContextDocument.setKraftwerkExecutionScheduleList(new ArrayList<>()); + dataProcessingContextDocument.setWithReview(false); for(int i = 0; i < expectedScheduleNumber; i++){ dataProcessingContextDocument.getKraftwerkExecutionScheduleList().add( @@ -85,11 +87,10 @@ public void add_context_with_schedule(String partitionId, int expectedScheduleNu @Given("We have a context in database with partition {string} and {int} expired schedule\\(s)") public void add_context_with_expired_schedule(String partitionId, int expectedScheduleNumber) { - DataProcessingContextDocument dataProcessingContextDocument = new DataProcessingContextDocument( - partitionId, - new ArrayList<>(), - false - ); + DataProcessingContextDocument dataProcessingContextDocument = new DataProcessingContextDocument(); + dataProcessingContextDocument.setPartitionId(partitionId); + dataProcessingContextDocument.setKraftwerkExecutionScheduleList(new ArrayList<>()); + dataProcessingContextDocument.setWithReview(false); for(int i = 0; i < expectedScheduleNumber; i++){ dataProcessingContextDocument.getKraftwerkExecutionScheduleList().add( diff --git a/src/test/java/cucumber/functional_tests/LunaticModelDefinitions.java b/src/test/java/cucumber/functional_tests/LunaticModelDefinitions.java index efb5916c..dca3aac2 100644 --- a/src/test/java/cucumber/functional_tests/LunaticModelDefinitions.java +++ b/src/test/java/cucumber/functional_tests/LunaticModelDefinitions.java @@ -17,7 +17,7 @@ import fr.insee.genesis.stubs.ConfigStub; import fr.insee.genesis.stubs.DataProcessingContextPersistancePortStub; import fr.insee.genesis.stubs.LunaticModelPersistanceStub; -import fr.insee.genesis.stubs.QuestionnaireMetadataPersistancePortStub; +import fr.insee.genesis.stubs.QuestionnaireMetadataPersistencePortStub; import fr.insee.genesis.stubs.SurveyUnitPersistencePortStub; import io.cucumber.java.Before; import io.cucumber.java.en.Given; @@ -49,11 +49,11 @@ public class LunaticModelDefinitions { LunaticModelController lunaticModelController = new LunaticModelController(new LunaticModelService(lunaticModelPersistanceStub)); SurveyUnitPersistencePortStub surveyUnitPersistencePortStub = new SurveyUnitPersistencePortStub(); - static QuestionnaireMetadataPersistancePortStub questionnaireMetadataPersistancePortStub = new QuestionnaireMetadataPersistancePortStub(); + static QuestionnaireMetadataPersistencePortStub questionnaireMetadataPersistencePortStub = new QuestionnaireMetadataPersistencePortStub(); QuestionnaireController questionnaireController = new QuestionnaireController( new SurveyUnitService( surveyUnitPersistencePortStub, - new QuestionnaireMetadataService(questionnaireMetadataPersistancePortStub), + new QuestionnaireMetadataService(questionnaireMetadataPersistencePortStub), new FileUtils(new ConfigStub()) ), new DataProcessingContextService( @@ -108,7 +108,7 @@ public void load_response(String campaignId, String questionnaireId, String inte surveyUnitPersistencePortStub.getMongoStub().add( SurveyUnitModel.builder() .campaignId(campaignId) - .questionnaireId(questionnaireId) + .collectionInstrumentId(questionnaireId) .interrogationId(interrogationId) .build() ); diff --git a/src/test/java/cucumber/functional_tests/MainDefinitions.java b/src/test/java/cucumber/functional_tests/MainDefinitions.java index f0dbb2dc..4f92755d 100644 --- a/src/test/java/cucumber/functional_tests/MainDefinitions.java +++ b/src/test/java/cucumber/functional_tests/MainDefinitions.java @@ -31,7 +31,7 @@ import fr.insee.genesis.infrastructure.utils.FileUtils; import fr.insee.genesis.stubs.ConfigStub; import fr.insee.genesis.stubs.DataProcessingContextPersistancePortStub; -import fr.insee.genesis.stubs.QuestionnaireMetadataPersistancePortStub; +import fr.insee.genesis.stubs.QuestionnaireMetadataPersistencePortStub; import fr.insee.genesis.stubs.SurveyUnitPersistencePortStub; import io.cucumber.java.After; import io.cucumber.java.Before; @@ -65,10 +65,10 @@ public class MainDefinitions { SurveyUnitQualityService surveyUnitQualityService = new SurveyUnitQualityService(); SurveyUnitPersistencePortStub surveyUnitPersistence = new SurveyUnitPersistencePortStub(); - static QuestionnaireMetadataPersistancePortStub questionnaireMetadataPersistancePortStub = - new QuestionnaireMetadataPersistancePortStub(); + static QuestionnaireMetadataPersistencePortStub questionnaireMetadataPersistencePortStub = + new QuestionnaireMetadataPersistencePortStub(); static QuestionnaireMetadataService questionnaireMetadataService = - new QuestionnaireMetadataService(questionnaireMetadataPersistancePortStub); + new QuestionnaireMetadataService(questionnaireMetadataPersistencePortStub); DataProcessingContextPersistancePortStub dataProcessingContextPersistancePortStub = new DataProcessingContextPersistancePortStub(); DataProcessingContextApiPort dataProcessingContextApiPort = new DataProcessingContextService( @@ -81,7 +81,7 @@ public class MainDefinitions { ResponseEntity surveyUnitLatestStatesResponse; ResponseController responseController = new ResponseController( - new SurveyUnitService(surveyUnitPersistence, new QuestionnaireMetadataService(questionnaireMetadataPersistancePortStub), new FileUtils(config)), + new SurveyUnitService(surveyUnitPersistence, new QuestionnaireMetadataService(questionnaireMetadataPersistencePortStub), new FileUtils(config)), surveyUnitQualityService, new FileUtils(config), new ControllerUtils(new FileUtils(config)), @@ -122,24 +122,20 @@ public void copy_data_file(String dataFile) throws IOException { @Given("We have a context in database for that data with review {string}") public void create_context(String withReviewString) { - dataProcessingContextPersistancePortStub.getMongoStub().add( - new DataProcessingContextDocument( - directory, - new ArrayList<>(), - Boolean.parseBoolean(withReviewString) - ) - ); + DataProcessingContextDocument doc = new DataProcessingContextDocument(); + doc.setPartitionId(directory); + doc.setKraftwerkExecutionScheduleList(new ArrayList<>()); + doc.setWithReview(Boolean.parseBoolean(withReviewString)); + dataProcessingContextPersistancePortStub.getMongoStub().add(doc); } @Given("We have a context in database for partitionId {string} with review {string}") public void create_context(String partitionId, String withReviewString) { - dataProcessingContextPersistancePortStub.getMongoStub().add( - new DataProcessingContextDocument( - partitionId, - new ArrayList<>(), - Boolean.parseBoolean(withReviewString) - ) - ); + DataProcessingContextDocument doc = new DataProcessingContextDocument(); + doc.setPartitionId(partitionId); + doc.setKraftwerkExecutionScheduleList(new ArrayList<>()); + doc.setWithReview(Boolean.parseBoolean(withReviewString)); + dataProcessingContextPersistancePortStub.getMongoStub().add(doc); } @Given("We have a survey unit with campaignId {string} and interrogationId {string}") @@ -163,18 +159,18 @@ public void get_models(String fileName, String ddiName) throws IOException, Pars LunaticXmlCampaign campaign; campaign = parser.parseDataFile(filePath); List questionnaireMetadataModels = - questionnaireMetadataPersistancePortStub.find(directory, Mode.WEB); + questionnaireMetadataPersistencePortStub.find(directory, Mode.WEB); if(questionnaireMetadataModels.isEmpty()){ MetadataModel metadataModel = DDIReader.getMetadataFromDDI( ddiFilePath.toFile().toURI().toURL().toString(), new FileInputStream(ddiFilePath.toFile()) ); - questionnaireMetadataPersistancePortStub.save(new QuestionnaireMetadataModel( + questionnaireMetadataPersistencePortStub.save(new QuestionnaireMetadataModel( directory, Mode.WEB, metadataModel )); - questionnaireMetadataModels = questionnaireMetadataPersistancePortStub.find(directory, Mode.WEB); + questionnaireMetadataModels = questionnaireMetadataPersistencePortStub.find(directory, Mode.WEB); } List surveyUnitModels1 = new ArrayList<>(); for (LunaticXmlSurveyUnit su : campaign.getSurveyUnits()) { @@ -210,7 +206,7 @@ public void delete_directory() throws IOException { @When("We extract survey unit data with questionnaireId {string} and interrogationId {string}") public void extract_survey_data(String questionnaireId, String interrogationId) { - this.surveyUnitModelResponse = responseController.getLatestByInterrogation(interrogationId, questionnaireId.toUpperCase()); + this.surveyUnitModelResponse = responseController.getLatestByInterrogationAndCollectionInstrument(interrogationId, questionnaireId.toUpperCase()); } @When("We extract survey unit latest states with questionnaireId {string} and interrogationId {string}") diff --git a/src/test/java/cucumber/functional_tests/RawDataDefinitions.java b/src/test/java/cucumber/functional_tests/RawDataDefinitions.java index 137ffd4a..c2005dba 100644 --- a/src/test/java/cucumber/functional_tests/RawDataDefinitions.java +++ b/src/test/java/cucumber/functional_tests/RawDataDefinitions.java @@ -1,6 +1,7 @@ package cucumber.functional_tests; import cucumber.functional_tests.config.CucumberSpringConfiguration; +import fr.insee.bpm.metadata.model.VariablesMap; import fr.insee.genesis.TestConstants; import fr.insee.genesis.configuration.Config; import fr.insee.genesis.controller.rest.responses.RawResponseController; @@ -9,19 +10,26 @@ import fr.insee.genesis.domain.model.surveyunit.Mode; import fr.insee.genesis.domain.model.surveyunit.SurveyUnitModel; import fr.insee.genesis.domain.model.surveyunit.VariableModel; +import fr.insee.genesis.domain.model.surveyunit.rawdata.DataProcessResult; +import fr.insee.genesis.domain.model.surveyunit.rawdata.RawResponse; +import fr.insee.genesis.domain.ports.api.RawResponseApiPort; import fr.insee.genesis.domain.service.context.DataProcessingContextService; import fr.insee.genesis.domain.service.metadata.QuestionnaireMetadataService; import fr.insee.genesis.domain.service.rawdata.LunaticJsonRawDataService; import fr.insee.genesis.domain.service.surveyunit.SurveyUnitQualityService; import fr.insee.genesis.domain.service.surveyunit.SurveyUnitService; import fr.insee.genesis.domain.utils.JsonUtils; +import fr.insee.genesis.exceptions.GenesisError; +import fr.insee.genesis.exceptions.GenesisException; +import fr.insee.genesis.infrastructure.repository.RawResponseInputRepository; import fr.insee.genesis.infrastructure.utils.FileUtils; import fr.insee.genesis.stubs.ConfigStub; import fr.insee.genesis.stubs.DataProcessingContextPersistancePortStub; import fr.insee.genesis.stubs.LunaticJsonRawDataPersistanceStub; -import fr.insee.genesis.stubs.QuestionnaireMetadataPersistancePortStub; +import fr.insee.genesis.stubs.QuestionnaireMetadataPersistencePortStub; import fr.insee.genesis.stubs.SurveyUnitPersistencePortStub; import fr.insee.genesis.stubs.SurveyUnitQualityToolPerretAdapterStub; +import fr.insee.modelefiliere.RawResponseDto; import io.cucumber.java.Before; import io.cucumber.java.en.Given; import io.cucumber.java.en.Then; @@ -62,12 +70,12 @@ public class RawDataDefinitions { private TestRestTemplate rest; LunaticJsonRawDataPersistanceStub lunaticJsonRawDataPersistanceStub = new LunaticJsonRawDataPersistanceStub(); - static QuestionnaireMetadataPersistancePortStub questionnaireMetadataPersistancePortStub = new QuestionnaireMetadataPersistancePortStub(); + static QuestionnaireMetadataPersistencePortStub questionnaireMetadataPersistencePortStub = new QuestionnaireMetadataPersistencePortStub(); SurveyUnitPersistencePortStub surveyUnitPersistencePortStub = new SurveyUnitPersistencePortStub(); DataProcessingContextPersistancePortStub contextStub = new DataProcessingContextPersistancePortStub(); Config config = new ConfigStub(); FileUtils fileUtils = new FileUtils(config); - SurveyUnitService surveyUnitService = new SurveyUnitService(surveyUnitPersistencePortStub, new QuestionnaireMetadataService(questionnaireMetadataPersistancePortStub), fileUtils); + SurveyUnitService surveyUnitService = new SurveyUnitService(surveyUnitPersistencePortStub, new QuestionnaireMetadataService(questionnaireMetadataPersistencePortStub), fileUtils); ControllerUtils controllerUtils = new ControllerUtils(fileUtils); SurveyUnitQualityService surveyUnitQualityService = new SurveyUnitQualityService(); DataProcessingContextPersistancePortStub dataProcessingContextPersistancePortStub = @@ -77,7 +85,7 @@ public class RawDataDefinitions { new LunaticJsonRawDataService( lunaticJsonRawDataPersistanceStub, controllerUtils, - new QuestionnaireMetadataService(questionnaireMetadataPersistancePortStub), + new QuestionnaireMetadataService(questionnaireMetadataPersistencePortStub), surveyUnitService, surveyUnitQualityService, fileUtils, @@ -88,8 +96,48 @@ public class RawDataDefinitions { config, dataProcessingContextPersistancePortStub); + // TODO : implements real stubs + RawResponseInputRepository rawResponseInputRepositoryStub = new RawResponseInputRepository(null, null) { + @Override + public void saveAsRawJson(RawResponseDto dto) { + // Do nothing - stub for test + } + }; + + RawResponseApiPort rawResponseApiPortStub = new RawResponseApiPort() { + @Override + public List getRawResponses(String questionnaireModelId, Mode mode, List interrogationIdList) { + return List.of(); + } + + @Override + public DataProcessResult processRawResponses(String questionnaireId, List interrogationIdList, List errors) throws GenesisException { + return null; + } + + @Override + public DataProcessResult processRawResponses(String collectionInstrumentId) throws GenesisException { + return null; + } + + @Override + public List convertRawResponse(List rawResponses, VariablesMap variablesMap) { + return List.of(); + } + + @Override + public List getUnprocessedCollectionInstrumentIds() { + return List.of(); + } + + @Override + public void updateProcessDates(List surveyUnitModels) { + // Do nothing - stub for test + } + }; + RawResponseController rawResponseController = new RawResponseController( - lunaticJsonRawDataService + lunaticJsonRawDataService, rawResponseApiPortStub, rawResponseInputRepositoryStub ); Path rawDataFilePath; String rawJsonData; @@ -286,9 +334,9 @@ public void check_external_variable_content_in_mongo( } - @Then("In surveyUnit {string} of the campaign {string} we must have {string} as contextualId, " + + @Then("In surveyUnit {string} of the campaign {string} we must have " + "isCapturedIndirectly to {string} and validationDate null") - public void check_optional_values(String interrogationId, String campaignId, String expectedContextualId, + public void check_optional_values(String interrogationId, String campaignId, String expectedCapturedIndirectly) { //Get SurveyUnitModel List concernedSurveyUnitModels = surveyUnitPersistencePortStub.getMongoStub().stream().filter(surveyUnitModel -> @@ -298,7 +346,6 @@ public void check_optional_values(String interrogationId, String campaignId, Str ).toList(); Assertions.assertThat(concernedSurveyUnitModels).hasSize(1); - Assertions.assertThat(concernedSurveyUnitModels.getFirst().getContextualId()).isEqualTo(expectedContextualId); Assertions.assertThat(concernedSurveyUnitModels.getFirst().getIsCapturedIndirectly()).isEqualTo(Boolean.parseBoolean(expectedCapturedIndirectly)); Assertions.assertThat(concernedSurveyUnitModels.getFirst().getValidationDate()).isNull(); } diff --git a/src/test/java/cucumber/functional_tests/config/CucumberSpringConfiguration.java b/src/test/java/cucumber/functional_tests/config/CucumberSpringConfiguration.java index 81c8ef3d..45c92bc2 100644 --- a/src/test/java/cucumber/functional_tests/config/CucumberSpringConfiguration.java +++ b/src/test/java/cucumber/functional_tests/config/CucumberSpringConfiguration.java @@ -10,7 +10,6 @@ import fr.insee.genesis.infrastructure.repository.QuestionnaireMetadataMongoDBRepository; import fr.insee.genesis.infrastructure.repository.RundeckExecutionDBRepository; import fr.insee.genesis.infrastructure.repository.SurveyUnitMongoDBRepository; -import fr.insee.genesis.infrastructure.repository.VariableTypeMongoDBRepository; import io.cucumber.spring.CucumberContextConfiguration; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.boot.autoconfigure.data.mongo.MongoDataAutoConfiguration; @@ -34,8 +33,6 @@ public class CucumberSpringConfiguration { @MockitoBean private RundeckExecutionDBRepository rundeckExecutionDBRepository; @MockitoBean - private VariableTypeMongoDBRepository variableTypeMongoDBRepository; - @MockitoBean private LunaticModelMongoDBRepository lunaticModelMongoDBRepository; @MockitoBean private LastJsonExtractionMongoDBRepository lastJsonExtractionMongoDBRepository; diff --git a/src/test/java/fr/insee/genesis/controller/adapter/LunaticXmlAdapterTest.java b/src/test/java/fr/insee/genesis/controller/adapter/LunaticXmlAdapterTest.java index bcef461d..ac62bd65 100644 --- a/src/test/java/fr/insee/genesis/controller/adapter/LunaticXmlAdapterTest.java +++ b/src/test/java/fr/insee/genesis/controller/adapter/LunaticXmlAdapterTest.java @@ -230,7 +230,7 @@ void test02() { // When List surveyUnitModels = LunaticXmlAdapter.convert(lunaticXmlSurveyUnit1, metadataModel.getVariables(), CAMPAIGN_ID, Mode.WEB); // Then - Assertions.assertThat(surveyUnitModels.getFirst().getQuestionnaireId()).isEqualTo("questionnaireId1".toUpperCase()); + Assertions.assertThat(surveyUnitModels.getFirst().getCollectionInstrumentId()).isEqualTo("questionnaireId1".toUpperCase()); } @Test diff --git a/src/test/java/fr/insee/genesis/controller/rest/ControllerAccessTest.java b/src/test/java/fr/insee/genesis/controller/rest/ControllerAccessTest.java index 1d29e9fa..7591fb83 100644 --- a/src/test/java/fr/insee/genesis/controller/rest/ControllerAccessTest.java +++ b/src/test/java/fr/insee/genesis/controller/rest/ControllerAccessTest.java @@ -2,6 +2,7 @@ import fr.insee.genesis.domain.ports.api.DataProcessingContextApiPort; import fr.insee.genesis.domain.ports.api.LunaticJsonRawDataApiPort; +import fr.insee.genesis.domain.ports.api.RawResponseApiPort; import fr.insee.genesis.domain.ports.api.SurveyUnitApiPort; import fr.insee.genesis.infrastructure.repository.ContextualExternalVariableMongoDBRepository; import fr.insee.genesis.infrastructure.repository.ContextualPreviousVariableMongoDBRepository; @@ -10,9 +11,10 @@ import fr.insee.genesis.infrastructure.repository.LunaticJsonMongoDBRepository; import fr.insee.genesis.infrastructure.repository.LunaticModelMongoDBRepository; import fr.insee.genesis.infrastructure.repository.QuestionnaireMetadataMongoDBRepository; +import fr.insee.genesis.infrastructure.repository.RawResponseInputRepository; +import fr.insee.genesis.infrastructure.repository.RawResponseRepository; import fr.insee.genesis.infrastructure.repository.RundeckExecutionDBRepository; import fr.insee.genesis.infrastructure.repository.SurveyUnitMongoDBRepository; -import fr.insee.genesis.infrastructure.repository.VariableTypeMongoDBRepository; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; @@ -77,6 +79,10 @@ class ControllerAccessTest { @MockitoBean private LunaticJsonRawDataApiPort lunaticJsonRawDataApiPort; @MockitoBean + private RawResponseApiPort rawResponseApiPort; + @MockitoBean + private RawResponseInputRepository rawRepository; + @MockitoBean private SurveyUnitMongoDBRepository surveyUnitMongoDBRepository; @MockitoBean private LastJsonExtractionMongoDBRepository lastJsonExtractionMongoDBRepository; @@ -87,16 +93,15 @@ class ControllerAccessTest { @MockitoBean private DataProcessingContextMongoDBRepository dataProcessingContextMongoDBRepository; @MockitoBean - private VariableTypeMongoDBRepository variableTypeMongoDBRepository; - @MockitoBean private LunaticModelMongoDBRepository lunaticModelMongoDBRepository; @MockitoBean private ContextualPreviousVariableMongoDBRepository contextualPreviousVariableMongoDBRepository; @MockitoBean private ContextualExternalVariableMongoDBRepository contextualExternalVariableMongoDBRepository; - @MockitoBean private QuestionnaireMetadataMongoDBRepository questionnaireMetadataMongoDBRepository; + @MockitoBean + private RawResponseRepository rawResponseRepository; /** * Provides a stream of URIs that are allowed for reader. diff --git a/src/test/java/fr/insee/genesis/controller/rest/DataProcessingContextControllerTest.java b/src/test/java/fr/insee/genesis/controller/rest/DataProcessingContextControllerTest.java index 7a0c9e19..4f08ec28 100644 --- a/src/test/java/fr/insee/genesis/controller/rest/DataProcessingContextControllerTest.java +++ b/src/test/java/fr/insee/genesis/controller/rest/DataProcessingContextControllerTest.java @@ -6,7 +6,6 @@ import fr.insee.genesis.domain.model.context.schedule.KraftwerkExecutionSchedule; import fr.insee.genesis.domain.model.context.schedule.ServiceToCall; import fr.insee.genesis.domain.service.context.DataProcessingContextService; -import fr.insee.genesis.exceptions.GenesisException; import fr.insee.genesis.infrastructure.document.context.DataProcessingContextDocument; import fr.insee.genesis.infrastructure.mappers.DataProcessingContextMapper; import fr.insee.genesis.infrastructure.utils.FileUtils; @@ -64,6 +63,15 @@ void getAllSchedulesTest() { Assertions.assertThat(response.getStatusCode().is2xxSuccessful()).isTrue(); } + @Test + void getAllSchedulesV2Test() { + //When + ResponseEntity response = dataProcessingContextController.getAllSchedulesV2(); + + //Then + Assertions.assertThat(response.getStatusCode().is2xxSuccessful()).isTrue(); + } + @Test void addScheduleWithoutEncryptionTest() { //When @@ -91,6 +99,33 @@ void addScheduleWithoutEncryptionTest() { Assertions.assertThat(dataProcessingContextDocument.getKraftwerkExecutionScheduleList().getFirst().getTrustParameters()).isNull(); } + @Test + void addScheduleWithoutEncryptionTestUsingCollectionInstrumentId() { + //When + String collectionInstrumentId = "TESTADDSURVEY_CI"; + ServiceToCall serviceToCall = ServiceToCall.MAIN; + String frequency = "0 0 6 * * *"; + LocalDateTime scheduleBeginDate = LocalDateTime.now(); + LocalDateTime scheduleEndDate = LocalDateTime.now().plusMonths(1); + + dataProcessingContextController.saveScheduleWithCollectionInstrumentId(collectionInstrumentId, serviceToCall, frequency, scheduleBeginDate, scheduleEndDate, + false, "TEST", "TEST", false); + + //Then + Assertions.assertThat(dataProcessingContextPersistancePortStub.getMongoStub()).filteredOn(dataProcessingContextDocument -> + dataProcessingContextDocument.getCollectionInstrumentId().equals(collectionInstrumentId) + ).isNotEmpty(); + + List mongoStubFiltered = dataProcessingContextPersistancePortStub.getMongoStub().stream().filter(dataProcessingContextDocument -> + dataProcessingContextDocument.getCollectionInstrumentId().equals(collectionInstrumentId)).toList(); + + DataProcessingContextDocument dataProcessingContextDocument = mongoStubFiltered.getFirst(); + + Assertions.assertThat(dataProcessingContextDocument.getKraftwerkExecutionScheduleList()).isNotEmpty(); + Assertions.assertThat(dataProcessingContextDocument.getKraftwerkExecutionScheduleList().getFirst().getFrequency()).isEqualTo(frequency); + Assertions.assertThat(dataProcessingContextDocument.getKraftwerkExecutionScheduleList().getFirst().getTrustParameters()).isNull(); + } + @Test void addScheduleWithoutEncryptionTest_nullServiceToCall() { //When @@ -119,6 +154,34 @@ void addScheduleWithoutEncryptionTest_nullServiceToCall() { Assertions.assertThat(dataProcessingContextDocument.getKraftwerkExecutionScheduleList().getFirst().getTrustParameters()).isNull(); } + @Test + void addScheduleWithoutEncryptionTest_nullServiceToCall_collectionInstrumentId() { + //When + String collectionInstrumentId = "TESTADDSURVEY_CI"; + ServiceToCall serviceToCall = null; + String frequency = "0 0 6 * * *"; + LocalDateTime scheduleBeginDate = LocalDateTime.now(); + LocalDateTime scheduleEndDate = LocalDateTime.now().plusMonths(1); + + dataProcessingContextController.saveScheduleWithCollectionInstrumentId(collectionInstrumentId, serviceToCall, frequency, scheduleBeginDate, scheduleEndDate, + false, "TEST", "TEST", false); + + //Then + Assertions.assertThat(dataProcessingContextPersistancePortStub.getMongoStub()).filteredOn(dataProcessingContextDocument -> + dataProcessingContextDocument.getCollectionInstrumentId().equals(collectionInstrumentId) + ).isNotEmpty(); + + List mongoStubFiltered = dataProcessingContextPersistancePortStub.getMongoStub().stream().filter(dataProcessingContextDocument -> + dataProcessingContextDocument.getCollectionInstrumentId().equals(collectionInstrumentId)).toList(); + + DataProcessingContextDocument dataProcessingContextDocument = mongoStubFiltered.getFirst(); + + Assertions.assertThat(dataProcessingContextDocument.getKraftwerkExecutionScheduleList()).isNotEmpty(); + Assertions.assertThat(dataProcessingContextDocument.getKraftwerkExecutionScheduleList().getFirst().getServiceToCall()).isEqualTo(ServiceToCall.MAIN); + Assertions.assertThat(dataProcessingContextDocument.getKraftwerkExecutionScheduleList().getFirst().getFrequency()).isEqualTo(frequency); + Assertions.assertThat(dataProcessingContextDocument.getKraftwerkExecutionScheduleList().getFirst().getTrustParameters()).isNull(); + } + @Test void addScheduleWithEncryptionTest() { //When @@ -157,6 +220,44 @@ void addScheduleWithEncryptionTest() { Assertions.assertThat(dataProcessingContextDocument.getKraftwerkExecutionScheduleList().getFirst().getTrustParameters().isUseSignature()).isFalse(); } + @Test + void addScheduleWithEncryptionTest_collectionInstrumentId() { + //When + String collectionInstrumentId = "TESTADDSURVEY_CI"; + ServiceToCall serviceToCall = ServiceToCall.MAIN; + String frequency = "0 0 6 * * *"; + LocalDateTime scheduleBeginDate = LocalDateTime.now(); + LocalDateTime scheduleEndDate = LocalDateTime.now().plusMonths(1); + + + dataProcessingContextController.saveScheduleWithCollectionInstrumentId(collectionInstrumentId, serviceToCall, frequency, scheduleBeginDate, scheduleEndDate, true, + "testvault/testkey", + Path.of(TestConstants.TEST_RESOURCES_DIRECTORY).resolve("OUT_ENCRYPTED").resolve(collectionInstrumentId).toString(), + false + ); + + //Then + Assertions.assertThat(dataProcessingContextPersistancePortStub.getMongoStub()).filteredOn(dataProcessingContextDocument -> + dataProcessingContextDocument.getCollectionInstrumentId().equals(collectionInstrumentId) + ).isNotEmpty(); + + List mongoStubFiltered = dataProcessingContextPersistancePortStub.getMongoStub().stream().filter(dataProcessingContextDocument -> + dataProcessingContextDocument.getCollectionInstrumentId().equals(collectionInstrumentId)).toList(); + + DataProcessingContextDocument dataProcessingContextDocument = mongoStubFiltered.getFirst(); + + Assertions.assertThat(dataProcessingContextDocument.getKraftwerkExecutionScheduleList()).isNotEmpty(); + Assertions.assertThat(dataProcessingContextDocument.getKraftwerkExecutionScheduleList().getFirst().getFrequency()).isEqualTo(frequency); + Assertions.assertThat(dataProcessingContextDocument.getKraftwerkExecutionScheduleList().getFirst().getTrustParameters()).isNotNull(); + Assertions.assertThat(dataProcessingContextDocument.getKraftwerkExecutionScheduleList().getFirst().getTrustParameters().getInputPath()).contains( + "TESTADDSURVEY"); + Assertions.assertThat(dataProcessingContextDocument.getKraftwerkExecutionScheduleList().getFirst().getTrustParameters().getOutputFolder()).contains( + "TESTADDSURVEY"); + Assertions.assertThat(dataProcessingContextDocument.getKraftwerkExecutionScheduleList().getFirst().getTrustParameters().getVaultPath()).isEqualTo( + "testvault/testkey"); + Assertions.assertThat(dataProcessingContextDocument.getKraftwerkExecutionScheduleList().getFirst().getTrustParameters().isUseSignature()).isFalse(); + } + @Test void addAdditionnalScheduleTest() { //When @@ -185,6 +286,34 @@ void addAdditionnalScheduleTest() { Assertions.assertThat(dataProcessingContextDocument.getKraftwerkExecutionScheduleList().getFirst().getFrequency()).isEqualTo(frequency); } + @Test + void addAdditionnalScheduleTest_collectionInstrumentId() { + //When + String collectionInstrumentId = "TESTADDSURVEY_CI"; //Already exists in stub + ServiceToCall serviceToCall = ServiceToCall.MAIN; + String frequency = "0 0 6 * * *"; + LocalDateTime scheduleBeginDate = LocalDateTime.now(); + LocalDateTime scheduleEndDate = LocalDateTime.now().plusMonths(1); + + + dataProcessingContextController.saveScheduleWithCollectionInstrumentId(collectionInstrumentId, serviceToCall, frequency, scheduleBeginDate, scheduleEndDate, + false, "", "", false); + + //Then + Assertions.assertThat(dataProcessingContextPersistancePortStub.getMongoStub()).filteredOn(dataProcessingContextDocument -> + dataProcessingContextDocument.getCollectionInstrumentId().equals(collectionInstrumentId) + ).isNotEmpty().hasSize(1); + + List mongoStubFiltered = dataProcessingContextPersistancePortStub.getMongoStub().stream().filter(dataProcessingContextDocument -> + dataProcessingContextDocument.getCollectionInstrumentId().equals(collectionInstrumentId)).toList(); + + DataProcessingContextDocument dataProcessingContextDocument = mongoStubFiltered.getFirst(); + Assertions.assertThat(dataProcessingContextDocument.getLastExecution()).isNull(); + + Assertions.assertThat(dataProcessingContextDocument.getKraftwerkExecutionScheduleList()).isNotEmpty(); + Assertions.assertThat(dataProcessingContextDocument.getKraftwerkExecutionScheduleList().getFirst().getFrequency()).isEqualTo(frequency); + } + @Test void addScheduleDedupTest() { //Given @@ -215,6 +344,36 @@ void addScheduleDedupTest() { Assertions.assertThat(dataProcessingContextDocument.getKraftwerkExecutionScheduleList().getFirst().getFrequency()).isEqualTo(frequency); } + @Test + void addScheduleDedupTest_collectionInstrumentId() { + //Given + addNewDocumentToStubWithCollectionInstrumentId(); + + //When + String collectionInstrumentId = "TESTSURVEY_CI"; //Already exists in stub + ServiceToCall serviceToCall = ServiceToCall.MAIN; + String frequency = "0 0 6 * * *"; + LocalDateTime scheduleBeginDate = LocalDateTime.now(); + LocalDateTime scheduleEndDate = LocalDateTime.now().plusMonths(1); + + + dataProcessingContextController.saveScheduleWithCollectionInstrumentId(collectionInstrumentId, serviceToCall, frequency, scheduleBeginDate, scheduleEndDate, + false, "", "", false); + //Then + Assertions.assertThat(dataProcessingContextPersistancePortStub.getMongoStub()).filteredOn(dataProcessingContextDocument -> + dataProcessingContextDocument.getCollectionInstrumentId().equals(collectionInstrumentId) + ).isNotEmpty().hasSize(1); + + List mongoStubFiltered = dataProcessingContextPersistancePortStub.getMongoStub().stream().filter(dataProcessingContextDocument -> + dataProcessingContextDocument.getCollectionInstrumentId().equals(collectionInstrumentId)).toList(); + + DataProcessingContextDocument dataProcessingContextDocument = mongoStubFiltered.getFirst(); + Assertions.assertThat(dataProcessingContextDocument.getLastExecution()).isNull(); + + Assertions.assertThat(dataProcessingContextDocument.getKraftwerkExecutionScheduleList()).isNotEmpty(); + Assertions.assertThat(dataProcessingContextDocument.getKraftwerkExecutionScheduleList().getFirst().getFrequency()).isEqualTo(frequency); + } + @Test void updateLastExecutionTest(){ //Given @@ -229,6 +388,20 @@ void updateLastExecutionTest(){ Assertions.assertThat(mongoStubFiltered.getFirst().getLastExecution()).isNotNull(); } + @Test + void updateLastExecutionTest_collectionInstrumentId(){ + //Given + addNewDocumentToStubWithCollectionInstrumentId(); + + //When + dataProcessingContextController.setSurveyLastExecutionByCollectionInstrumentId("TESTSURVEY_CI", LocalDateTime.now()); + + //Then + List mongoStubFiltered = dataProcessingContextPersistancePortStub.getMongoStub().stream().filter(dataProcessingContextDocument -> + dataProcessingContextDocument.getCollectionInstrumentId().equals("TESTSURVEY_CI")).toList(); + Assertions.assertThat(mongoStubFiltered.getFirst().getLastExecution()).isNotNull(); + } + @Test void setLastExecutionTestToNull(){ //Given @@ -243,6 +416,20 @@ void setLastExecutionTestToNull(){ Assertions.assertThat(mongoStubFiltered.getFirst().getLastExecution()).isNull(); } + @Test + void setLastExecutionTestToNull_collectionInstrumentId(){ + //Given + addNewDocumentToStubWithCollectionInstrumentId(); + + //When + dataProcessingContextController.setSurveyLastExecutionByCollectionInstrumentId("TESTSURVEY_CI", null); + + //Then + List mongoStubFiltered = dataProcessingContextPersistancePortStub.getMongoStub().stream().filter(dataProcessingContextDocument -> + dataProcessingContextDocument.getCollectionInstrumentId().equals("TESTSURVEY_CI")).toList(); + Assertions.assertThat(mongoStubFiltered.getFirst().getLastExecution()).isNull(); + } + @Test void setLastExecutionTest(){ //Given @@ -258,6 +445,21 @@ void setLastExecutionTest(){ Assertions.assertThat(mongoStubFiltered.getFirst().getLastExecution()).isEqualTo(date); } + @Test + void setLastExecutionTest_collectionInstrumentId(){ + //Given + LocalDateTime date = LocalDateTime.now(); + addNewDocumentToStubWithCollectionInstrumentId(); + + //When + dataProcessingContextController.setSurveyLastExecutionByCollectionInstrumentId("TESTSURVEY_CI", date); + + //Then + List mongoStubFiltered = dataProcessingContextPersistancePortStub.getMongoStub().stream().filter(dataProcessingContextDocument -> + dataProcessingContextDocument.getCollectionInstrumentId().equals("TESTSURVEY_CI")).toList(); + Assertions.assertThat(mongoStubFiltered.getFirst().getLastExecution()).isEqualTo(date); + } + @Test void wrongFrequencyTest(){ //When+Then @@ -272,6 +474,20 @@ void wrongFrequencyTest(){ Assertions.assertThat(response.getStatusCode().is4xxClientError()).isTrue(); } + @Test + void wrongFrequencyTest_collectionInstrumentId(){ + //When+Then + String collectionInstrumentId = "TESTSURVEY_CI"; //Already exists in stub + ServiceToCall serviceToCall = ServiceToCall.MAIN; + String frequency = "ERROR"; + LocalDateTime scheduleBeginDate = LocalDateTime.now(); + LocalDateTime scheduleEndDate = LocalDateTime.now().plusMonths(1); + + ResponseEntity response = dataProcessingContextController.saveScheduleWithCollectionInstrumentId(collectionInstrumentId, serviceToCall, frequency, scheduleBeginDate, scheduleEndDate, + false, "", "", false); + Assertions.assertThat(response.getStatusCode().is4xxClientError()).isTrue(); + } + @Test void notFoundTest(){ //When+Then @@ -279,6 +495,13 @@ void notFoundTest(){ Assertions.assertThat(response.getStatusCode().is4xxClientError()).isTrue(); } + @Test + void notFoundTest_collectionInstrumentId(){ + //When+Then + ResponseEntity response = dataProcessingContextController.setSurveyLastExecutionByCollectionInstrumentId("ERROR", LocalDateTime.now()); + Assertions.assertThat(response.getStatusCode().is4xxClientError()).isTrue(); + } + @Test void deleteScheduleTest(){ //When @@ -291,12 +514,24 @@ void deleteScheduleTest(){ } @Test - void deleteExpiredScheduleTest_execution() throws IOException, GenesisException { + void deleteScheduleTest_collectionInstrumentId(){ + //When + dataProcessingContextController.deleteSchedulesByCollectionInstrumentId("TESTSURVEY_CI"); + + //Then + Assertions.assertThat(dataProcessingContextPersistancePortStub.getMongoStub()).filteredOn(dataProcessingContextDocument -> + dataProcessingContextDocument.getPartitionId().equals("TESTSURVEY_CI") + ).isEmpty(); + } + + @Test + void deleteExpiredScheduleTest_execution() { //Given DataProcessingContextModel dataProcessingContextModel = new DataProcessingContextModel( null, "TESTSURVEYADDED", null, + null, new ArrayList<>(), false ); @@ -338,10 +573,59 @@ void deleteExpiredScheduleTest_execution() throws IOException, GenesisException } @Test - void deleteExpiredScheduleTest_wholeSurvey() throws IOException, GenesisException { + void deleteExpiredScheduleTest_execution_collectionInstrumentId() { //Given DataProcessingContextModel dataProcessingContextModel = new DataProcessingContextModel( null, + null, + "TESTSURVEYADDED_CI", + null, + new ArrayList<>(), + false + ); + KraftwerkExecutionSchedule kraftwerkExecutionSchedule = new KraftwerkExecutionSchedule( + "0 0 6 * * *", + ServiceToCall.MAIN, + LocalDateTime.of(2000, Month.JANUARY, 1, 1, 1, 1), + LocalDateTime.of(2000, Month.DECEMBER, 1, 1, 1, 1), + null + ); + dataProcessingContextModel.getKraftwerkExecutionScheduleList().add(kraftwerkExecutionSchedule); + kraftwerkExecutionSchedule = new KraftwerkExecutionSchedule( + "0 0 6 * * *", + ServiceToCall.MAIN, + LocalDateTime.of(2023, Month.FEBRUARY, 1, 1, 1, 1), + LocalDateTime.of(5023, Month.DECEMBER, 1, 1, 1, 1), + null + ); + dataProcessingContextModel.getKraftwerkExecutionScheduleList().add(kraftwerkExecutionSchedule); + dataProcessingContextPersistancePortStub.getMongoStub().add(DataProcessingContextMapper.INSTANCE.modelToDocument(dataProcessingContextModel)); + + //When + dataProcessingContextController.deleteExpiredSchedules(); + + //Then + //Expired schedule deleted + Assertions.assertThat(dataProcessingContextPersistancePortStub.getMongoStub()).filteredOn(dataProcessingContextDocument -> + dataProcessingContextDocument.getCollectionInstrumentId().equals("TESTSURVEYADDED_CI") + ).isNotEmpty(); + Assertions.assertThat(dataProcessingContextPersistancePortStub.getMongoStub().stream().filter(dataProcessingContextDocument -> + dataProcessingContextDocument.getCollectionInstrumentId().equals("TESTSURVEYADDED_CI")).toList().getFirst().getKraftwerkExecutionScheduleList() + ).isNotEmpty().hasSize(1); + + //Expired schedule to log json file + Assertions.assertThat(Path.of(TestConstants.TEST_RESOURCES_DIRECTORY) + .resolve(Constants.SCHEDULE_ARCHIVE_FOLDER_NAME) + .resolve("TESTSURVEYADDED_CI.json") + .toFile()).exists().content().isNotEmpty().contains("2000").doesNotContain("5023"); + } + + @Test + void deleteExpiredScheduleTest_wholeSurvey() { + //Given + DataProcessingContextModel dataProcessingContextModel = new DataProcessingContextModel( + null, + "TESTSURVEYADDED", "TESTSURVEYADDED", null, new ArrayList<>(), @@ -386,10 +670,60 @@ void deleteExpiredScheduleTest_wholeSurvey() throws IOException, GenesisExceptio } @Test - void deleteExpiredScheduleTest_appendLog() throws IOException, GenesisException { + void deleteExpiredScheduleTest_wholeSurvey_collectionInstrumentId() { //Given DataProcessingContextModel dataProcessingContextModel = new DataProcessingContextModel( null, + null, + "TESTSURVEYADDED_CI", + null, + new ArrayList<>(), + false + ); + KraftwerkExecutionSchedule kraftwerkExecutionSchedule = new KraftwerkExecutionSchedule( + "0 0 6 * * *", + ServiceToCall.MAIN, + LocalDateTime.of(2001, Month.JANUARY, 1, 1, 1, 1), + LocalDateTime.of(2001, Month.DECEMBER, 1, 1, 1, 1), + null + ); + dataProcessingContextModel.getKraftwerkExecutionScheduleList().add(kraftwerkExecutionSchedule); + kraftwerkExecutionSchedule = new KraftwerkExecutionSchedule( + "0 0 6 * * *", + ServiceToCall.MAIN, + LocalDateTime.of(2002, Month.FEBRUARY, 1, 1, 1, 1), + LocalDateTime.of(2002, Month.DECEMBER, 1, 1, 1, 1), + null + ); + dataProcessingContextModel.getKraftwerkExecutionScheduleList().add(kraftwerkExecutionSchedule); + dataProcessingContextPersistancePortStub.getMongoStub().add(DataProcessingContextMapper.INSTANCE.modelToDocument(dataProcessingContextModel)); + + //When + dataProcessingContextController.deleteExpiredSchedules(); + + //Then + //Expired schedule deleted + Assertions.assertThat(dataProcessingContextPersistancePortStub.getMongoStub()).filteredOn(dataProcessingContextDocument -> + dataProcessingContextDocument.getCollectionInstrumentId().equals("TESTSURVEYADDED_CI") + ).hasSize(1); + Assertions.assertThat(dataProcessingContextPersistancePortStub.getMongoStub().stream().filter(dataProcessingContextDocument -> + dataProcessingContextDocument.getCollectionInstrumentId().equals("TESTSURVEYADDED_CI")).toList().getFirst().getKraftwerkExecutionScheduleList() + ).isEmpty(); + + + //Expired schedule to log json file + Assertions.assertThat(Path.of(TestConstants.TEST_RESOURCES_DIRECTORY) + .resolve(Constants.SCHEDULE_ARCHIVE_FOLDER_NAME) + .resolve("TESTSURVEYADDED_CI.json") + .toFile()).exists().content().isNotEmpty().contains("2001","2002"); + } + + @Test + void deleteExpiredScheduleTest_appendLog() { + //Given + DataProcessingContextModel dataProcessingContextModel = new DataProcessingContextModel( + null, + "TESTSURVEYADDED2", "TESTSURVEYADDED2", null, new ArrayList<>(), @@ -443,18 +777,75 @@ void deleteExpiredScheduleTest_appendLog() throws IOException, GenesisException .toFile()).exists().content().isNotEmpty().contains("2000","2001"); } + @Test + void deleteExpiredScheduleTest_appendLog_collectionInstrumentId() { + //Given + DataProcessingContextModel dataProcessingContextModel = new DataProcessingContextModel( + null, + null, + "TESTSURVEYADDED2_CI", + null, + new ArrayList<>(), + false + ); + KraftwerkExecutionSchedule kraftwerkExecutionSchedule = new KraftwerkExecutionSchedule( + "0 0 6 * * *", + ServiceToCall.MAIN, + LocalDateTime.of(2000, Month.JANUARY, 1, 1, 1, 1), + LocalDateTime.of(2000, Month.DECEMBER, 1, 1, 1, 1), + null + ); + dataProcessingContextModel.getKraftwerkExecutionScheduleList().add(kraftwerkExecutionSchedule); + kraftwerkExecutionSchedule = new KraftwerkExecutionSchedule( + "0 0 6 * * *", + ServiceToCall.MAIN, + LocalDateTime.of(2023, Month.FEBRUARY, 1, 1, 1, 1), + LocalDateTime.of(5023, Month.DECEMBER, 1, 1, 1, 1), + null + ); + dataProcessingContextModel.getKraftwerkExecutionScheduleList().add(kraftwerkExecutionSchedule); + dataProcessingContextPersistancePortStub.getMongoStub().add(DataProcessingContextMapper.INSTANCE.modelToDocument(dataProcessingContextModel)); + + //When + dataProcessingContextController.deleteExpiredSchedules(); + kraftwerkExecutionSchedule = new KraftwerkExecutionSchedule( + "0 0 6 * * *", + ServiceToCall.MAIN, + LocalDateTime.of(2001, Month.FEBRUARY, 1, 1, 1, 1), + LocalDateTime.of(2001, Month.DECEMBER, 1, 1, 1, 1), + null + ); + dataProcessingContextModel.getKraftwerkExecutionScheduleList().add(kraftwerkExecutionSchedule); + dataProcessingContextPersistancePortStub.getMongoStub().add(DataProcessingContextMapper.INSTANCE.modelToDocument(dataProcessingContextModel)); + dataProcessingContextController.deleteExpiredSchedules(); + + //Then + //Expired schedules deleted + Assertions.assertThat(dataProcessingContextPersistancePortStub.getMongoStub()).filteredOn(dataProcessingContextDocument -> + dataProcessingContextDocument.getCollectionInstrumentId().equals("TESTSURVEYADDED2_CI") + ).isNotEmpty(); + Assertions.assertThat(dataProcessingContextPersistancePortStub.getMongoStub().stream().filter(dataProcessingContextDocument -> + dataProcessingContextDocument + .getCollectionInstrumentId().equals("TESTSURVEYADDED2_CI")).toList().getFirst().getKraftwerkExecutionScheduleList() + ).isNotEmpty().hasSize(1); + + //Expired schedules to only one log json file + Assertions.assertThat(Path.of(TestConstants.TEST_RESOURCES_DIRECTORY) + .resolve(Constants.SCHEDULE_ARCHIVE_FOLDER_NAME) + .resolve("TESTSURVEYADDED2_CI.json") + .toFile()).exists().content().isNotEmpty().contains("2000","2001"); + } + @ParameterizedTest @ValueSource(booleans = {false, true}) void getReview_test(boolean withReview){ //GIVEN String partitionId = "TESTPARTITION"; - dataProcessingContextPersistancePortStub.getMongoStub().add( - new DataProcessingContextDocument( - "TESTPARTITION", - new ArrayList<>(), - withReview - ) - ); + DataProcessingContextDocument doc = new DataProcessingContextDocument(); + doc.setPartitionId("TESTPARTITION"); + doc.setKraftwerkExecutionScheduleList(new ArrayList<>()); + doc.setWithReview(withReview); + dataProcessingContextPersistancePortStub.getMongoStub().add(doc); //WHEN ResponseEntity response = dataProcessingContextController.getReviewIndicator(partitionId); @@ -465,6 +856,26 @@ void getReview_test(boolean withReview){ Assertions.assertThat((Boolean) response.getBody()).isEqualTo(withReview); } + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void getReview_test_collectionInstrumentId(boolean withReview){ + //GIVEN + String collectionInstrumentId = "TESTPARTITION_CI"; + DataProcessingContextDocument doc = new DataProcessingContextDocument(); + doc.setCollectionInstrumentId("TESTPARTITION_CI"); + doc.setKraftwerkExecutionScheduleList(new ArrayList<>()); + doc.setWithReview(withReview); + dataProcessingContextPersistancePortStub.getMongoStub().add(doc); + + //WHEN + ResponseEntity response = dataProcessingContextController.getReviewIndicatorByCollectionInstrumentId(collectionInstrumentId); + + //THEN + Assertions.assertThat(response.getStatusCode().value()).isEqualTo(200); + Assertions.assertThat(response.getBody().getClass()).isEqualTo(Boolean.class); + Assertions.assertThat((Boolean) response.getBody()).isEqualTo(withReview); + } + @Test void getReview_no_context_test(){ //WHEN @@ -476,14 +887,49 @@ void getReview_no_context_test(){ Assertions.assertThat(response.getStatusCode().value()).isEqualTo(404); } + @Test + void getReview_no_context_test_collectionInstrumentId(){ + //WHEN + ResponseEntity response = dataProcessingContextController.getReviewIndicatorByCollectionInstrumentId( + "TESTPARTITIONIDNOCONTEXT_CI" + ); + + //THEN + Assertions.assertThat(response.getStatusCode().value()).isEqualTo(404); + } + //UTILITY private void addNewDocumentToStub() { - DataProcessingContextDocument dataProcessingContextDocumentTest = new DataProcessingContextDocument( - "TESTSURVEY", - new ArrayList<>(), - false + DataProcessingContextDocument dataProcessingContextDocumentTest = new DataProcessingContextDocument(); + dataProcessingContextDocumentTest.setPartitionId("TESTSURVEY"); + dataProcessingContextDocumentTest.setKraftwerkExecutionScheduleList(new ArrayList<>()); + dataProcessingContextDocumentTest.setWithReview(false); + + KraftwerkExecutionSchedule kraftwerkExecutionSchedule = new KraftwerkExecutionSchedule( + "0 0 6 * * *", + ServiceToCall.MAIN, + LocalDateTime.of(2023, Month.JANUARY, 1, 1, 1, 1), + LocalDateTime.of(2023, Month.DECEMBER, 1, 1, 1, 1), + null ); + dataProcessingContextDocumentTest.getKraftwerkExecutionScheduleList().add(kraftwerkExecutionSchedule); + kraftwerkExecutionSchedule = new KraftwerkExecutionSchedule( + "0 0 6 * * *", + ServiceToCall.MAIN, + LocalDateTime.of(2023, Month.FEBRUARY, 1, 1, 1, 1), + LocalDateTime.of(2023, Month.DECEMBER, 1, 1, 1, 1), + null + ); + dataProcessingContextDocumentTest.getKraftwerkExecutionScheduleList().add(kraftwerkExecutionSchedule); + dataProcessingContextPersistancePortStub.getMongoStub().add(dataProcessingContextDocumentTest); + } + + private void addNewDocumentToStubWithCollectionInstrumentId() { + DataProcessingContextDocument dataProcessingContextDocumentTest = new DataProcessingContextDocument(); + dataProcessingContextDocumentTest.setCollectionInstrumentId("TESTSURVEY_CI"); + dataProcessingContextDocumentTest.setKraftwerkExecutionScheduleList(new ArrayList<>()); + dataProcessingContextDocumentTest.setWithReview(false); KraftwerkExecutionSchedule kraftwerkExecutionSchedule = new KraftwerkExecutionSchedule( "0 0 6 * * *", ServiceToCall.MAIN, diff --git a/src/test/java/fr/insee/genesis/controller/rest/HealthCheckControllerTest.java b/src/test/java/fr/insee/genesis/controller/rest/HealthCheckControllerTest.java index f3a5dc57..b5ebd30a 100644 --- a/src/test/java/fr/insee/genesis/controller/rest/HealthCheckControllerTest.java +++ b/src/test/java/fr/insee/genesis/controller/rest/HealthCheckControllerTest.java @@ -11,7 +11,7 @@ import fr.insee.genesis.infrastructure.utils.FileUtils; import fr.insee.genesis.stubs.ConfigStub; import fr.insee.genesis.stubs.DataProcessingContextPersistancePortStub; -import fr.insee.genesis.stubs.QuestionnaireMetadataPersistancePortStub; +import fr.insee.genesis.stubs.QuestionnaireMetadataPersistencePortStub; import fr.insee.genesis.stubs.SurveyUnitPersistencePortStub; import lombok.extern.slf4j.Slf4j; import org.assertj.core.api.Assertions; @@ -34,7 +34,7 @@ static void init() { surveyUnitPersistencePortStub = new SurveyUnitPersistencePortStub(); SurveyUnitApiPort surveyUnitApiPort = new SurveyUnitService( surveyUnitPersistencePortStub, - new QuestionnaireMetadataService(new QuestionnaireMetadataPersistancePortStub()), + new QuestionnaireMetadataService(new QuestionnaireMetadataPersistencePortStub()), new FileUtils(new ConfigStub()) ); List externalVariableList = new ArrayList<>(); @@ -72,7 +72,7 @@ static void init() { .campaignId("TESTCAMPAIGNID") .mode(Mode.WEB) .interrogationId("TESTINTERROGATIONID") - .questionnaireId("TESTQUESTIONNAIREID") + .collectionInstrumentId("TESTQUESTIONNAIREID") .state(DataState.COLLECTED) .fileDate(LocalDateTime.of(2023, 1, 1, 0, 0 , 0)) .recordDate(LocalDateTime.of(2024, 1, 1, 0, 0, 0)) diff --git a/src/test/java/fr/insee/genesis/controller/rest/QuestionnaireMetadataControllerTest.java b/src/test/java/fr/insee/genesis/controller/rest/QuestionnaireMetadataControllerTest.java index 2756cde5..b1a18a77 100644 --- a/src/test/java/fr/insee/genesis/controller/rest/QuestionnaireMetadataControllerTest.java +++ b/src/test/java/fr/insee/genesis/controller/rest/QuestionnaireMetadataControllerTest.java @@ -6,7 +6,7 @@ import fr.insee.genesis.domain.model.surveyunit.Mode; import fr.insee.genesis.domain.service.metadata.QuestionnaireMetadataService; import fr.insee.genesis.infrastructure.document.metadata.QuestionnaireMetadataDocument; -import fr.insee.genesis.stubs.QuestionnaireMetadataPersistancePortStub; +import fr.insee.genesis.stubs.QuestionnaireMetadataPersistencePortStub; import org.assertj.core.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -14,17 +14,17 @@ class QuestionnaireMetadataControllerTest { public static final String VARIABLE_NAME = "TESTVAR"; - static QuestionnaireMetadataPersistancePortStub questionnaireMetadataPersistancePortStub - = new QuestionnaireMetadataPersistancePortStub(); + static QuestionnaireMetadataPersistencePortStub questionnaireMetadataPersistencePortStub + = new QuestionnaireMetadataPersistencePortStub(); static QuestionnaireMetadataController questionnaireMetadataController = new QuestionnaireMetadataController( - new QuestionnaireMetadataService(questionnaireMetadataPersistancePortStub) + new QuestionnaireMetadataService(questionnaireMetadataPersistencePortStub) ); @BeforeEach void clean(){ - questionnaireMetadataPersistancePortStub.getMongoStub().clear(); + questionnaireMetadataPersistencePortStub.getMongoStub().clear(); } @Test @@ -33,14 +33,15 @@ void getMetadataTest(){ String questionnaireId = "TESTQUEST"; Mode mode = Mode.WEB; String variableName = VARIABLE_NAME; - questionnaireMetadataPersistancePortStub.getMongoStub().add( + questionnaireMetadataPersistencePortStub.getMongoStub().add( new QuestionnaireMetadataDocument( questionnaireId, + null, mode, getMetadataModel(variableName) ) ); - Assertions.assertThat(questionnaireMetadataPersistancePortStub.getMongoStub()).hasSize(1); + Assertions.assertThat(questionnaireMetadataPersistencePortStub.getMongoStub()).hasSize(1); //WHEN ResponseEntity response = questionnaireMetadataController.getMetadata(questionnaireId, mode); @@ -83,16 +84,16 @@ void saveMetadataTest(){ //THEN Assertions.assertThat(response.getStatusCode().is2xxSuccessful()).isTrue(); - Assertions.assertThat(questionnaireMetadataPersistancePortStub.getMongoStub()).hasSize(1); - Assertions.assertThat(questionnaireMetadataPersistancePortStub.getMongoStub().getFirst().questionnaireId()) + Assertions.assertThat(questionnaireMetadataPersistencePortStub.getMongoStub()).hasSize(1); + Assertions.assertThat(questionnaireMetadataPersistencePortStub.getMongoStub().getFirst().collectionInstrumentId()) .isEqualTo(questionnaireId); - Assertions.assertThat(questionnaireMetadataPersistancePortStub.getMongoStub().getFirst().mode()) + Assertions.assertThat(questionnaireMetadataPersistencePortStub.getMongoStub().getFirst().mode()) .isEqualTo(mode); - Assertions.assertThat(questionnaireMetadataPersistancePortStub.getMongoStub().getFirst().metadataModel().getVariables() + Assertions.assertThat(questionnaireMetadataPersistencePortStub.getMongoStub().getFirst().metadataModel().getVariables() .hasVariable(variableName)).isTrue(); - Assertions.assertThat(questionnaireMetadataPersistancePortStub.getMongoStub().getFirst().metadataModel().getVariables() + Assertions.assertThat(questionnaireMetadataPersistencePortStub.getMongoStub().getFirst().metadataModel().getVariables() .getVariable(variableName).getGroup()).isEqualTo(metadataModel.getRootGroup()); - Assertions.assertThat(questionnaireMetadataPersistancePortStub.getMongoStub().getFirst().metadataModel().getVariables() + Assertions.assertThat(questionnaireMetadataPersistencePortStub.getMongoStub().getFirst().metadataModel().getVariables() .getVariable(variableName).getType()).isEqualTo(VariableType.STRING); } @@ -103,9 +104,10 @@ void saveMetadataTest_overwrite(){ Mode mode = Mode.WEB; String variableName = VARIABLE_NAME; MetadataModel metadataModel = getMetadataModel(variableName); - questionnaireMetadataPersistancePortStub.getMongoStub().add( + questionnaireMetadataPersistencePortStub.getMongoStub().add( new QuestionnaireMetadataDocument( questionnaireId, + null, mode, metadataModel ) @@ -125,16 +127,16 @@ void saveMetadataTest_overwrite(){ //THEN Assertions.assertThat(response.getStatusCode().is2xxSuccessful()).isTrue(); - Assertions.assertThat(questionnaireMetadataPersistancePortStub.getMongoStub()).hasSize(1); - Assertions.assertThat(questionnaireMetadataPersistancePortStub.getMongoStub().getFirst().questionnaireId()) + Assertions.assertThat(questionnaireMetadataPersistencePortStub.getMongoStub()).hasSize(1); + Assertions.assertThat(questionnaireMetadataPersistencePortStub.getMongoStub().getFirst().collectionInstrumentId()) .isEqualTo(questionnaireId); - Assertions.assertThat(questionnaireMetadataPersistancePortStub.getMongoStub().getFirst().mode()) + Assertions.assertThat(questionnaireMetadataPersistencePortStub.getMongoStub().getFirst().mode()) .isEqualTo(mode); - Assertions.assertThat(questionnaireMetadataPersistancePortStub.getMongoStub().getFirst().metadataModel().getVariables() + Assertions.assertThat(questionnaireMetadataPersistencePortStub.getMongoStub().getFirst().metadataModel().getVariables() .hasVariable(variableName)).isTrue(); - Assertions.assertThat(questionnaireMetadataPersistancePortStub.getMongoStub().getFirst().metadataModel().getVariables() + Assertions.assertThat(questionnaireMetadataPersistencePortStub.getMongoStub().getFirst().metadataModel().getVariables() .getVariable(variableName).getGroup()).isEqualTo(metadataModel.getRootGroup()); - Assertions.assertThat(questionnaireMetadataPersistancePortStub.getMongoStub().getFirst().metadataModel().getVariables() + Assertions.assertThat(questionnaireMetadataPersistencePortStub.getMongoStub().getFirst().metadataModel().getVariables() .getVariable(variableName).getType()).isEqualTo(VariableType.INTEGER); } @@ -143,20 +145,21 @@ void deleteMetadataTest(){ //GIVEN String questionnaireId = "TESTQUEST"; Mode mode = Mode.WEB; - questionnaireMetadataPersistancePortStub.getMongoStub().add( + questionnaireMetadataPersistencePortStub.getMongoStub().add( new QuestionnaireMetadataDocument( questionnaireId, + null, mode, new MetadataModel() ) ); - Assertions.assertThat(questionnaireMetadataPersistancePortStub.getMongoStub()).hasSize(1); + Assertions.assertThat(questionnaireMetadataPersistencePortStub.getMongoStub()).hasSize(1); //WHEN questionnaireMetadataController.deleteMetadata(questionnaireId, mode); //THEN - Assertions.assertThat(questionnaireMetadataPersistancePortStub.getMongoStub()).isEmpty(); + Assertions.assertThat(questionnaireMetadataPersistencePortStub.getMongoStub()).isEmpty(); } private static MetadataModel getMetadataModel(String variableName) { diff --git a/src/test/java/fr/insee/genesis/controller/rest/UtilsControllerTest.java b/src/test/java/fr/insee/genesis/controller/rest/UtilsControllerTest.java index aad43cb1..da751d0f 100644 --- a/src/test/java/fr/insee/genesis/controller/rest/UtilsControllerTest.java +++ b/src/test/java/fr/insee/genesis/controller/rest/UtilsControllerTest.java @@ -22,7 +22,7 @@ import fr.insee.genesis.stubs.ConfigStub; import fr.insee.genesis.stubs.DataProcessingContextPersistancePortStub; import fr.insee.genesis.stubs.LunaticJsonRawDataPersistanceStub; -import fr.insee.genesis.stubs.QuestionnaireMetadataPersistancePortStub; +import fr.insee.genesis.stubs.QuestionnaireMetadataPersistencePortStub; import fr.insee.genesis.stubs.SurveyUnitPersistencePortStub; import fr.insee.genesis.stubs.SurveyUnitQualityToolPerretAdapterStub; import org.assertj.core.api.Assertions; @@ -54,7 +54,7 @@ class UtilsControllerTest { static List interrogationIdList; static FileUtils fileUtils = new FileUtils(new ConfigStub()); static ControllerUtils controllerUtils = new ControllerUtils(fileUtils); - static QuestionnaireMetadataService metadataService = new QuestionnaireMetadataService(new QuestionnaireMetadataPersistancePortStub()); + static QuestionnaireMetadataService metadataService = new QuestionnaireMetadataService(new QuestionnaireMetadataPersistencePortStub()); static SurveyUnitService surveyUnitService = new SurveyUnitService(new SurveyUnitPersistencePortStub(), metadataService, fileUtils); static SurveyUnitQualityService surveyUnitQualityService = new SurveyUnitQualityService(); //Constants @@ -132,7 +132,7 @@ void reset() throws IOException { .campaignId("TEST-TABLEAUX") .mode(Mode.WEB) .interrogationId(DEFAULT_INTERROGATION_ID) - .questionnaireId(DEFAULT_QUESTIONNAIRE_ID) + .collectionInstrumentId(DEFAULT_QUESTIONNAIRE_ID) .state(DataState.COLLECTED) .fileDate(LocalDateTime.of(2023, 1, 1, 0, 0, 0)) .recordDate(LocalDateTime.of(2024, 1, 1, 0, 0, 0)) @@ -451,7 +451,7 @@ private void addAdditionalSurveyUnitModelToMongoStub(String campaignId, String q .campaignId(campaignId) .mode(Mode.WEB) .interrogationId(DEFAULT_INTERROGATION_ID) - .questionnaireId(questionnaireId) + .collectionInstrumentId(questionnaireId) .state(DataState.COLLECTED) .fileDate(LocalDateTime.of(2023, 2, 2, 0, 0, 0)) .recordDate(LocalDateTime.of(2024, 2, 2, 0, 0, 0)) diff --git a/src/test/java/fr/insee/genesis/controller/rest/responses/CampaignControllerTest.java b/src/test/java/fr/insee/genesis/controller/rest/responses/CampaignControllerTest.java index 981553ee..45b07a62 100644 --- a/src/test/java/fr/insee/genesis/controller/rest/responses/CampaignControllerTest.java +++ b/src/test/java/fr/insee/genesis/controller/rest/responses/CampaignControllerTest.java @@ -6,7 +6,7 @@ import fr.insee.genesis.domain.service.surveyunit.SurveyUnitService; import fr.insee.genesis.infrastructure.utils.FileUtils; import fr.insee.genesis.stubs.ConfigStub; -import fr.insee.genesis.stubs.QuestionnaireMetadataPersistancePortStub; +import fr.insee.genesis.stubs.QuestionnaireMetadataPersistencePortStub; import fr.insee.genesis.stubs.SurveyUnitPersistencePortStub; import org.assertj.core.api.Assertions; import org.junit.jupiter.api.BeforeAll; @@ -32,7 +32,7 @@ static void init() { surveyUnitPersistencePortStub = new SurveyUnitPersistencePortStub(); SurveyUnitApiPort surveyUnitApiPort = new SurveyUnitService( surveyUnitPersistencePortStub, - new QuestionnaireMetadataService(new QuestionnaireMetadataPersistancePortStub()), + new QuestionnaireMetadataService(new QuestionnaireMetadataPersistencePortStub()), new FileUtils(new ConfigStub()) ); diff --git a/src/test/java/fr/insee/genesis/controller/rest/responses/ContextualVariableControllerTest.java b/src/test/java/fr/insee/genesis/controller/rest/responses/ContextualVariableControllerTest.java index 78659354..df482854 100644 --- a/src/test/java/fr/insee/genesis/controller/rest/responses/ContextualVariableControllerTest.java +++ b/src/test/java/fr/insee/genesis/controller/rest/responses/ContextualVariableControllerTest.java @@ -212,7 +212,7 @@ void saveContextualVariables_previous_test( ContextualPreviousVariableDocument firstDocument = filter.getFirst(); Assertions.assertNull(firstDocument.getSourceState()); - Assertions.assertEquals(QUESTIONNAIRE_ID_PREVIOUS,firstDocument.getQuestionnaireId()); + Assertions.assertEquals(QUESTIONNAIRE_ID_PREVIOUS,firstDocument.getCollectionInstrumentId()); Assertions.assertEquals(expectedVarCount,firstDocument.getVariables().size()); @@ -306,7 +306,7 @@ void saveContextualVariables_external_test( String interrogationId, Assertions.assertEquals(1,filter.size()); ContextualExternalVariableDocument firstDoc = filter.getFirst(); - Assertions.assertEquals(QUESTIONNAIRE_ID_EXTERNAL, firstDoc.getQuestionnaireId()); + Assertions.assertEquals(QUESTIONNAIRE_ID_EXTERNAL, firstDoc.getCollectionInstrumentId()); Assertions.assertEquals(expectedVarCount, firstDoc.getVariables().size()); assertionsForDoc.accept(firstDoc); @@ -427,7 +427,7 @@ private void testOKCase(String sourceState) throws IOException { Assertions.assertNotNull(filter.getFirst().getSourceState()); Assertions.assertEquals(sourceState, filter.getFirst().getSourceState()); } - Assertions.assertEquals(QUESTIONNAIRE_ID_PREVIOUS,filter.getFirst().getQuestionnaireId()); + Assertions.assertEquals(QUESTIONNAIRE_ID_PREVIOUS,filter.getFirst().getCollectionInstrumentId()); Assertions.assertEquals(15,filter.getFirst().getVariables().size()); assertVariable(filter.getFirst(), "TEXTECOURT", ""); @@ -460,7 +460,7 @@ private void testOKCase(String sourceState) throws IOException { Assertions.assertNotNull(filter.getFirst().getSourceState()); Assertions.assertEquals(sourceState, filter.getFirst().getSourceState()); } - Assertions.assertEquals(QUESTIONNAIRE_ID_PREVIOUS,filter.getFirst().getQuestionnaireId()); + Assertions.assertEquals(QUESTIONNAIRE_ID_PREVIOUS,filter.getFirst().getCollectionInstrumentId()); Assertions.assertEquals(14,filter.getFirst().getVariables().size()); assertVariable(filter.getFirst(), "TEXTECOURT", "test previous"); @@ -519,17 +519,17 @@ void readPreviousJson_override_interrogation_id( String interrogationId, .stream().filter(doc -> doc.getInterrogationId().equals(interrogationId)).toList(); Assertions.assertEquals(1, filter.size()); ContextualPreviousVariableDocument firstDoc = filter.getFirst(); - Assertions.assertEquals(QUESTIONNAIRE_ID_PREVIOUS, firstDoc.getQuestionnaireId()); + Assertions.assertEquals(QUESTIONNAIRE_ID_PREVIOUS, firstDoc.getCollectionInstrumentId()); Assertions.assertAll(interrogationId + " metadata", - () -> Assertions.assertEquals(QUESTIONNAIRE_ID_PREVIOUS, firstDoc.getQuestionnaireId()), + () -> Assertions.assertEquals(QUESTIONNAIRE_ID_PREVIOUS, firstDoc.getCollectionInstrumentId()), () -> Assertions.assertEquals(expectedVarCount, firstDoc.getVariables().size()) ); assertionsForDoc.accept(firstDoc); Assertions.assertTrue( previousStub.getMongoStub().get(Constants.MONGODB_CONTEXTUAL_PREVIOUS_COLLECTION_NAME).stream() - .noneMatch(d -> "AUTO108".equals(d.getInterrogationId())), + .noneMatch(d -> "AUTO108".equals(d.getCollectionInstrumentId())), "AUTO108 ne devrait plus être présent après override" ); } @@ -703,7 +703,7 @@ void readExternalJson_test( String interrogationId, Assertions.assertEquals(1,filter.size()); ContextualExternalVariableDocument firstDoc = filter.getFirst(); - Assertions.assertEquals(QUESTIONNAIRE_ID_EXTERNAL, firstDoc.getQuestionnaireId()); + Assertions.assertEquals(QUESTIONNAIRE_ID_EXTERNAL, firstDoc.getCollectionInstrumentId()); Assertions.assertEquals(expectedVarCount, firstDoc.getVariables().size()); assertionsForDoc.accept(firstDoc); } @@ -793,7 +793,7 @@ void readExternalJson_override_interrogation_id( String interrogationId, externalStub.getMongoStub().get(Constants.MONGODB_CONTEXTUAL_EXTERNAL_COLLECTION_NAME) .stream().filter(doc -> doc.getInterrogationId().equals(interrogationId)).toList(); Assertions.assertEquals(1,filter.size()); - Assertions.assertEquals(QUESTIONNAIRE_ID_EXTERNAL,filter.getFirst().getQuestionnaireId()); + Assertions.assertEquals(QUESTIONNAIRE_ID_EXTERNAL,filter.getFirst().getCollectionInstrumentId()); Assertions.assertEquals(expectedVarCount,filter.getFirst().getVariables().size()); assertionsForDoc.accept(filter.getFirst()); @@ -801,7 +801,7 @@ void readExternalJson_override_interrogation_id( String interrogationId, // L'ancien AUTO208 ne doit plus être présent Assertions.assertTrue( externalStub.getMongoStub().get(Constants.MONGODB_CONTEXTUAL_EXTERNAL_COLLECTION_NAME) - .stream().noneMatch(d -> "AUTO208".equals(d.getInterrogationId())), + .stream().noneMatch(d -> "AUTO208".equals(d.getCollectionInstrumentId())), "AUTO208 ne devrait plus être présent après override" ); } diff --git a/src/test/java/fr/insee/genesis/controller/rest/responses/InterrogationControllerTest.java b/src/test/java/fr/insee/genesis/controller/rest/responses/InterrogationControllerTest.java index 323ac14e..739dbc13 100644 --- a/src/test/java/fr/insee/genesis/controller/rest/responses/InterrogationControllerTest.java +++ b/src/test/java/fr/insee/genesis/controller/rest/responses/InterrogationControllerTest.java @@ -6,7 +6,7 @@ import fr.insee.genesis.domain.service.surveyunit.SurveyUnitService; import fr.insee.genesis.infrastructure.utils.FileUtils; import fr.insee.genesis.stubs.ConfigStub; -import fr.insee.genesis.stubs.QuestionnaireMetadataPersistancePortStub; +import fr.insee.genesis.stubs.QuestionnaireMetadataPersistencePortStub; import fr.insee.genesis.stubs.SurveyUnitPersistencePortStub; import org.assertj.core.api.Assertions; import org.junit.jupiter.api.BeforeAll; @@ -30,7 +30,7 @@ static void init() { surveyUnitPersistencePortStub = new SurveyUnitPersistencePortStub(); SurveyUnitApiPort surveyUnitApiPort = new SurveyUnitService( surveyUnitPersistencePortStub, - new QuestionnaireMetadataService(new QuestionnaireMetadataPersistancePortStub()), + new QuestionnaireMetadataService(new QuestionnaireMetadataPersistencePortStub()), new FileUtils(new ConfigStub()) ); diff --git a/src/test/java/fr/insee/genesis/controller/rest/responses/ModeControllerTest.java b/src/test/java/fr/insee/genesis/controller/rest/responses/ModeControllerTest.java index 45643112..339a27ab 100644 --- a/src/test/java/fr/insee/genesis/controller/rest/responses/ModeControllerTest.java +++ b/src/test/java/fr/insee/genesis/controller/rest/responses/ModeControllerTest.java @@ -6,7 +6,7 @@ import fr.insee.genesis.domain.service.surveyunit.SurveyUnitService; import fr.insee.genesis.infrastructure.utils.FileUtils; import fr.insee.genesis.stubs.ConfigStub; -import fr.insee.genesis.stubs.QuestionnaireMetadataPersistancePortStub; +import fr.insee.genesis.stubs.QuestionnaireMetadataPersistencePortStub; import fr.insee.genesis.stubs.SurveyUnitPersistencePortStub; import org.assertj.core.api.Assertions; import org.junit.jupiter.api.BeforeAll; @@ -30,7 +30,7 @@ static void init() { surveyUnitPersistencePortStub = new SurveyUnitPersistencePortStub(); SurveyUnitApiPort surveyUnitApiPort = new SurveyUnitService( surveyUnitPersistencePortStub, - new QuestionnaireMetadataService(new QuestionnaireMetadataPersistancePortStub()), + new QuestionnaireMetadataService(new QuestionnaireMetadataPersistencePortStub()), new FileUtils(new ConfigStub()) ); diff --git a/src/test/java/fr/insee/genesis/controller/rest/responses/QuestionnaireControllerTest.java b/src/test/java/fr/insee/genesis/controller/rest/responses/QuestionnaireControllerTest.java index a93d8888..6e5e24ea 100644 --- a/src/test/java/fr/insee/genesis/controller/rest/responses/QuestionnaireControllerTest.java +++ b/src/test/java/fr/insee/genesis/controller/rest/responses/QuestionnaireControllerTest.java @@ -9,7 +9,7 @@ import fr.insee.genesis.infrastructure.utils.FileUtils; import fr.insee.genesis.stubs.ConfigStub; import fr.insee.genesis.stubs.DataProcessingContextPersistancePortStub; -import fr.insee.genesis.stubs.QuestionnaireMetadataPersistancePortStub; +import fr.insee.genesis.stubs.QuestionnaireMetadataPersistencePortStub; import fr.insee.genesis.stubs.SurveyUnitPersistencePortStub; import org.assertj.core.api.Assertions; import org.junit.jupiter.api.BeforeAll; @@ -36,7 +36,7 @@ static void init() { dataProcessingContextPersistancePortStub = new DataProcessingContextPersistancePortStub(); SurveyUnitApiPort surveyUnitApiPort = new SurveyUnitService( surveyUnitPersistencePortStub, - new QuestionnaireMetadataService(new QuestionnaireMetadataPersistancePortStub()), + new QuestionnaireMetadataService(new QuestionnaireMetadataPersistencePortStub()), new FileUtils(new ConfigStub()) ); @@ -77,26 +77,22 @@ void getQuestionnairesWithReviewTest() { Assertions.assertThat(response.getStatusCode().is2xxSuccessful()).isTrue(); Assertions.assertThat(response.getBody()).isNotNull().isEmpty(); - dataProcessingContextPersistancePortStub.getMongoStub().add( - new DataProcessingContextDocument( - DEFAULT_QUESTIONNAIRE_ID, - new ArrayList<>(), - true - ) - ); + DataProcessingContextDocument doc = new DataProcessingContextDocument(); + doc.setPartitionId(DEFAULT_QUESTIONNAIRE_ID); + doc.setKraftwerkExecutionScheduleList(new ArrayList<>()); + doc.setWithReview(true); + dataProcessingContextPersistancePortStub.getMongoStub().add(doc); response = questionnaireControllerStatic.getQuestionnairesWithReview(true); Assertions.assertThat(response.getStatusCode().is2xxSuccessful()).isTrue(); Assertions.assertThat(response.getBody()).isNotNull().isNotEmpty().containsOnly( DEFAULT_QUESTIONNAIRE_ID); - dataProcessingContextPersistancePortStub.getMongoStub().add( - new DataProcessingContextDocument( - questionnaireId, - new ArrayList<>(), - false - ) - ); + DataProcessingContextDocument doc2 = new DataProcessingContextDocument(); + doc2.setPartitionId(questionnaireId); + doc2.setKraftwerkExecutionScheduleList(new ArrayList<>()); + doc2.setWithReview(false); + dataProcessingContextPersistancePortStub.getMongoStub().add(doc2); response = questionnaireControllerStatic.getQuestionnairesWithReview(true); Assertions.assertThat(response.getStatusCode().is2xxSuccessful()).isTrue(); @@ -114,26 +110,22 @@ void getQuestionnairesWithoutReviewTest() { Assertions.assertThat(response.getStatusCode().is2xxSuccessful()).isTrue(); Assertions.assertThat(response.getBody()).isNotNull().isEmpty(); - dataProcessingContextPersistancePortStub.getMongoStub().add( - new DataProcessingContextDocument( - DEFAULT_QUESTIONNAIRE_ID, - new ArrayList<>(), - false - ) - ); + DataProcessingContextDocument doc = new DataProcessingContextDocument(); + doc.setPartitionId(DEFAULT_QUESTIONNAIRE_ID); + doc.setKraftwerkExecutionScheduleList(new ArrayList<>()); + doc.setWithReview(false); + dataProcessingContextPersistancePortStub.getMongoStub().add(doc); response = questionnaireControllerStatic.getQuestionnairesWithReview(false); Assertions.assertThat(response.getStatusCode().is2xxSuccessful()).isTrue(); Assertions.assertThat(response.getBody()).isNotNull().isNotEmpty().containsOnly( DEFAULT_QUESTIONNAIRE_ID); - dataProcessingContextPersistancePortStub.getMongoStub().add( - new DataProcessingContextDocument( - questionnaireId, - new ArrayList<>(), - true - ) - ); + DataProcessingContextDocument doc2 = new DataProcessingContextDocument(); + doc2.setPartitionId(questionnaireId); + doc2.setKraftwerkExecutionScheduleList(new ArrayList<>()); + doc2.setWithReview(true); + dataProcessingContextPersistancePortStub.getMongoStub().add(doc2); response = questionnaireControllerStatic.getQuestionnairesWithReview(false); Assertions.assertThat(response.getStatusCode().is2xxSuccessful()).isTrue(); Assertions.assertThat(response.getBody()).isNotNull().isNotEmpty().containsOnly( diff --git a/src/test/java/fr/insee/genesis/controller/rest/responses/RawResponseControllerTest.java b/src/test/java/fr/insee/genesis/controller/rest/responses/RawResponseControllerTest.java index 7d3cc2b9..cca7fa34 100644 --- a/src/test/java/fr/insee/genesis/controller/rest/responses/RawResponseControllerTest.java +++ b/src/test/java/fr/insee/genesis/controller/rest/responses/RawResponseControllerTest.java @@ -1,28 +1,37 @@ package fr.insee.genesis.controller.rest.responses; +import fr.insee.bpm.metadata.model.VariablesMap; import fr.insee.genesis.controller.dto.rawdata.LunaticJsonRawDataUnprocessedDto; import fr.insee.genesis.controller.utils.ControllerUtils; import fr.insee.genesis.domain.model.context.DataProcessingContextModel; import fr.insee.genesis.domain.model.surveyunit.DataState; import fr.insee.genesis.domain.model.surveyunit.Mode; +import fr.insee.genesis.domain.model.surveyunit.SurveyUnitModel; +import fr.insee.genesis.domain.model.surveyunit.rawdata.DataProcessResult; import fr.insee.genesis.domain.model.surveyunit.rawdata.LunaticJsonRawDataModel; +import fr.insee.genesis.domain.model.surveyunit.rawdata.RawResponse; import fr.insee.genesis.domain.ports.api.LunaticJsonRawDataApiPort; +import fr.insee.genesis.domain.ports.api.RawResponseApiPort; import fr.insee.genesis.domain.service.context.DataProcessingContextService; import fr.insee.genesis.domain.service.metadata.QuestionnaireMetadataService; import fr.insee.genesis.domain.service.rawdata.LunaticJsonRawDataService; import fr.insee.genesis.domain.service.surveyunit.SurveyUnitQualityService; import fr.insee.genesis.domain.service.surveyunit.SurveyUnitService; import fr.insee.genesis.domain.utils.JsonUtils; +import fr.insee.genesis.exceptions.GenesisError; +import fr.insee.genesis.exceptions.GenesisException; import fr.insee.genesis.infrastructure.document.rawdata.LunaticJsonRawDataDocument; import fr.insee.genesis.infrastructure.mappers.DataProcessingContextMapper; +import fr.insee.genesis.infrastructure.repository.RawResponseInputRepository; import fr.insee.genesis.infrastructure.utils.FileUtils; import fr.insee.genesis.stubs.ConfigStub; import fr.insee.genesis.stubs.DataProcessingContextPersistancePortStub; import fr.insee.genesis.stubs.LunaticJsonRawDataPersistanceStub; -import fr.insee.genesis.stubs.QuestionnaireMetadataPersistancePortStub; +import fr.insee.genesis.stubs.QuestionnaireMetadataPersistencePortStub; import fr.insee.genesis.stubs.SurveyUnitPersistencePortStub; import fr.insee.genesis.stubs.SurveyUnitQualityToolPerretAdapterStub; +import fr.insee.modelefiliere.RawResponseDto; import lombok.extern.slf4j.Slf4j; import org.assertj.core.api.Assertions; import org.junit.jupiter.api.Test; @@ -44,14 +53,14 @@ class RawResponseControllerTest { private final SurveyUnitQualityToolPerretAdapterStub surveyUnitQualityToolPerretAdapterStub = new SurveyUnitQualityToolPerretAdapterStub(); private final DataProcessingContextPersistancePortStub dataProcessingContextPersistancePortStub = new DataProcessingContextPersistancePortStub(); - private static final QuestionnaireMetadataPersistancePortStub questionnaireMetadataPersistancePortStub = - new QuestionnaireMetadataPersistancePortStub(); + private static final QuestionnaireMetadataPersistencePortStub questionnaireMetadataPersistencePortStub = + new QuestionnaireMetadataPersistencePortStub(); private final LunaticJsonRawDataApiPort lunaticJsonRawDataApiPort = new LunaticJsonRawDataService(lunaticJsonRawDataPersistanceStub, new ControllerUtils(fileUtils), - new QuestionnaireMetadataService(questionnaireMetadataPersistancePortStub), - new SurveyUnitService(surveyUnitPersistencePortStub, new QuestionnaireMetadataService(questionnaireMetadataPersistancePortStub), fileUtils), + new QuestionnaireMetadataService(questionnaireMetadataPersistencePortStub), + new SurveyUnitService(surveyUnitPersistencePortStub, new QuestionnaireMetadataService(questionnaireMetadataPersistencePortStub), fileUtils), new SurveyUnitQualityService(), fileUtils, new DataProcessingContextService(dataProcessingContextPersistancePortStub, surveyUnitPersistencePortStub), @@ -60,7 +69,47 @@ class RawResponseControllerTest { new DataProcessingContextPersistancePortStub() ); - private final RawResponseController rawResponseController = new RawResponseController(lunaticJsonRawDataApiPort); + // TODO: change this + RawResponseInputRepository rawResponseInputRepositoryStub = new RawResponseInputRepository(null, null) { + @Override + public void saveAsRawJson(RawResponseDto dto) { + // Ne rien faire — stub pour les tests + } + }; + + RawResponseApiPort rawResponseApiPortStub = new RawResponseApiPort() { + @Override + public List getRawResponses(String questionnaireModelId, Mode mode, List interrogationIdList) { + return List.of(); + } + + @Override + public DataProcessResult processRawResponses(String questionnaireId, List interrogationIdList, List errors) throws GenesisException { + return null; + } + + @Override + public DataProcessResult processRawResponses(String collectionInstrumentId) throws GenesisException { + return null; + } + + @Override + public List convertRawResponse(List rawResponses, VariablesMap variablesMap) { + return List.of(); + } + + @Override + public List getUnprocessedCollectionInstrumentIds() { + return List.of(); + } + + @Override + public void updateProcessDates(List surveyUnitModels) { + + } + }; + + private final RawResponseController rawResponseController = new RawResponseController(lunaticJsonRawDataApiPort, rawResponseApiPortStub, rawResponseInputRepositoryStub); @Test @@ -113,7 +162,7 @@ void getUnprocessedDataTest(){ addJsonRawDataDocumentToStub(campaignId, questionnaireId, interrogationId, null); //WHEN - List dtos = rawResponseController.getUnproccessedJsonRawData().getBody(); + List dtos = rawResponseController.getUnprocessedJsonRawData().getBody(); //THEN Assertions.assertThat(dtos).isNotNull().isNotEmpty().hasSize(1); @@ -132,7 +181,7 @@ void getUnprocessedDataTest_processDate_present(){ addJsonRawDataDocumentToStub(campaignId, questionnaireId, interrogationId, LocalDateTime.now()); //WHEN - List dtos = rawResponseController.getUnproccessedJsonRawData().getBody(); + List dtos = rawResponseController.getUnprocessedJsonRawData().getBody(); //THEN Assertions.assertThat(dtos).isNotNull().isEmpty(); @@ -178,10 +227,10 @@ void processJsonRawDataTest(){ Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub()).isNotNull().isNotEmpty().hasSize(1); Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getFirst()).isNotNull(); Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getFirst().getCampaignId()).isEqualTo(campaignId); - Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getFirst().getQuestionnaireId()).isNotNull().isEqualTo(questionnaireId); + Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getFirst().getCollectionInstrumentId()).isNotNull().isEqualTo(questionnaireId); Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getFirst().getMode()).isNotNull().isEqualTo(Mode.WEB); Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getFirst().getInterrogationId()).isEqualTo(interrogationId); - Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getFirst().getIdUE()).isEqualTo(idUE); + Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getFirst().getUsualSurveyUnitId()).isEqualTo(idUE); Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getFirst().getFileDate()).isNotNull(); Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getFirst().getRecordDate()).isNotNull(); Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getFirst().getCollectedVariables()).isNotNull().isNotEmpty().hasSize(1); @@ -238,10 +287,10 @@ void processJsonRawDataV2Test(){ Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub()).isNotNull().isNotEmpty().hasSize(1); Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getFirst()).isNotNull(); Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getFirst().getCampaignId()).isEqualTo(questionnaireId); - Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getFirst().getQuestionnaireId()).isNotNull().isEqualTo(questionnaireId); + Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getFirst().getCollectionInstrumentId()).isNotNull().isEqualTo(questionnaireId); Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getFirst().getMode()).isNotNull().isEqualTo(Mode.WEB); Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getFirst().getInterrogationId()).isEqualTo(interrogationId); - Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getFirst().getIdUE()).isEqualTo(idUE); + Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getFirst().getUsualSurveyUnitId()).isEqualTo(idUE); Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getFirst().getFileDate()).isNotNull(); Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getFirst().getRecordDate()).isNotNull(); Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getFirst().getCollectedVariables()).isNotNull().isNotEmpty().hasSize(1); diff --git a/src/test/java/fr/insee/genesis/controller/rest/responses/ResponseControllerTest.java b/src/test/java/fr/insee/genesis/controller/rest/responses/ResponseControllerTest.java index 0b3b8b71..0a01d0a0 100644 --- a/src/test/java/fr/insee/genesis/controller/rest/responses/ResponseControllerTest.java +++ b/src/test/java/fr/insee/genesis/controller/rest/responses/ResponseControllerTest.java @@ -26,7 +26,7 @@ import fr.insee.genesis.infrastructure.utils.FileUtils; import fr.insee.genesis.stubs.ConfigStub; import fr.insee.genesis.stubs.DataProcessingContextPersistancePortStub; -import fr.insee.genesis.stubs.QuestionnaireMetadataPersistancePortStub; +import fr.insee.genesis.stubs.QuestionnaireMetadataPersistencePortStub; import fr.insee.genesis.stubs.SurveyUnitPersistencePortStub; import org.assertj.core.api.Assertions; import org.junit.jupiter.api.BeforeAll; @@ -47,7 +47,7 @@ class ResponseControllerTest { static ResponseController responseControllerStatic; static SurveyUnitPersistencePortStub surveyUnitPersistencePortStub; static DataProcessingContextPersistancePortStub dataProcessingContextPersistancePortStub; - static QuestionnaireMetadataPersistancePortStub questionnaireMetadataPersistancePortStub; + static QuestionnaireMetadataPersistencePortStub questionnaireMetadataPersistencePortStub; static List interrogationIdList; //Constants @@ -62,13 +62,13 @@ static void init() { surveyUnitPersistencePortStub = new SurveyUnitPersistencePortStub(); dataProcessingContextPersistancePortStub = new DataProcessingContextPersistancePortStub(); - questionnaireMetadataPersistancePortStub = new QuestionnaireMetadataPersistancePortStub(); + questionnaireMetadataPersistencePortStub = new QuestionnaireMetadataPersistencePortStub(); Config config = new ConfigStub(); FileUtils fileUtils = new FileUtils(config); SurveyUnitApiPort surveyUnitApiPort = new SurveyUnitService( surveyUnitPersistencePortStub, - new QuestionnaireMetadataService(questionnaireMetadataPersistancePortStub), + new QuestionnaireMetadataService(questionnaireMetadataPersistencePortStub), fileUtils ); @@ -78,7 +78,7 @@ static void init() { , fileUtils , new ControllerUtils(fileUtils) , new AuthUtils(config) - , new QuestionnaireMetadataService(questionnaireMetadataPersistancePortStub) + , new QuestionnaireMetadataService(questionnaireMetadataPersistencePortStub) , new DataProcessingContextService(dataProcessingContextPersistancePortStub, surveyUnitPersistencePortStub) ); @@ -152,26 +152,26 @@ void saveResponsesFromAllCampaignFoldersTests(){ //Gets @Test void findResponsesByUEAndQuestionnaireTest() { - ResponseEntity> response = responseControllerStatic.findResponsesByInterrogationAndQuestionnaire(DEFAULT_INTERROGATION_ID, DEFAULT_QUESTIONNAIRE_ID); + ResponseEntity> response = responseControllerStatic.findResponsesByInterrogationAndCollectionInstrument(DEFAULT_INTERROGATION_ID, DEFAULT_QUESTIONNAIRE_ID); Assertions.assertThat(response.getStatusCode().is2xxSuccessful()).isTrue(); Assertions.assertThat(response.getBody()).isNotNull().isNotEmpty(); Assertions.assertThat(response.getBody().getFirst().getInterrogationId()).isEqualTo(DEFAULT_INTERROGATION_ID); - Assertions.assertThat(response.getBody().getFirst().getIdUE()).isEqualTo(DEFAULT_ID_UE); - Assertions.assertThat(response.getBody().getFirst().getQuestionnaireId()).isEqualTo(DEFAULT_QUESTIONNAIRE_ID); + Assertions.assertThat(response.getBody().getFirst().getUsualSurveyUnitId()).isEqualTo(DEFAULT_ID_UE); + Assertions.assertThat(response.getBody().getFirst().getCollectionInstrumentId()).isEqualTo(DEFAULT_QUESTIONNAIRE_ID); } @Test void getLatestByUETest() { Utils.addAdditionalSurveyUnitModelToMongoStub(surveyUnitPersistencePortStub); - ResponseEntity> response = responseControllerStatic.getLatestByInterrogation(DEFAULT_INTERROGATION_ID, DEFAULT_QUESTIONNAIRE_ID); + ResponseEntity> response = responseControllerStatic.getLatestByInterrogationAndCollectionInstrument(DEFAULT_INTERROGATION_ID, DEFAULT_QUESTIONNAIRE_ID); Assertions.assertThat(response.getStatusCode().is2xxSuccessful()).isTrue(); Assertions.assertThat(response.getBody()).isNotNull().isNotEmpty(); Assertions.assertThat(response.getBody().getFirst().getInterrogationId()).isEqualTo(DEFAULT_INTERROGATION_ID); - Assertions.assertThat(response.getBody().getFirst().getIdUE()).isEqualTo(DEFAULT_ID_UE); - Assertions.assertThat(response.getBody().getFirst().getQuestionnaireId()).isEqualTo(DEFAULT_QUESTIONNAIRE_ID); + Assertions.assertThat(response.getBody().getFirst().getUsualSurveyUnitId()).isEqualTo(DEFAULT_ID_UE); + Assertions.assertThat(response.getBody().getFirst().getCollectionInstrumentId()).isEqualTo(DEFAULT_QUESTIONNAIRE_ID); Assertions.assertThat(response.getBody().getFirst().getFileDate()).hasMonth(Month.FEBRUARY); } @@ -182,18 +182,18 @@ void getLatestByUEOneObjectTest() { Assertions.assertThat(response.getStatusCode().is2xxSuccessful()).isTrue(); Assertions.assertThat(response.getBody()).isNotNull(); Assertions.assertThat(response.getBody().getInterrogationId()).isEqualTo(DEFAULT_INTERROGATION_ID); - Assertions.assertThat(response.getBody().getSurveyUnitId()).isEqualTo(DEFAULT_ID_UE); - Assertions.assertThat(response.getBody().getQuestionnaireId()).isEqualTo(DEFAULT_QUESTIONNAIRE_ID); + Assertions.assertThat(response.getBody().getUsualSurveyUnitId()).isEqualTo(DEFAULT_ID_UE); + Assertions.assertThat(response.getBody().getCollectionInstrumentId()).isEqualTo(DEFAULT_QUESTIONNAIRE_ID); } @Test void getLatestForUEListTest() { - ResponseEntity> response = responseControllerStatic.getLatestForInterrogationList(DEFAULT_QUESTIONNAIRE_ID, interrogationIdList); + ResponseEntity> response = responseControllerStatic.getLatestForInterrogationListAndCollectionInstrument(DEFAULT_QUESTIONNAIRE_ID, interrogationIdList); Assertions.assertThat(response.getStatusCode().is2xxSuccessful()).isTrue(); Assertions.assertThat(response.getBody()).isNotNull().isNotEmpty(); Assertions.assertThat(response.getBody().getFirst().getInterrogationId()).isEqualTo(DEFAULT_INTERROGATION_ID); - Assertions.assertThat(response.getBody().getFirst().getSurveyUnitId()).isEqualTo(DEFAULT_ID_UE); + Assertions.assertThat(response.getBody().getFirst().getUsualSurveyUnitId()).isEqualTo(DEFAULT_ID_UE); } // Perret tests @@ -228,10 +228,11 @@ void getLatestByStatesSurveyDataTest() throws GenesisException { surveyUnitPersistencePortStub ); - dataProcessingContextPersistancePortStub.getMongoStub().add(new DataProcessingContextDocument( - "TEST-TABLEAUX", new ArrayList<>(), true - - )); + DataProcessingContextDocument doc = new DataProcessingContextDocument(); + doc.setPartitionId("TEST-TABLEAUX"); + doc.setKraftwerkExecutionScheduleList(new ArrayList<>()); + doc.setWithReview(true); + dataProcessingContextPersistancePortStub.getMongoStub().add(doc); //WHEN @@ -322,10 +323,11 @@ void getLatestByStatesSurveyDataTest_invalid_collected() throws GenesisException surveyUnitPersistencePortStub ); - dataProcessingContextPersistancePortStub.getMongoStub().add(new DataProcessingContextDocument( - "TEST-TABLEAUX", new ArrayList<>(), true - - )); + DataProcessingContextDocument doc = new DataProcessingContextDocument(); + doc.setPartitionId("TEST-TABLEAUX"); + doc.setKraftwerkExecutionScheduleList(new ArrayList<>()); + doc.setWithReview(true); + dataProcessingContextPersistancePortStub.getMongoStub().add(doc); //WHEN @@ -393,7 +395,7 @@ void saveEditedTest() { .campaignId(campaignId) .state(DataState.COLLECTED) .mode(Mode.WEB) - .questionnaireId(questionnaireId) + .collectionInstrumentId(questionnaireId) .interrogationId(DEFAULT_INTERROGATION_ID) .collectedVariables(List.of()) .build(); @@ -406,7 +408,7 @@ void saveEditedTest() { SurveyUnitModel docSaved = surveyUnitPersistencePortStub.getMongoStub().get(1); Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub()).hasSize(2); Assertions.assertThat(docSaved.getCampaignId()).isEqualTo(campaignId); - Assertions.assertThat(docSaved.getQuestionnaireId()).isEqualTo(questionnaireId); + Assertions.assertThat(docSaved.getCollectionInstrumentId()).isEqualTo(questionnaireId); Assertions.assertThat(docSaved.getMode()).isEqualTo(Mode.WEB); Assertions.assertThat(docSaved.getState()).isEqualTo(DataState.EDITED); Assertions.assertThat(docSaved.getFileDate()).isNull(); @@ -469,7 +471,7 @@ void saveEditedTest_DocumentEdited() { .campaignId(campaignId) .state(DataState.COLLECTED) .mode(Mode.WEB) - .questionnaireId(questionnaireId) + .collectionInstrumentId(questionnaireId) .interrogationId(DEFAULT_INTERROGATION_ID) .collectedVariables(List.of()) .build(); @@ -482,7 +484,7 @@ void saveEditedTest_DocumentEdited() { //EDITED document assertions Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub()).hasSize(3); Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().get(1).getCampaignId()).isEqualTo(campaignId); - Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().get(1).getQuestionnaireId()).isEqualTo(questionnaireId); + Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().get(1).getCollectionInstrumentId()).isEqualTo(questionnaireId); Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().get(1).getState()).isEqualTo(DataState.EDITED); Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().get(1).getMode()).isEqualTo(Mode.WEB); Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().get(1).getFileDate()).isNull(); @@ -544,7 +546,7 @@ void saveEditedTest_DocumentFormatted() { .campaignId(campaignId) .state(DataState.COLLECTED) .mode(Mode.WEB) - .questionnaireId(questionnaireId) + .collectionInstrumentId(questionnaireId) .interrogationId(DEFAULT_INTERROGATION_ID) .collectedVariables(List.of()) .build(); @@ -558,7 +560,7 @@ void saveEditedTest_DocumentFormatted() { //FORMATTED document assertions Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getLast().getCampaignId()).isEqualTo(campaignId); - Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getLast().getQuestionnaireId()).isEqualTo(questionnaireId); + Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getLast().getCollectionInstrumentId()).isEqualTo(questionnaireId); Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getLast().getState()).isEqualTo(DataState.FORMATTED); Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getLast().getMode()).isEqualTo(Mode.WEB); Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getLast().getFileDate()).isNull(); @@ -605,7 +607,7 @@ void saveEditedTest_No_Metadata_Error() { .campaignId(campaignId) .state(DataState.COLLECTED) .mode(Mode.WEB) - .questionnaireId(campaignId) + .collectionInstrumentId(campaignId) .interrogationId(DEFAULT_INTERROGATION_ID) .collectedVariables(List.of()) .build(); @@ -650,7 +652,7 @@ void saveTest_With_Collected_State_Error(){ .campaignId(CAMPAIGN_ID_WITH_DDI) .state(DataState.COLLECTED) .mode(Mode.WEB) - .questionnaireId(DEFAULT_QUESTIONNAIRE_ID) + .collectionInstrumentId(DEFAULT_QUESTIONNAIRE_ID) .interrogationId(DEFAULT_INTERROGATION_ID) .collectedVariables(List.of()) .build(); @@ -695,7 +697,7 @@ void saveEditedTest_int() { .campaignId(campaignId) .state(DataState.COLLECTED) .mode(Mode.WEB) - .questionnaireId(questionnaireId) + .collectionInstrumentId(questionnaireId) .interrogationId(DEFAULT_INTERROGATION_ID) .collectedVariables(List.of()) .build(); @@ -708,7 +710,7 @@ void saveEditedTest_int() { SurveyUnitModel docSaved = surveyUnitPersistencePortStub.getMongoStub().get(1); Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub()).hasSize(2); Assertions.assertThat(docSaved.getCampaignId()).isEqualTo(campaignId); - Assertions.assertThat(docSaved.getQuestionnaireId()).isEqualTo(questionnaireId); + Assertions.assertThat(docSaved.getCollectionInstrumentId()).isEqualTo(questionnaireId); Assertions.assertThat(docSaved.getMode()).isEqualTo(Mode.WEB); Assertions.assertThat(docSaved.getState()).isEqualTo(DataState.EDITED); Assertions.assertThat(docSaved.getFileDate()).isNull(); @@ -760,7 +762,7 @@ void saveEditedTest_null() { .campaignId(campaignId) .state(DataState.COLLECTED) .mode(Mode.WEB) - .questionnaireId(questionnaireId) + .collectionInstrumentId(questionnaireId) .interrogationId(DEFAULT_INTERROGATION_ID) .collectedVariables(List.of()) .build(); @@ -773,7 +775,7 @@ void saveEditedTest_null() { SurveyUnitModel docSaved = surveyUnitPersistencePortStub.getMongoStub().get(1); Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub()).hasSize(2); Assertions.assertThat(docSaved.getCampaignId()).isEqualTo(campaignId); - Assertions.assertThat(docSaved.getQuestionnaireId()).isEqualTo(questionnaireId); + Assertions.assertThat(docSaved.getCollectionInstrumentId()).isEqualTo(questionnaireId); Assertions.assertThat(docSaved.getMode()).isEqualTo(Mode.WEB); Assertions.assertThat(docSaved.getState()).isEqualTo(DataState.EDITED); Assertions.assertThat(docSaved.getFileDate()).isNull(); diff --git a/src/test/java/fr/insee/genesis/controller/rest/responses/Utils.java b/src/test/java/fr/insee/genesis/controller/rest/responses/Utils.java index 3ce6d67f..bdb914f5 100644 --- a/src/test/java/fr/insee/genesis/controller/rest/responses/Utils.java +++ b/src/test/java/fr/insee/genesis/controller/rest/responses/Utils.java @@ -221,8 +221,8 @@ static void addAdditionalSurveyUnitModelToMongoStub(String campaignId, String qu .campaignId(campaignId) .mode(Mode.WEB) .interrogationId(DEFAULT_INTERROGATION_ID) - .idUE(idUE) - .questionnaireId(questionnaireId) + .usualSurveyUnitId(idUE) + .collectionInstrumentId(questionnaireId) .state(DataState.COLLECTED) .fileDate(fileDate) .recordDate(recordDate) @@ -263,7 +263,7 @@ static void addAdditionalSurveyUnitModelToMongoStub(DataState state, .campaignId("TEST-TABLEAUX") .mode(Mode.WEB) .interrogationId(DEFAULT_INTERROGATION_ID) - .questionnaireId(DEFAULT_QUESTIONNAIRE_ID) + .collectionInstrumentId(DEFAULT_QUESTIONNAIRE_ID) .state(state) .fileDate(fileDate) .recordDate(recordDate) @@ -303,7 +303,7 @@ static void addAdditionalSurveyUnitModelToMongoStub(DataState state, .campaignId("TEST-TABLEAUX") .mode(Mode.WEB) .interrogationId(DEFAULT_INTERROGATION_ID) - .questionnaireId(DEFAULT_QUESTIONNAIRE_ID) + .collectionInstrumentId(DEFAULT_QUESTIONNAIRE_ID) .state(state) .fileDate(fileDate) .recordDate(recordDate) diff --git a/src/test/java/fr/insee/genesis/domain/service/context/DataProcessingContextServiceTest.java b/src/test/java/fr/insee/genesis/domain/service/context/DataProcessingContextServiceTest.java index 6b57c18c..c7e93a97 100644 --- a/src/test/java/fr/insee/genesis/domain/service/context/DataProcessingContextServiceTest.java +++ b/src/test/java/fr/insee/genesis/domain/service/context/DataProcessingContextServiceTest.java @@ -6,6 +6,8 @@ import fr.insee.genesis.domain.model.surveyunit.SurveyUnitModel; import fr.insee.genesis.exceptions.GenesisException; import fr.insee.genesis.infrastructure.document.context.DataProcessingContextDocument; +import fr.insee.genesis.infrastructure.utils.FileUtils; +import fr.insee.genesis.stubs.ConfigStub; import fr.insee.genesis.stubs.DataProcessingContextPersistancePortStub; import fr.insee.genesis.stubs.SurveyUnitPersistencePortStub; import org.assertj.core.api.Assertions; @@ -24,6 +26,7 @@ class DataProcessingContextServiceTest { static SurveyUnitPersistencePortStub surveyUnitPersistencePortStub; static DataProcessingContextService dataProcessingContextService; static DataProcessingContextPersistancePortStub dataProcessingContextPersistencePortStub; + FileUtils fileUtils = new FileUtils(new ConfigStub()); @BeforeAll static void init(){ @@ -46,11 +49,11 @@ void reset(){ LocalDateTime.MAX, null )); - dataProcessingContextPersistencePortStub.getMongoStub().add(new DataProcessingContextDocument( - "TEST", - kraftwerkExecutionScheduleList, - false - )); + DataProcessingContextDocument doc = new DataProcessingContextDocument(); + doc.setPartitionId("TEST"); + doc.setKraftwerkExecutionScheduleList(kraftwerkExecutionScheduleList); + doc.setWithReview(false); + dataProcessingContextPersistencePortStub.getMongoStub().add(doc); } @Test @@ -146,7 +149,7 @@ void removeExpiredSchedules_test_existing_schedule() throws GenesisException { )); //When - dataProcessingContextService.deleteExpiredSchedules("TEST"); + dataProcessingContextService.deleteExpiredSchedules(fileUtils.getLogFolder()); //Then //Execution schedule deleted @@ -167,14 +170,14 @@ void removeExpiredSchedules_test_delete_document() throws GenesisException { LocalDateTime.of(2000,1,1,1,1,1), null )); - dataProcessingContextPersistencePortStub.getMongoStub().add(new DataProcessingContextDocument( - "TEST2", - kraftwerkExecutionScheduleList, - false - )); + DataProcessingContextDocument doc = new DataProcessingContextDocument(); + doc.setPartitionId("TEST2"); + doc.setKraftwerkExecutionScheduleList(kraftwerkExecutionScheduleList); + doc.setWithReview(false); + dataProcessingContextPersistencePortStub.getMongoStub().add(doc); //When - dataProcessingContextService.deleteExpiredSchedules("TEST2"); + dataProcessingContextService.deleteExpiredSchedules(fileUtils.getLogFolder()); //Then //Survey schedule document deleted @@ -209,7 +212,7 @@ void getContext_shouldThrow500IfMultiplePartitions() { //To ensure test is portable on Unix/Linux/macOS and windows systems String normalizedMessage = ex.getMessage().replaceAll("\\r?\\n", ""); Assertions.assertThat(ex.getStatus()).isEqualTo(500); - Assertions.assertThat(normalizedMessage).isEqualTo("Multiple partitions for interrogation 00001 [CAMPAIGN2, CAMPAIGN1]"); + Assertions.assertThat(normalizedMessage).isEqualTo("Multiple partitions for interrogation 00001"); } @Test diff --git a/src/test/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataServiceTest.java b/src/test/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataServiceTest.java index 5bcd9a5c..7f41b15b 100644 --- a/src/test/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataServiceTest.java +++ b/src/test/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataServiceTest.java @@ -20,7 +20,7 @@ import fr.insee.genesis.stubs.ConfigStub; import fr.insee.genesis.stubs.DataProcessingContextPersistancePortStub; import fr.insee.genesis.stubs.LunaticJsonRawDataPersistanceStub; -import fr.insee.genesis.stubs.QuestionnaireMetadataPersistancePortStub; +import fr.insee.genesis.stubs.QuestionnaireMetadataPersistencePortStub; import fr.insee.genesis.stubs.SurveyUnitPersistencePortStub; import fr.insee.genesis.stubs.SurveyUnitQualityToolPerretAdapterStub; import org.assertj.core.api.Assertions; @@ -42,7 +42,7 @@ class LunaticJsonRawDataServiceTest { ControllerUtils controllerUtils = new ControllerUtils(fileUtils); static QuestionnaireMetadataService metadataService = - new QuestionnaireMetadataService(new QuestionnaireMetadataPersistancePortStub()); + new QuestionnaireMetadataService(new QuestionnaireMetadataPersistencePortStub()); SurveyUnitPersistencePortStub surveyUnitPersistencePortStub = new SurveyUnitPersistencePortStub(); DataProcessingContextPersistancePortStub dataProcessingContextPersistancePortStub = new DataProcessingContextPersistancePortStub(); diff --git a/src/test/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitServiceTest.java b/src/test/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitServiceTest.java index 34c7eb69..c7c8b0ab 100644 --- a/src/test/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitServiceTest.java +++ b/src/test/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitServiceTest.java @@ -9,7 +9,7 @@ import fr.insee.genesis.domain.service.metadata.QuestionnaireMetadataService; import fr.insee.genesis.infrastructure.utils.FileUtils; import fr.insee.genesis.stubs.ConfigStub; -import fr.insee.genesis.stubs.QuestionnaireMetadataPersistancePortStub; +import fr.insee.genesis.stubs.QuestionnaireMetadataPersistencePortStub; import fr.insee.genesis.stubs.SurveyUnitPersistencePortStub; import lombok.SneakyThrows; import org.assertj.core.api.Assertions; @@ -31,14 +31,14 @@ class SurveyUnitServiceTest { //Constants static final String DEFAULT_INTERROGATION_ID = "TESTINTERROGATIONID"; - static final String DEFAULT_QUESTIONNAIRE_ID = "TESTQUESTIONNAIREID"; + static final String DEFAULT_COLLECTION_INSTRUMENT_ID = "TESTCOLLECTIONINSTRUMENTID"; @BeforeAll static void init(){ surveyUnitPersistencePortStub = new SurveyUnitPersistencePortStub(); surveyUnitServiceStatic = new SurveyUnitService(surveyUnitPersistencePortStub, - new QuestionnaireMetadataService(new QuestionnaireMetadataPersistancePortStub()), + new QuestionnaireMetadataService(new QuestionnaireMetadataPersistencePortStub()), new FileUtils(new ConfigStub())); } @@ -80,7 +80,7 @@ void reset(){ .campaignId("TEST-TABLEAUX") .mode(Mode.WEB) .interrogationId(DEFAULT_INTERROGATION_ID) - .questionnaireId(DEFAULT_QUESTIONNAIRE_ID) + .collectionInstrumentId(DEFAULT_COLLECTION_INSTRUMENT_ID) .state(DataState.COLLECTED) .fileDate(LocalDateTime.of(2023,1,1,0,0,0)) .recordDate(LocalDateTime.of(2024,1,1,0,0,0)) @@ -133,7 +133,7 @@ void saveAllTest(){ .campaignId("TEST-TABLEAUX") .mode(Mode.WEB) .interrogationId("TESTINTERROGATIONID2") - .questionnaireId(DEFAULT_QUESTIONNAIRE_ID) + .collectionInstrumentId(DEFAULT_COLLECTION_INSTRUMENT_ID) .state(DataState.COLLECTED) .fileDate(LocalDateTime.of(2023,1,1,0,0,0)) .recordDate(LocalDateTime.of(2024,1,1,0,0,0)) @@ -148,7 +148,7 @@ void saveAllTest(){ surveyUnitModel.getCampaignId().equals("TEST-TABLEAUX") && surveyUnitModel.getMode().equals(Mode.WEB) && surveyUnitModel.getInterrogationId().equals("TESTINTERROGATIONID2") - && surveyUnitModel.getQuestionnaireId().equals(DEFAULT_QUESTIONNAIRE_ID) + && surveyUnitModel.getCollectionInstrumentId().equals(DEFAULT_COLLECTION_INSTRUMENT_ID) && surveyUnitModel.getState().equals(DataState.COLLECTED) && surveyUnitModel.getFileDate().equals(LocalDateTime.of(2023,1,1,0,0,0)) && surveyUnitModel.getRecordDate().equals(LocalDateTime.of(2024,1,1,0,0,0)) @@ -167,10 +167,10 @@ void saveAllTest(){ @Test void findByIdsUEAndQuestionnaireTest(){ - Assertions.assertThat(surveyUnitServiceStatic.findByIdsInterrogationAndQuestionnaire(DEFAULT_INTERROGATION_ID, DEFAULT_QUESTIONNAIRE_ID)).filteredOn( + Assertions.assertThat(surveyUnitServiceStatic.findByIdsInterrogationAndCollectionInstrument(DEFAULT_INTERROGATION_ID, DEFAULT_COLLECTION_INSTRUMENT_ID)).filteredOn( surveyUnitModel -> surveyUnitModel.getInterrogationId().equals(DEFAULT_INTERROGATION_ID) - && surveyUnitModel.getQuestionnaireId().equals(DEFAULT_QUESTIONNAIRE_ID) + && surveyUnitModel.getCollectionInstrumentId().equals(DEFAULT_COLLECTION_INSTRUMENT_ID) ).isNotEmpty(); } @@ -182,20 +182,13 @@ void findByInterrogationIdTest(){ ).isNotEmpty(); } - @Test - void findByQuestionnaireIdTest(){ - Assertions.assertThat(surveyUnitServiceStatic.findByQuestionnaireId(DEFAULT_QUESTIONNAIRE_ID)).filteredOn( - surveyUnitModel -> surveyUnitModel.getQuestionnaireId().equals(DEFAULT_QUESTIONNAIRE_ID) - ).isNotEmpty(); - } - @Test void findLatestByIdAndByModeTest(){ addAdditionnalSurveyUnitModelToMongoStub(); - Assertions.assertThat(surveyUnitServiceStatic.findLatestByIdAndByQuestionnaireId(DEFAULT_INTERROGATION_ID, DEFAULT_QUESTIONNAIRE_ID)).filteredOn( + Assertions.assertThat(surveyUnitServiceStatic.findLatestByIdAndByCollectionInstrumentId(DEFAULT_INTERROGATION_ID, DEFAULT_COLLECTION_INSTRUMENT_ID)).filteredOn( surveyUnitModel -> surveyUnitModel.getInterrogationId().equals(DEFAULT_INTERROGATION_ID) - && surveyUnitModel.getQuestionnaireId().equals(DEFAULT_QUESTIONNAIRE_ID) + && surveyUnitModel.getCollectionInstrumentId().equals(DEFAULT_COLLECTION_INSTRUMENT_ID) && surveyUnitModel.getFileDate().getMonth().equals(Month.FEBRUARY) ).isNotEmpty(); } @@ -210,9 +203,9 @@ void findResponsesByUEAndQuestionnaireTest_null_collectedVariables() { ); surveyUnitPersistencePortStub.getMongoStub().getLast().setCollectedVariables(null); - Assertions.assertThat(surveyUnitServiceStatic.findLatestByIdAndByQuestionnaireId(DEFAULT_INTERROGATION_ID, DEFAULT_QUESTIONNAIRE_ID)).filteredOn( + Assertions.assertThat(surveyUnitServiceStatic.findLatestByIdAndByCollectionInstrumentId(DEFAULT_INTERROGATION_ID, DEFAULT_COLLECTION_INSTRUMENT_ID)).filteredOn( surveyUnitModel -> surveyUnitModel.getInterrogationId().equals(DEFAULT_INTERROGATION_ID) - && surveyUnitModel.getQuestionnaireId().equals(DEFAULT_QUESTIONNAIRE_ID) + && surveyUnitModel.getCollectionInstrumentId().equals(DEFAULT_COLLECTION_INSTRUMENT_ID) && surveyUnitModel.getFileDate().getMonth().equals(Month.FEBRUARY) ).isNotEmpty(); } @@ -226,9 +219,9 @@ void findResponsesByUEAndQuestionnaireTest_null_externalVariables() { ); surveyUnitPersistencePortStub.getMongoStub().getLast().setExternalVariables(null); - Assertions.assertThat(surveyUnitServiceStatic.findLatestByIdAndByQuestionnaireId(DEFAULT_INTERROGATION_ID, DEFAULT_QUESTIONNAIRE_ID)).filteredOn( + Assertions.assertThat(surveyUnitServiceStatic.findLatestByIdAndByCollectionInstrumentId(DEFAULT_INTERROGATION_ID, DEFAULT_COLLECTION_INSTRUMENT_ID)).filteredOn( surveyUnitModel -> surveyUnitModel.getInterrogationId().equals(DEFAULT_INTERROGATION_ID) - && surveyUnitModel.getQuestionnaireId().equals(DEFAULT_QUESTIONNAIRE_ID) + && surveyUnitModel.getCollectionInstrumentId().equals(DEFAULT_COLLECTION_INSTRUMENT_ID) && surveyUnitModel.getFileDate().getMonth().equals(Month.FEBRUARY) ).isNotEmpty(); } @@ -237,7 +230,7 @@ void findResponsesByUEAndQuestionnaireTest_null_externalVariables() { void findDistinctInterrogationIdsByQuestionnaireIdTest(){ addAdditionnalSurveyUnitModelToMongoStub(); - Assertions.assertThat(surveyUnitServiceStatic.findDistinctInterrogationIdsByQuestionnaireId(DEFAULT_QUESTIONNAIRE_ID)).filteredOn( + Assertions.assertThat(surveyUnitServiceStatic.findDistinctInterrogationIdsByQuestionnaireId(DEFAULT_COLLECTION_INSTRUMENT_ID)).filteredOn( interrogationId -> interrogationId.getInterrogationId().equals(DEFAULT_INTERROGATION_ID) ).isNotEmpty().hasSize(1); } @@ -246,7 +239,7 @@ void findDistinctInterrogationIdsByQuestionnaireIdTest(){ void findDistinctInterrogationIdsByQuestionnaireIdAndDateAfterTest_no_doc_in_period(){ addAdditionnalSurveyUnitModelToMongoStub(); - Assertions.assertThat(surveyUnitServiceStatic.findDistinctInterrogationIdsByQuestionnaireIdAndDateAfter(DEFAULT_QUESTIONNAIRE_ID,LocalDateTime.of(2025,9,1,0,0,0))).filteredOn( + Assertions.assertThat(surveyUnitServiceStatic.findDistinctInterrogationIdsByQuestionnaireIdAndDateAfter(DEFAULT_COLLECTION_INSTRUMENT_ID,LocalDateTime.of(2025,9,1,0,0,0))).filteredOn( interrogationId -> interrogationId.getInterrogationId().equals(DEFAULT_INTERROGATION_ID) ).isEmpty(); } @@ -255,14 +248,14 @@ void findDistinctInterrogationIdsByQuestionnaireIdAndDateAfterTest_no_doc_in_per void findDistinctInterrogationIdsByQuestionnaireIdAndDateAfterTest_doc_in_period(){ addAdditionnalSurveyUnitModelToMongoStub(); - Assertions.assertThat(surveyUnitServiceStatic.findDistinctInterrogationIdsByQuestionnaireIdAndDateAfter(DEFAULT_QUESTIONNAIRE_ID,LocalDateTime.of(2022,1,1,0,0,0))).filteredOn( + Assertions.assertThat(surveyUnitServiceStatic.findDistinctInterrogationIdsByQuestionnaireIdAndDateAfter(DEFAULT_COLLECTION_INSTRUMENT_ID,LocalDateTime.of(2022,1,1,0,0,0))).filteredOn( interrogationId -> interrogationId.getInterrogationId().equals(DEFAULT_INTERROGATION_ID) ).isNotEmpty().hasSize(1); } @Test void findInterrogationIdsByQuestionnaireIdTest(){ - Assertions.assertThat(surveyUnitServiceStatic.findModesByQuestionnaireId(DEFAULT_QUESTIONNAIRE_ID)).filteredOn( + Assertions.assertThat(surveyUnitServiceStatic.findModesByCollectionInstrumentId(DEFAULT_COLLECTION_INSTRUMENT_ID)).filteredOn( mode -> mode.equals(Mode.WEB) ).isNotEmpty(); } @@ -270,7 +263,7 @@ void findInterrogationIdsByQuestionnaireIdTest(){ //========= OPTIMISATIONS PERFS (START) ========== @Test void findModesByQuestionnaireIdV2Test(){ - Assertions.assertThat(surveyUnitServiceStatic.findModesByQuestionnaireIdV2(DEFAULT_QUESTIONNAIRE_ID)).filteredOn( + Assertions.assertThat(surveyUnitServiceStatic.findModesByQuestionnaireIdV2(DEFAULT_COLLECTION_INSTRUMENT_ID)).filteredOn( mode -> mode.equals(Mode.WEB) ).isNotEmpty(); } @@ -320,9 +313,9 @@ void findLatestByIdAndByQuestionnaireIdPerretTest(){ //When - SurveyUnitDto surveyUnitDto = surveyUnitServiceStatic.findLatestValuesByStateByIdAndByQuestionnaireId( + SurveyUnitDto surveyUnitDto = surveyUnitServiceStatic.findLatestValuesByStateByIdAndByCollectionInstrumentId( DEFAULT_INTERROGATION_ID, - DEFAULT_QUESTIONNAIRE_ID + DEFAULT_COLLECTION_INSTRUMENT_ID ); @@ -396,9 +389,9 @@ void findLatestByIdAndByQuestionnaireIdPerretTest_null_collectedVariables(){ //When - SurveyUnitDto surveyUnitDto = surveyUnitServiceStatic.findLatestValuesByStateByIdAndByQuestionnaireId( + SurveyUnitDto surveyUnitDto = surveyUnitServiceStatic.findLatestValuesByStateByIdAndByCollectionInstrumentId( DEFAULT_INTERROGATION_ID, - DEFAULT_QUESTIONNAIRE_ID + DEFAULT_COLLECTION_INSTRUMENT_ID ); @@ -470,9 +463,9 @@ void findLatestByIdAndByQuestionnaireIdPerretTest_null_externalVariables(){ //When - SurveyUnitDto surveyUnitDto = surveyUnitServiceStatic.findLatestValuesByStateByIdAndByQuestionnaireId( + SurveyUnitDto surveyUnitDto = surveyUnitServiceStatic.findLatestValuesByStateByIdAndByCollectionInstrumentId( DEFAULT_INTERROGATION_ID, - DEFAULT_QUESTIONNAIRE_ID + DEFAULT_COLLECTION_INSTRUMENT_ID ); @@ -540,9 +533,9 @@ void findLatestValuesByStateByIdAndByQuestionnaireId_should_return_empty_values_ ); //When - SurveyUnitDto surveyUnitDto = surveyUnitServiceStatic.findLatestValuesByStateByIdAndByQuestionnaireId( + SurveyUnitDto surveyUnitDto = surveyUnitServiceStatic.findLatestValuesByStateByIdAndByCollectionInstrumentId( DEFAULT_INTERROGATION_ID, - DEFAULT_QUESTIONNAIRE_ID + DEFAULT_COLLECTION_INSTRUMENT_ID ); List variableDtos = surveyUnitDto.getCollectedVariables().stream().filter( variableDto -> variableDto.getVariableName().equals("TESTVARID") @@ -568,9 +561,9 @@ void findLatestValuesByStateByIdAndByQuestionnaireId_should_return_null_values() ); //When - SurveyUnitDto surveyUnitDto = surveyUnitServiceStatic.findLatestValuesByStateByIdAndByQuestionnaireId( + SurveyUnitDto surveyUnitDto = surveyUnitServiceStatic.findLatestValuesByStateByIdAndByCollectionInstrumentId( DEFAULT_INTERROGATION_ID, - DEFAULT_QUESTIONNAIRE_ID + DEFAULT_COLLECTION_INSTRUMENT_ID ); List variableDtos = surveyUnitDto.getCollectedVariables().stream().filter( variableDto -> variableDto.getVariableName().equals("TABLEAUTIC11") @@ -612,7 +605,7 @@ private void addAdditionnalSurveyUnitModelToMongoStub(){ .campaignId("TEST-TABLEAUX") .mode(Mode.WEB) .interrogationId(DEFAULT_INTERROGATION_ID) - .questionnaireId(DEFAULT_QUESTIONNAIRE_ID) + .collectionInstrumentId(DEFAULT_COLLECTION_INSTRUMENT_ID) .state(DataState.COLLECTED) .fileDate(LocalDateTime.of(2023,2,2,0,0,0)) .recordDate(LocalDateTime.of(2024,2,2,0,0,0)) @@ -659,7 +652,7 @@ private void addAdditionnalSurveyUnitModelToMongoStub(String questionnaireId) { .campaignId("TEST-TABLEAUX") .mode(Mode.WEB) .interrogationId(DEFAULT_INTERROGATION_ID) - .questionnaireId(questionnaireId) + .collectionInstrumentId(questionnaireId) .state(DataState.COLLECTED) .fileDate(LocalDateTime.of(2023, 2, 2, 0, 0, 0)) .recordDate(LocalDateTime.of(2024, 2, 2, 0, 0, 0)) @@ -697,7 +690,7 @@ private void addAdditionnalSurveyUnitModelToMongoStub(DataState state, .campaignId("TEST-TABLEAUX") .mode(Mode.WEB) .interrogationId(DEFAULT_INTERROGATION_ID) - .questionnaireId(DEFAULT_QUESTIONNAIRE_ID) + .collectionInstrumentId(DEFAULT_COLLECTION_INSTRUMENT_ID) .state(state) .fileDate(fileDate) .recordDate(recordDate) @@ -736,7 +729,7 @@ private void addAdditionnalSurveyUnitModelToMongoStub(DataState state, .campaignId("TEST-TABLEAUX") .mode(Mode.WEB) .interrogationId(DEFAULT_INTERROGATION_ID) - .questionnaireId(DEFAULT_QUESTIONNAIRE_ID) + .collectionInstrumentId(DEFAULT_COLLECTION_INSTRUMENT_ID) .state(state) .fileDate(fileDate) .recordDate(recordDate) diff --git a/src/test/java/fr/insee/genesis/domain/utils/DataVerifierTest.java b/src/test/java/fr/insee/genesis/domain/utils/DataVerifierTest.java index addb8aba..3024d349 100644 --- a/src/test/java/fr/insee/genesis/domain/utils/DataVerifierTest.java +++ b/src/test/java/fr/insee/genesis/domain/utils/DataVerifierTest.java @@ -60,7 +60,7 @@ void setUp() { SurveyUnitModel surveyUnit = SurveyUnitModel.builder() .interrogationId("UE1100000001") - .questionnaireId("Quest1") + .collectionInstrumentId("Quest1") .campaignId("Camp1") .state(DataState.COLLECTED) .collectedVariables(List.of(collectedVariable1, collectedVariable2)) @@ -102,7 +102,7 @@ void shouldAddFormattedSurveyUnit_WhenInvalidCollectedVariable() { .build(); SurveyUnitModel surveyUnit = SurveyUnitModel.builder() .interrogationId("UE1100000001") - .questionnaireId("Quest1") + .collectionInstrumentId("Quest1") .campaignId("Camp1") .state(DataState.COLLECTED) .collectedVariables(List.of(collectedVariable1, collectedVariable2)) @@ -140,7 +140,7 @@ void shouldAddFormattedSurveyUnit_WhenInvalidExternalVariable() { SurveyUnitModel surveyUnitWithInvalidExt = SurveyUnitModel.builder() .interrogationId("UE1100000002") - .questionnaireId("Quest1") + .collectionInstrumentId("Quest1") .campaignId("Camp1") .state(DataState.COLLECTED) .collectedVariables(List.of()) @@ -186,7 +186,7 @@ void shouldCorrectInvalidValuesInFormattedSurveyUnit() { .build(); SurveyUnitModel surveyUnit = SurveyUnitModel.builder() .interrogationId("UE1100000001") - .questionnaireId("Quest1") + .collectionInstrumentId("Quest1") .campaignId("Camp1") .state(DataState.COLLECTED) .collectedVariables(List.of(collectedVariable1, collectedVariable2, collectedVariable3)) @@ -239,7 +239,7 @@ void shouldCorrectInvalidIterationOnFormattedSurveyUnit() { .build(); SurveyUnitModel surveyUnit = SurveyUnitModel.builder() .interrogationId("UE1100000001") - .questionnaireId("Quest1") + .collectionInstrumentId("Quest1") .campaignId("Camp1") .state(DataState.COLLECTED) .collectedVariables(List.of(collectedVariable1, collectedVariable2, collectedVariable3, collectedVariable4)) @@ -287,7 +287,7 @@ void shouldCreateFormattedIfNull(DataState dataState) { .build(); SurveyUnitModel surveyUnit = SurveyUnitModel.builder() .interrogationId("UE1100000001") - .questionnaireId("Quest1") + .collectionInstrumentId("Quest1") .campaignId("Camp1") .state(dataState) .collectedVariables(List.of(collectedVariable1)) @@ -340,7 +340,7 @@ void shouldNotCreateFormattedIfNull(DataState dataState) { .build(); SurveyUnitModel surveyUnit = SurveyUnitModel.builder() .interrogationId("UE1100000001") - .questionnaireId("Quest1") + .collectionInstrumentId("Quest1") .campaignId("Camp1") .state(dataState) .collectedVariables(List.of(collectedVariable1)) diff --git a/src/test/java/fr/insee/genesis/infrastructure/mapper/SurveyUnitDocumentMapperImplTest.java b/src/test/java/fr/insee/genesis/infrastructure/mapper/SurveyUnitDocumentMapperImplTest.java index b5500369..238c9103 100644 --- a/src/test/java/fr/insee/genesis/infrastructure/mapper/SurveyUnitDocumentMapperImplTest.java +++ b/src/test/java/fr/insee/genesis/infrastructure/mapper/SurveyUnitDocumentMapperImplTest.java @@ -25,6 +25,8 @@ class SurveyUnitDocumentMapperImplTest { static SurveyUnitModel surveyUnitStatic; + // TODO : make different tests for document to model transformation, for old documents and new ones + @BeforeAll static void init(){ surveyUnitDocumentMapperImplStatic = new SurveyUnitDocumentMapperImpl(); @@ -71,7 +73,7 @@ static void init(){ .campaignId("TESTCAMPAIGNID") .mode(Mode.WEB) .interrogationId("TESTINTERROGATIONID") - .questionnaireId("TESTQUESTIONNAIREID") + .collectionInstrumentId("TESTQUESTIONNAIREID") .state(DataState.COLLECTED) .fileDate(LocalDateTime.of(2023,1,1,0,0,0)) .recordDate(LocalDateTime.of(2024,1,1,0,0,0)) @@ -99,7 +101,7 @@ void shouldReturnModelFromDocument(){ Assertions.assertThat(surveyUnit.getCampaignId()).isEqualTo("TESTCAMPAIGNID"); Assertions.assertThat(surveyUnit.getMode()).isEqualTo(Mode.WEB); Assertions.assertThat(surveyUnit.getInterrogationId()).isEqualTo("TESTINTERROGATIONID"); - Assertions.assertThat(surveyUnit.getQuestionnaireId()).isEqualTo("TESTQUESTIONNAIREID"); + Assertions.assertThat(surveyUnit.getCollectionInstrumentId()).isEqualTo("TESTQUESTIONNAIREID"); Assertions.assertThat(surveyUnit.getState()).isEqualTo(DataState.COLLECTED); Assertions.assertThat(surveyUnit.getFileDate()).isEqualTo(LocalDateTime.of(2023,1,1,0,0,0)); @@ -123,7 +125,7 @@ void shouldReturnDocumentFromModel(){ Assertions.assertThat(surveyUnitDocument.getCampaignId()).isEqualTo("TESTCAMPAIGNID"); Assertions.assertThat(surveyUnitDocument.getMode()).isEqualTo("WEB"); Assertions.assertThat(surveyUnitDocument.getInterrogationId()).isEqualTo("TESTINTERROGATIONID"); - Assertions.assertThat(surveyUnitDocument.getQuestionnaireId()).isEqualTo("TESTQUESTIONNAIREID"); + Assertions.assertThat(surveyUnitDocument.getCollectionInstrumentId()).isEqualTo("TESTQUESTIONNAIREID"); Assertions.assertThat(surveyUnitDocument.getState()).isEqualTo("COLLECTED"); Assertions.assertThat(surveyUnitDocument.getFileDate()).isEqualTo(LocalDateTime.of(2023,1,1,0,0,0)); @@ -151,7 +153,7 @@ void shouldReturnModelListFromDocumentList(){ Assertions.assertThat(surveyUnitList.getFirst().getCampaignId()).isEqualTo("TESTCAMPAIGNID"); Assertions.assertThat(surveyUnitList.getFirst().getMode()).isEqualTo(Mode.WEB); Assertions.assertThat(surveyUnitList.getFirst().getInterrogationId()).isEqualTo("TESTINTERROGATIONID"); - Assertions.assertThat(surveyUnitList.getFirst().getQuestionnaireId()).isEqualTo("TESTQUESTIONNAIREID"); + Assertions.assertThat(surveyUnitList.getFirst().getCollectionInstrumentId()).isEqualTo("TESTQUESTIONNAIREID"); Assertions.assertThat(surveyUnitList.getFirst().getState()).isEqualTo(DataState.COLLECTED); Assertions.assertThat(surveyUnitList.getFirst().getFileDate()).isEqualTo(LocalDateTime.of(2023,1,1,0,0,0)); @@ -177,7 +179,7 @@ void shouldReturnDocumentListFromModelList(){ Assertions.assertThat(surveyUnitDocumentList.getFirst().getCampaignId()).isEqualTo("TESTCAMPAIGNID"); Assertions.assertThat(surveyUnitDocumentList.getFirst().getMode()).isEqualTo("WEB"); Assertions.assertThat(surveyUnitDocumentList.getFirst().getInterrogationId()).isEqualTo("TESTINTERROGATIONID"); - Assertions.assertThat(surveyUnitDocumentList.getFirst().getQuestionnaireId()).isEqualTo("TESTQUESTIONNAIREID"); + Assertions.assertThat(surveyUnitDocumentList.getFirst().getCollectionInstrumentId()).isEqualTo("TESTQUESTIONNAIREID"); Assertions.assertThat(surveyUnitDocumentList.getFirst().getState()).isEqualTo("COLLECTED"); Assertions.assertThat(surveyUnitDocumentList.getFirst().getFileDate()).isEqualTo(LocalDateTime.of(2023,1,1,0,0,0)); diff --git a/src/test/java/fr/insee/genesis/infrastructure/utils/context/ContextDedupUtilsTest.java b/src/test/java/fr/insee/genesis/infrastructure/utils/context/ContextDedupUtilsTest.java index e62ecf0a..daabe6c7 100644 --- a/src/test/java/fr/insee/genesis/infrastructure/utils/context/ContextDedupUtilsTest.java +++ b/src/test/java/fr/insee/genesis/infrastructure/utils/context/ContextDedupUtilsTest.java @@ -31,11 +31,11 @@ void emptyListTest() { void oneElementListTest() { //Given List dataProcessingContextDocuments = new ArrayList<>(); - DataProcessingContextDocument existingDocument = new DataProcessingContextDocument( - partitionId, - new ArrayList<>(), - false - ); + DataProcessingContextDocument existingDocument = new DataProcessingContextDocument(); + existingDocument.setPartitionId(partitionId); + existingDocument.setKraftwerkExecutionScheduleList(new ArrayList<>()); + existingDocument.setWithReview(false); + dataProcessingContextDocuments.add(existingDocument); existingDocument.getKraftwerkExecutionScheduleList().add( new KraftwerkExecutionSchedule( @@ -61,11 +61,11 @@ void oneElementListTest() { void multipleElementsListTest_both() { //Given List dataProcessingContextDocuments = new ArrayList<>(); - DataProcessingContextDocument existingDocument = new DataProcessingContextDocument( - partitionId, - new ArrayList<>(), - false - ); + DataProcessingContextDocument existingDocument = new DataProcessingContextDocument(); + existingDocument.setPartitionId(partitionId); + existingDocument.setKraftwerkExecutionScheduleList(new ArrayList<>()); + existingDocument.setWithReview(false); + existingDocument.getKraftwerkExecutionScheduleList().add( new KraftwerkExecutionSchedule( "0 0 0 * * *", @@ -77,11 +77,11 @@ void multipleElementsListTest_both() { ); dataProcessingContextDocuments.add(existingDocument); - existingDocument = new DataProcessingContextDocument( - partitionId, - new ArrayList<>(), - true - ); + existingDocument = new DataProcessingContextDocument(); + existingDocument.setPartitionId(partitionId); + existingDocument.setKraftwerkExecutionScheduleList(new ArrayList<>()); + existingDocument.setWithReview(true); + existingDocument.getKraftwerkExecutionScheduleList().add( new KraftwerkExecutionSchedule( "0 0 6 * * *", @@ -109,11 +109,11 @@ void multipleElementsListTest_both() { void multipleElementsListTest_true() { //Given List dataProcessingContextDocuments = new ArrayList<>(); - DataProcessingContextDocument existingDocument = new DataProcessingContextDocument( - partitionId, - new ArrayList<>(), - true - ); + DataProcessingContextDocument existingDocument = new DataProcessingContextDocument(); + existingDocument.setPartitionId(partitionId); + existingDocument.setKraftwerkExecutionScheduleList(new ArrayList<>()); + existingDocument.setWithReview(true); + existingDocument.getKraftwerkExecutionScheduleList().add( new KraftwerkExecutionSchedule( "0 0 0 * * *", @@ -125,11 +125,11 @@ void multipleElementsListTest_true() { ); dataProcessingContextDocuments.add(existingDocument); - existingDocument = new DataProcessingContextDocument( - partitionId, - new ArrayList<>(), - true - ); + existingDocument = new DataProcessingContextDocument(); + existingDocument.setPartitionId(partitionId); + existingDocument.setKraftwerkExecutionScheduleList(new ArrayList<>()); + existingDocument.setWithReview(true); + existingDocument.getKraftwerkExecutionScheduleList().add( new KraftwerkExecutionSchedule( "0 0 6 * * *", @@ -157,11 +157,11 @@ void multipleElementsListTest_true() { void duplicateScheduleListTest() { //Given List dataProcessingContextDocuments = new ArrayList<>(); - DataProcessingContextDocument existingDocument = new DataProcessingContextDocument( - partitionId, - new ArrayList<>(), - false - ); + DataProcessingContextDocument existingDocument = new DataProcessingContextDocument(); + existingDocument.setPartitionId(partitionId); + existingDocument.setKraftwerkExecutionScheduleList(new ArrayList<>()); + existingDocument.setWithReview(false); + existingDocument.getKraftwerkExecutionScheduleList().add( new KraftwerkExecutionSchedule( "0 0 0 * * *", @@ -182,11 +182,11 @@ void duplicateScheduleListTest() { ); dataProcessingContextDocuments.add(existingDocument); - existingDocument = new DataProcessingContextDocument( - partitionId, - new ArrayList<>(), - false - ); + existingDocument = new DataProcessingContextDocument(); + existingDocument.setPartitionId(partitionId); + existingDocument.setKraftwerkExecutionScheduleList(new ArrayList<>()); + existingDocument.setWithReview(false); + existingDocument.getKraftwerkExecutionScheduleList().add( new KraftwerkExecutionSchedule( "0 0 0 * * *", @@ -212,11 +212,11 @@ void duplicateScheduleListTest() { void duplicateAllContextsTest() { //Given List dataProcessingContextDocuments = new ArrayList<>(); - DataProcessingContextDocument existingDocument = new DataProcessingContextDocument( - partitionId, - new ArrayList<>(), - false - ); + DataProcessingContextDocument existingDocument = new DataProcessingContextDocument(); + existingDocument.setPartitionId(partitionId); + existingDocument.setKraftwerkExecutionScheduleList(new ArrayList<>()); + existingDocument.setWithReview(false); + existingDocument.getKraftwerkExecutionScheduleList().add( new KraftwerkExecutionSchedule( "0 0 0 * * *", @@ -246,11 +246,11 @@ void duplicateAllContextsTest() { ); dataProcessingContextDocuments.add(existingDocument); - existingDocument = new DataProcessingContextDocument( - partitionId, - new ArrayList<>(), - false - ); + existingDocument = new DataProcessingContextDocument(); + existingDocument.setPartitionId(partitionId); + existingDocument.setKraftwerkExecutionScheduleList(new ArrayList<>()); + existingDocument.setWithReview(false); + existingDocument.getKraftwerkExecutionScheduleList().add( new KraftwerkExecutionSchedule( "0 0 0 * * *", @@ -262,11 +262,11 @@ void duplicateAllContextsTest() { ); dataProcessingContextDocuments.add(existingDocument); - existingDocument = new DataProcessingContextDocument( - partitionId + "_2", - new ArrayList<>(), - false - ); + existingDocument = new DataProcessingContextDocument(); + existingDocument.setPartitionId(partitionId + "_2"); + existingDocument.setKraftwerkExecutionScheduleList(new ArrayList<>()); + existingDocument.setWithReview(false); + existingDocument.getKraftwerkExecutionScheduleList().add( new KraftwerkExecutionSchedule( "0 0 0 * * *", @@ -286,11 +286,11 @@ void duplicateAllContextsTest() { ) ); dataProcessingContextDocuments.add(existingDocument); - existingDocument = new DataProcessingContextDocument( - partitionId + "_3", - new ArrayList<>(), - false - ); + existingDocument = new DataProcessingContextDocument(); + existingDocument.setPartitionId(partitionId + "_3"); + existingDocument.setKraftwerkExecutionScheduleList(new ArrayList<>()); + existingDocument.setWithReview(false); + existingDocument.getKraftwerkExecutionScheduleList().add( new KraftwerkExecutionSchedule( "0 0 6 * * *", @@ -301,11 +301,11 @@ void duplicateAllContextsTest() { ) ); dataProcessingContextDocuments.add(existingDocument); - existingDocument = new DataProcessingContextDocument( - partitionId + "_4", - new ArrayList<>(), - false - ); + existingDocument = new DataProcessingContextDocument(); + existingDocument.setPartitionId(partitionId + "_4"); + existingDocument.setKraftwerkExecutionScheduleList(new ArrayList<>()); + existingDocument.setWithReview(false); + dataProcessingContextDocuments.add(existingDocument); diff --git a/src/test/java/fr/insee/genesis/stubs/ContextualExternalVariablePersistancePortStub.java b/src/test/java/fr/insee/genesis/stubs/ContextualExternalVariablePersistancePortStub.java index 6f5cdd2a..7e31e527 100644 --- a/src/test/java/fr/insee/genesis/stubs/ContextualExternalVariablePersistancePortStub.java +++ b/src/test/java/fr/insee/genesis/stubs/ContextualExternalVariablePersistancePortStub.java @@ -52,12 +52,13 @@ public void saveAll(List contextualExternalVari @Override public void delete(String questionnaireId) { mongoStub.get(Constants.MONGODB_CONTEXTUAL_EXTERNAL_COLLECTION_NAME).removeIf( - contextualExternalVariableDocument -> contextualExternalVariableDocument.getQuestionnaireId().equals(questionnaireId) + extDoc -> (extDoc.getQuestionnaireId() != null && extDoc.getQuestionnaireId().equals(questionnaireId)) || + (extDoc.getCollectionInstrumentId() != null && extDoc.getCollectionInstrumentId().equals(questionnaireId)) ); } @Override - public ContextualExternalVariableModel findByQuestionnaireIdAndInterrogationId(String questionnaireId, String interrogationId) { + public ContextualExternalVariableModel findByCollectionInstrumentIdAndInterrogationId(String questionnaireId, String interrogationId) { List contextualExternalVariableDocumentList = mongoStub.get(Constants.MONGODB_CONTEXTUAL_EXTERNAL_COLLECTION_NAME).stream().filter( contextualExternalVariableDocument -> contextualExternalVariableDocument.getQuestionnaireId().equals(questionnaireId) diff --git a/src/test/java/fr/insee/genesis/stubs/ContextualPreviousVariablePersistancePortStub.java b/src/test/java/fr/insee/genesis/stubs/ContextualPreviousVariablePersistancePortStub.java index 255c6a9b..3fa59bbd 100644 --- a/src/test/java/fr/insee/genesis/stubs/ContextualPreviousVariablePersistancePortStub.java +++ b/src/test/java/fr/insee/genesis/stubs/ContextualPreviousVariablePersistancePortStub.java @@ -52,12 +52,15 @@ public void saveAll(List contextualPreviousVari @Override public void delete(String questionnaireId) { mongoStub.get(Constants.MONGODB_CONTEXTUAL_PREVIOUS_COLLECTION_NAME).removeIf( - contextualPreviousVariableDocument -> contextualPreviousVariableDocument.getQuestionnaireId().equals(questionnaireId) + previousDoc -> + (previousDoc.getQuestionnaireId()!=null && previousDoc.getQuestionnaireId().equals(questionnaireId)) || + (previousDoc.getCollectionInstrumentId() != null && previousDoc.getCollectionInstrumentId().equals(questionnaireId)) + ); } @Override - public ContextualPreviousVariableModel findByQuestionnaireIdAndInterrogationId(String questionnaireId, String interrogationId) { + public ContextualPreviousVariableModel findByCollectionInstrumentIdAndInterrogationId(String questionnaireId, String interrogationId) { List contextualPreviousVariableDocumentList = mongoStub.get(Constants.MONGODB_CONTEXTUAL_PREVIOUS_COLLECTION_NAME).stream().filter( contextualPreviousVariableDocument -> diff --git a/src/test/java/fr/insee/genesis/stubs/DataProcessingContextPersistancePortStub.java b/src/test/java/fr/insee/genesis/stubs/DataProcessingContextPersistancePortStub.java index 77a93995..528e4a33 100644 --- a/src/test/java/fr/insee/genesis/stubs/DataProcessingContextPersistancePortStub.java +++ b/src/test/java/fr/insee/genesis/stubs/DataProcessingContextPersistancePortStub.java @@ -38,10 +38,31 @@ public List findByPartitionIds(List partitio ); } + @Override + public DataProcessingContextModel findByCollectionInstrumentId(String collectionInstrumentId) { + return DataProcessingContextMapper.INSTANCE.documentToModel(ContextDedupUtils.deduplicateContexts(collectionInstrumentId, + mongoStub.stream().filter( + dataProcessingContextDocument -> dataProcessingContextDocument.getCollectionInstrumentId().equals(collectionInstrumentId) + ).toList()) + ); + } + + @Override + public List findByCollectionInstrumentIds(List collectionInstrumentIds) { + return DataProcessingContextMapper.INSTANCE.listDocumentToListModel( + ContextDedupUtils.deduplicateContexts( + mongoStub.stream().filter( + dataProcessingContextDocument -> collectionInstrumentIds.contains(dataProcessingContextDocument.getCollectionInstrumentId()) + ).toList() + ) + ); + } + @Override public void save(DataProcessingContextDocument dataProcessingContextDocument) { - mongoStub.removeIf(existingDoc -> existingDoc.getPartitionId().equals( - dataProcessingContextDocument.getPartitionId()) + mongoStub.removeIf(existingDoc -> (existingDoc.getCollectionInstrumentId() != null &&existingDoc.getCollectionInstrumentId().equals( + dataProcessingContextDocument.getCollectionInstrumentId())) ||(existingDoc.getPartitionId()!= null &&existingDoc.getPartitionId().equals( + dataProcessingContextDocument.getPartitionId())) ); mongoStub.add(dataProcessingContextDocument); } @@ -78,7 +99,7 @@ public List removeExpiredSchedules(DataProcessingCon kraftwerkExecutionScheduleToRemove.getScheduleEndDate()); } //Update mongo stub - mongoStub.removeIf(scheduleDocument -> scheduleDocument.getPartitionId().equals(dataProcessingContextModel.getPartitionId())); + mongoStub.removeIf(scheduleDocument -> (scheduleDocument.getCollectionInstrumentId()!=null && scheduleDocument.getCollectionInstrumentId().equals(dataProcessingContextModel.getCollectionInstrumentId())) ||(scheduleDocument.getPartitionId()!=null && scheduleDocument.getPartitionId().equals(dataProcessingContextModel.getPartitionId()))); mongoStub.add(DataProcessingContextMapper.INSTANCE.modelToDocument(dataProcessingContextModel)); return kraftwerkExecutionSchedulesToRemove; } diff --git a/src/test/java/fr/insee/genesis/stubs/LastJsonExtractionPersistencePortStub.java b/src/test/java/fr/insee/genesis/stubs/LastJsonExtractionPersistencePortStub.java index 15a94c46..defb61a0 100644 --- a/src/test/java/fr/insee/genesis/stubs/LastJsonExtractionPersistencePortStub.java +++ b/src/test/java/fr/insee/genesis/stubs/LastJsonExtractionPersistencePortStub.java @@ -21,9 +21,9 @@ public void save(LastJsonExtractionModel extraction) { } @Override - public LastJsonExtractionModel getLastExecutionDate(String questionnaireModelId, Mode mode) throws GenesisException { + public LastJsonExtractionModel getLastExecutionDate(String collectionInstrumentId, Mode mode) throws GenesisException { LastJsonExtractionDocument extraction = mongoStub.stream() - .filter(doc -> doc.getId().equals(String.format("%s_%s",questionnaireModelId,mode))) + .filter(doc -> doc.getId().equals(String.format("%s_%s",collectionInstrumentId,mode))) .findFirst() .orElseThrow(); return LastJsonExtractionDocumentMapper.INSTANCE.documentToModel(extraction); diff --git a/src/test/java/fr/insee/genesis/stubs/LunaticModelPersistanceStub.java b/src/test/java/fr/insee/genesis/stubs/LunaticModelPersistanceStub.java index 5beaebcb..8b0ca13d 100644 --- a/src/test/java/fr/insee/genesis/stubs/LunaticModelPersistanceStub.java +++ b/src/test/java/fr/insee/genesis/stubs/LunaticModelPersistanceStub.java @@ -16,14 +16,15 @@ public class LunaticModelPersistanceStub implements LunaticModelPersistancePort @Override public void save(LunaticModelModel lunaticModelModel) { //Replace - mongoStub.removeIf(document -> document.questionnaireId().equals(lunaticModelModel.questionnaireId())); + mongoStub.removeIf(document -> document.questionnaireId().equals(lunaticModelModel.collectionInstrumentId())); + mongoStub.removeIf(document -> document.collectionInstrumentId().equals(lunaticModelModel.collectionInstrumentId())); mongoStub.add(LunaticModelMapper.INSTANCE.modelToDocument(lunaticModelModel)); } @Override - public List find(String questionnaireId) { + public List find(String collectionInstrumentId) { return mongoStub.stream().filter( - document -> document.questionnaireId().equals(questionnaireId) + document -> (document.questionnaireId().equals(collectionInstrumentId) || document.collectionInstrumentId().equals(collectionInstrumentId)) ).toList(); } } diff --git a/src/test/java/fr/insee/genesis/stubs/QuestionnaireMetadataPersistancePortStub.java b/src/test/java/fr/insee/genesis/stubs/QuestionnaireMetadataPersistencePortStub.java similarity index 50% rename from src/test/java/fr/insee/genesis/stubs/QuestionnaireMetadataPersistancePortStub.java rename to src/test/java/fr/insee/genesis/stubs/QuestionnaireMetadataPersistencePortStub.java index d9e4e0e3..accf752f 100644 --- a/src/test/java/fr/insee/genesis/stubs/QuestionnaireMetadataPersistancePortStub.java +++ b/src/test/java/fr/insee/genesis/stubs/QuestionnaireMetadataPersistencePortStub.java @@ -2,7 +2,7 @@ import fr.insee.genesis.domain.model.metadata.QuestionnaireMetadataModel; import fr.insee.genesis.domain.model.surveyunit.Mode; -import fr.insee.genesis.domain.ports.spi.QuestionnaireMetadataPersistancePort; +import fr.insee.genesis.domain.ports.spi.QuestionnaireMetadataPersistencePort; import fr.insee.genesis.infrastructure.document.metadata.QuestionnaireMetadataDocument; import fr.insee.genesis.infrastructure.mappers.QuestionnaireMetadataDocumentMapper; import lombok.Getter; @@ -11,27 +11,30 @@ import java.util.List; @Getter -public class QuestionnaireMetadataPersistancePortStub implements QuestionnaireMetadataPersistancePort { +public class QuestionnaireMetadataPersistencePortStub implements QuestionnaireMetadataPersistencePort { List mongoStub = new ArrayList<>(); @Override - public List find(String questionnaireId, Mode mode) { + public List find(String collectionInstrumentId, Mode mode) { return QuestionnaireMetadataDocumentMapper.INSTANCE.listDocumentToListModel(mongoStub.stream().filter( - questionnaireMetadataDocument -> questionnaireMetadataDocument.questionnaireId().equals(questionnaireId) + questionnaireMetadataDocument -> (questionnaireMetadataDocument.questionnaireId()!= null && questionnaireMetadataDocument.questionnaireId().equals(collectionInstrumentId) || questionnaireMetadataDocument.collectionInstrumentId()!= null && questionnaireMetadataDocument.collectionInstrumentId().equals(collectionInstrumentId)) && questionnaireMetadataDocument.mode().equals(mode)) .toList()); } @Override public void save(QuestionnaireMetadataModel questionnaireMetadataModel) { - remove(questionnaireMetadataModel.questionnaireId(), questionnaireMetadataModel.mode()); + remove(questionnaireMetadataModel.collectionInstrumentId(), questionnaireMetadataModel.mode()); mongoStub.add(QuestionnaireMetadataDocumentMapper.INSTANCE.modelToDocument(questionnaireMetadataModel)); } @Override - public void remove(String questionnaireId, Mode mode) { - mongoStub.removeIf(questionnaireMetadataDocument -> questionnaireMetadataDocument.questionnaireId().equals(questionnaireId) + public void remove(String collectionInstrumentId, Mode mode) { + mongoStub.removeIf(questionnaireMetadataDocument -> questionnaireMetadataDocument.collectionInstrumentId()!= null && questionnaireMetadataDocument.collectionInstrumentId().equals(collectionInstrumentId) && questionnaireMetadataDocument.mode().equals(mode) ); + mongoStub.removeIf(questionnaireMetadataDocument -> questionnaireMetadataDocument.questionnaireId()!= null && questionnaireMetadataDocument.questionnaireId().equals(collectionInstrumentId) + && questionnaireMetadataDocument.mode().equals(mode) + ); } } diff --git a/src/test/java/fr/insee/genesis/stubs/SurveyUnitPersistencePortStub.java b/src/test/java/fr/insee/genesis/stubs/SurveyUnitPersistencePortStub.java index a0be834b..0dfeda2c 100644 --- a/src/test/java/fr/insee/genesis/stubs/SurveyUnitPersistencePortStub.java +++ b/src/test/java/fr/insee/genesis/stubs/SurveyUnitPersistencePortStub.java @@ -21,10 +21,10 @@ public void saveAll(List suList) { } @Override - public List findByIds(String interrogationId, String questionnaireId) { + public List findByIds(String interrogationId, String collectionInstrumentId) { List surveyUnitModelList = new ArrayList<>(); for(SurveyUnitModel SurveyUnitModel : mongoStub){ - if(SurveyUnitModel.getInterrogationId().equals(interrogationId) && SurveyUnitModel.getQuestionnaireId().equals(questionnaireId)) + if(SurveyUnitModel.getInterrogationId().equals(interrogationId) && SurveyUnitModel.getCollectionInstrumentId().equals(collectionInstrumentId)) surveyUnitModelList.add(SurveyUnitModel); } @@ -57,7 +57,7 @@ public List findByInterrogationIdsAndQuestionnaireId(List surveyUnitModelList = new ArrayList<>(); for(SurveyUnitModel surveyUnitModel : interrogationIds) { for (SurveyUnitModel document : mongoStub) { - if (surveyUnitModel.getInterrogationId().equals(document.getInterrogationId()) && document.getQuestionnaireId().equals(questionnaireId)) + if (surveyUnitModel.getInterrogationId().equals(document.getInterrogationId()) && document.getCollectionInstrumentId().equals(questionnaireId)) surveyUnitModelList.add(document); } } @@ -69,7 +69,7 @@ public List findByInterrogationIdsAndQuestionnaireId(List findByQuestionnaireId(String questionnaireId) { List surveyUnitModelList = new ArrayList<>(); for(SurveyUnitModel SurveyUnitModel : mongoStub){ - if(SurveyUnitModel.getQuestionnaireId().equals(questionnaireId)) + if(SurveyUnitModel.getCollectionInstrumentId().equals(questionnaireId)) surveyUnitModelList.add(SurveyUnitModel); } @@ -77,21 +77,20 @@ public Stream findByQuestionnaireId(String questionnaireId) { } @Override - public List findInterrogationIdsByQuestionnaireId(String questionnaireId) { + public List findInterrogationIdsByCollectionInstrumentId(String collectionInstrumentId) { List surveyUnitModelList = new ArrayList<>(); for(SurveyUnitModel SurveyUnitModel : mongoStub){ - if(SurveyUnitModel.getQuestionnaireId().equals(questionnaireId)) + if(SurveyUnitModel.getCollectionInstrumentId().equals(collectionInstrumentId)) surveyUnitModelList.add( new SurveyUnitModel(SurveyUnitModel.getInterrogationId(), SurveyUnitModel.getMode()) ); } - return surveyUnitModelList; } @Override public List findModesByQuestionnaireIdV2(String questionnaireId) { - return findInterrogationIdsByQuestionnaireId(questionnaireId); + return findInterrogationIdsByCollectionInstrumentId(questionnaireId); } @Override @@ -115,7 +114,7 @@ public List findModesByCampaignIdV2(String campaignId) { public List findInterrogationIdsByQuestionnaireIdAndDateAfter(String questionnaireId, LocalDateTime since) { List surveyUnitModelList = new ArrayList<>(); for(SurveyUnitModel surveyUnitModel : mongoStub){ - if(surveyUnitModel.getQuestionnaireId().equals(questionnaireId) && surveyUnitModel.getRecordDate().isAfter(since)) + if(surveyUnitModel.getCollectionInstrumentId().equals(questionnaireId) && surveyUnitModel.getRecordDate().isAfter(since)) surveyUnitModelList.add( new SurveyUnitModel(surveyUnitModel.getInterrogationId(), surveyUnitModel.getMode()) ); @@ -140,17 +139,11 @@ public List findPageableInterrogationIdsByQuestionnaireId(Strin return List.of(); } - - - //======= OPTIMISATIONS PERFS (END) ========= - - - @Override - public Long deleteByQuestionnaireId(String questionnaireId) { - return null; + public Long deleteByCollectionInstrumentId(String collectionInstrumentId) { + return (long) mongoStub.stream().filter(su -> !su.getCollectionInstrumentId().equals(collectionInstrumentId)).toList().size(); } @Override @@ -163,7 +156,7 @@ public Set findQuestionnaireIdsByCampaignId(String campaignId) { Set questionnaireIdSet = new HashSet<>(); for(SurveyUnitModel SurveyUnitModel : mongoStub){ if(SurveyUnitModel.getCampaignId().equals(campaignId)) - questionnaireIdSet.add(SurveyUnitModel.getQuestionnaireId()); + questionnaireIdSet.add(SurveyUnitModel.getCollectionInstrumentId()); } return questionnaireIdSet; @@ -196,7 +189,7 @@ public long countByCampaignId(String campaignId) { public Set findDistinctQuestionnaireIds() { Set questionnaireIds = new HashSet<>(); for(SurveyUnitModel SurveyUnitModel : mongoStub){ - questionnaireIds.add(SurveyUnitModel.getQuestionnaireId()); + questionnaireIds.add(SurveyUnitModel.getCollectionInstrumentId()); } return questionnaireIds; } @@ -205,7 +198,7 @@ public Set findDistinctQuestionnaireIds() { public Set findCampaignIdsByQuestionnaireId(String questionnaireId) { Set campaignIdSet = new HashSet<>(); for(SurveyUnitModel SurveyUnitModel : mongoStub){ - if(SurveyUnitModel.getQuestionnaireId().equals(questionnaireId)) + if(SurveyUnitModel.getCollectionInstrumentId().equals(questionnaireId)) campaignIdSet.add(SurveyUnitModel.getCampaignId()); } diff --git a/src/test/java/fr/insee/genesis/stubs/VariableTypePersistanceStub.java b/src/test/java/fr/insee/genesis/stubs/VariableTypePersistanceStub.java deleted file mode 100644 index e15220b4..00000000 --- a/src/test/java/fr/insee/genesis/stubs/VariableTypePersistanceStub.java +++ /dev/null @@ -1,30 +0,0 @@ -package fr.insee.genesis.stubs; - -import fr.insee.genesis.domain.model.surveyunit.Mode; -import fr.insee.genesis.domain.model.variabletype.VariableTypeModel; -import fr.insee.genesis.domain.ports.spi.VariableTypePersistancePort; -import fr.insee.genesis.infrastructure.document.variabletype.VariableTypeDocument; -import fr.insee.genesis.infrastructure.mappers.VariableTypeDocumentMapper; -import lombok.Getter; - -import java.util.ArrayList; -import java.util.List; - -@Getter -public class VariableTypePersistanceStub implements VariableTypePersistancePort { - List mongoStub = new ArrayList<>(); - - @Override - public void save(VariableTypeModel variableTypeModel) { - mongoStub.add(VariableTypeDocumentMapper.INSTANCE.modelToDocument(variableTypeModel)); - } - - @Override - public VariableTypeDocument find(String campaignId, String questionnaireId, Mode mode) { - return mongoStub.stream().filter(variableTypeDocument -> - variableTypeDocument.getCampaignId().equals(campaignId) - && variableTypeDocument.getQuestionnaireId().equals(questionnaireId) - && variableTypeDocument.getMode().equals(mode) - ).toList().getFirst(); - } -} diff --git a/src/test/resources/fr/insee/genesis/features/do_we_process_raw_data.feature b/src/test/resources/fr/insee/genesis/features/do_we_process_raw_data.feature index 00e1424c..53ca94c7 100644 --- a/src/test/resources/fr/insee/genesis/features/do_we_process_raw_data.feature +++ b/src/test/resources/fr/insee/genesis/features/do_we_process_raw_data.feature @@ -16,7 +16,7 @@ Feature: Raw data processing And We process raw data for campaign "", questionnaire "" and interrogation "" Then For collected variable "" in survey unit "" we should have "" for iteration 1 And For external variable "" in survey unit "" we should have "" for iteration 1 - And In surveyUnit "" of the campaign "" we must have "TESTCONTEXT" as contextualId, isCapturedIndirectly to "true" and validationDate null + And In surveyUnit "" of the campaign "" we must have isCapturedIndirectly to "true" and validationDate null Examples: | JsonFile | CampaignId | QuestionnaireId | InterrogationId | CollectedVariableName | ExpectedCollectedValue | ExternalVariableName | ExpectedExternalValue | | raw_data/rawdatasample_filieremodel_optionals.json | RAWDATATESTCAMPAIGN | TESTQUEST | TESTUE00001 | PRENOM_PAR1 | Farid | RPPRENOM | Robert |