From 21dccf4b2b206cb2d8a1805bf1aa447c3285b726 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Henninger?= Date: Wed, 12 Nov 2025 10:46:00 +0100 Subject: [PATCH 01/42] feat: new endpoints to save raw response using the modele filiere library --- .mvn/wrapper/maven-wrapper.properties | 2 + pom.xml | 18 +++-- .../rest/responses/RawResponseController.java | 75 ++++++++++++++----- .../utils/ExtendedJsonNormalizer.java | 53 +++++++++++++ .../controller/utils/JsonSchemaValidator.java | 64 ++++++++++++++++ .../genesis/controller/utils/SchemaType.java | 26 +++++++ .../exceptions/SchemaValidationException.java | 16 ++++ .../RawResponseInputRepository.java | 41 ++++++++++ .../functional_tests/RawDataDefinitions.java | 10 ++- .../responses/RawResponseControllerTest.java | 12 ++- 10 files changed, 288 insertions(+), 29 deletions(-) create mode 100644 .mvn/wrapper/maven-wrapper.properties create mode 100644 src/main/java/fr/insee/genesis/controller/utils/ExtendedJsonNormalizer.java create mode 100644 src/main/java/fr/insee/genesis/controller/utils/JsonSchemaValidator.java create mode 100644 src/main/java/fr/insee/genesis/controller/utils/SchemaType.java create mode 100644 src/main/java/fr/insee/genesis/exceptions/SchemaValidationException.java create mode 100644 src/main/java/fr/insee/genesis/infrastructure/repository/RawResponseInputRepository.java diff --git a/.mvn/wrapper/maven-wrapper.properties b/.mvn/wrapper/maven-wrapper.properties new file mode 100644 index 00000000..8659fbea --- /dev/null +++ b/.mvn/wrapper/maven-wrapper.properties @@ -0,0 +1,2 @@ +distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.9.11/apache-maven-3.9.11-bin.zip +wrapperUrl=https://repo.maven.apache.org/maven2/org/apache/maven/wrapper/maven-wrapper/3.3.4/maven-wrapper-3.3.4.jar diff --git a/pom.xml b/pom.xml index 5a7451cf..c8fcf8a1 100644 --- a/pom.xml +++ b/pom.xml @@ -98,7 +98,16 @@ springdoc-openapi-starter-webmvc-ui ${springdoc.version} - + + fr.insee + modelefiliere + add-processed-response-SNAPSHOT + + + com.networknt + json-schema-validator + 1.5.9 + @@ -107,13 +116,6 @@ 12.9 - - - com.networknt - json-schema-validator - 2.0.0 - - org.mapstruct diff --git a/src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseController.java b/src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseController.java index ed41b205..baf0cbb0 100644 --- a/src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseController.java +++ b/src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseController.java @@ -1,18 +1,22 @@ package fr.insee.genesis.controller.rest.responses; -import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; -import com.networknt.schema.Error; -import com.networknt.schema.Schema; -import com.networknt.schema.SchemaRegistry; -import com.networknt.schema.dialect.Dialects; +import com.fasterxml.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; import fr.insee.genesis.controller.dto.rawdata.LunaticJsonRawDataUnprocessedDto; +import fr.insee.genesis.controller.utils.ExtendedJsonNormalizer; +import fr.insee.genesis.controller.utils.JsonSchemaValidator; +import fr.insee.genesis.controller.utils.SchemaType; import fr.insee.genesis.domain.model.surveyunit.Mode; import fr.insee.genesis.domain.model.surveyunit.rawdata.DataProcessResult; import fr.insee.genesis.domain.model.surveyunit.rawdata.LunaticJsonRawDataModel; import fr.insee.genesis.domain.ports.api.LunaticJsonRawDataApiPort; import fr.insee.genesis.exceptions.GenesisError; import fr.insee.genesis.exceptions.GenesisException; +import fr.insee.genesis.exceptions.SchemaValidationException; +import fr.insee.genesis.infrastructure.repository.RawResponseInputRepository; +import fr.insee.modelefiliere.RawResponseDto; import io.swagger.v3.oas.annotations.Hidden; import io.swagger.v3.oas.annotations.Operation; import lombok.extern.slf4j.Slf4j; @@ -30,9 +34,9 @@ import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.PutMapping; import org.springframework.web.bind.annotation.RequestBody; -import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestParam; +import java.io.IOException; import java.time.Instant; import java.time.LocalDateTime; import java.util.ArrayList; @@ -42,20 +46,22 @@ @Slf4j @Controller -@RequestMapping(path = "/responses/raw") public class RawResponseController { private static final String SUCCESS_MESSAGE = "Interrogation %s saved"; private static final String PARTITION_ID = "partitionId"; private static final String INTERROGATION_ID = "interrogationId"; private final LunaticJsonRawDataApiPort lunaticJsonRawDataApiPort; + private final RawResponseInputRepository rawRepository; - public RawResponseController(LunaticJsonRawDataApiPort lunaticJsonRawDataApiPort) { + + public RawResponseController(LunaticJsonRawDataApiPort lunaticJsonRawDataApiPort, RawResponseInputRepository rawRepository) { this.lunaticJsonRawDataApiPort = lunaticJsonRawDataApiPort; + this.rawRepository = rawRepository; } @Operation(summary = "Save lunatic json data from one interrogation in Genesis Database") - @PutMapping(path = "/lunatic-json/save") + @PutMapping(path = "/responses/raw/lunatic-json/save") @PreAuthorize("hasRole('COLLECT_PLATFORM')") public ResponseEntity saveRawResponsesFromJsonBody( @@ -86,16 +92,18 @@ public ResponseEntity saveRawResponsesFromJsonBody( return ResponseEntity.status(201).body(String.format(SUCCESS_MESSAGE, interrogationId)); } - @Operation(summary = "Save lunatic json data from one interrogation in Genesis Database (with json " + - "schema validation)") - @PutMapping(path = "/lunatic-json") +/* @Operation(summary = "Deprecated") + @PutMapping(path="/lunatic-json") @PreAuthorize("hasRole('COLLECT_PLATFORM')") - public ResponseEntity saveRawResponsesFromJsonBodyWithValidation( + // Check version when merging + @Deprecated(since="1.13.0", forRemoval=true) + public ResponseEntity saveRawResponsesFromJsonBodyWithValidationDeprecated( @RequestBody Map body ) { + SchemaRegistry schemaRegistry = SchemaRegistry.withDialect(Dialects.getDraft202012(), SchemaRegistry.Builder::build); Schema jsonSchema = schemaRegistry - .getSchema(RawResponseController.class.getResourceAsStream("/jsonSchemas/RawResponse.json") + .getSchema(RawResponseController.class.getResourceAsStream("/modele-filiere-spec/RawResponse.json") ); try { if (jsonSchema == null) { @@ -134,11 +142,40 @@ public ResponseEntity saveRawResponsesFromJsonBodyWithValidation( log.info("Data saved for interrogationId {} and partition {}", body.get(INTERROGATION_ID).toString(), body.get(PARTITION_ID).toString()); return ResponseEntity.status(201).body(String.format(SUCCESS_MESSAGE, body.get(INTERROGATION_ID).toString())); + }*/ + + @Operation(summary = "Save lunatic json data from one interrogation in Genesis Database (with json " + + "schema validation)") + @PutMapping(path="/raw-responses") + @PreAuthorize("hasRole('COLLECT_PLATFORM')") + public ResponseEntity saveRawResponsesFromJsonBodyWithValidation( + @RequestBody Map body + ) { + ObjectMapper objectMapperLocal = new ObjectMapper(); + objectMapperLocal.registerModule(new JavaTimeModule()); + objectMapperLocal.disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS); // ISO-8601 + objectMapperLocal.disable(DeserializationFeature.ADJUST_DATES_TO_CONTEXT_TIME_ZONE); + + try { + RawResponseDto rawResponseDto = JsonSchemaValidator.readAndValidateFromClasspath( + ExtendedJsonNormalizer.normalize(new ObjectMapper().readTree( + new ObjectMapper().writeValueAsString(body))), + SchemaType.RAW_RESPONSE.getSchemaFileName(), + RawResponseDto.class, + objectMapperLocal + ); + rawRepository.saveAsRawJson(rawResponseDto); + } catch (SchemaValidationException | IOException e) { + return ResponseEntity.status(400).body(e.toString()); + } + return ResponseEntity.ok("Change this when ready"); + } + } //GET unprocessed @Operation(summary = "Get campaign id and interrogationId from all unprocessed raw json data") - @GetMapping(path = "/lunatic-json/get/unprocessed") + @GetMapping(path = "/responses/raw/lunatic-json/get/unprocessed") @PreAuthorize("hasRole('SCHEDULER')") public ResponseEntity> getUnproccessedJsonRawData() { log.info("Try to get unprocessed raw JSON datas..."); @@ -146,7 +183,7 @@ public ResponseEntity> getUnproccessedJso } @Hidden - @GetMapping(path = "/lunatic-json/get/by-interrogation-mode-and-campaign") + @GetMapping(path = "/responses/raw/lunatic-json/get/by-interrogation-mode-and-campaign") @PreAuthorize("hasRole('ADMIN')") public ResponseEntity getJsonRawData( @RequestParam(INTERROGATION_ID) String interrogationId, @@ -159,7 +196,7 @@ public ResponseEntity getJsonRawData( //PROCESS @Operation(summary = "Process raw data of a campaign") - @PostMapping(path = "/lunatic-json/process") + @PostMapping(path = "/responses/raw/lunatic-json/process") @PreAuthorize("hasRole('SCHEDULER')") public ResponseEntity processJsonRawData( @RequestParam("campaignName") String campaignName, @@ -181,7 +218,7 @@ public ResponseEntity processJsonRawData( } @Operation(summary = "Get processed data ids from last n hours (default 24h)") - @GetMapping(path = "/lunatic-json/processed/ids") + @GetMapping(path = "/responses/raw/lunatic-json/processed/ids") @PreAuthorize("hasRole('ADMIN')") public ResponseEntity>> getProcessedDataIdsSinceHours( @RequestParam("questionnaireId") String questionnaireId, @@ -193,7 +230,7 @@ public ResponseEntity>> getProcessedDataIdsSinceHours( } @Operation(summary = "Get lunatic JSON data from one campaign in Genesis Database, filtered by start and end dates") - @GetMapping(path = "/lunatic-json/{campaignId}") + @GetMapping(path = "/responses/raw/lunatic-json/{campaignId}") @PreAuthorize("hasRole('USER_BATCH_GENERIC')") public ResponseEntity> getRawResponsesFromJsonBody( @PathVariable String campaignId, diff --git a/src/main/java/fr/insee/genesis/controller/utils/ExtendedJsonNormalizer.java b/src/main/java/fr/insee/genesis/controller/utils/ExtendedJsonNormalizer.java new file mode 100644 index 00000000..62bbd94d --- /dev/null +++ b/src/main/java/fr/insee/genesis/controller/utils/ExtendedJsonNormalizer.java @@ -0,0 +1,53 @@ +package fr.insee.genesis.controller.utils; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.*; + +import java.util.Iterator; +import java.util.Map; +public class ExtendedJsonNormalizer { + + private ExtendedJsonNormalizer() {} + + /** + * Recursively converts Mongo Extended JSON objects into simple types expected by the schema: + * - {"$date": "..."} -> TextNode("...") + * + * Leaves all other values untouched. Returns a structural copy of the node (does not mutate the original). + */ + + public static JsonNode normalize(JsonNode node) { + if (node == null || node.isNull() || node.isMissingNode()) return node; + + if (node.isObject()) { + ObjectNode obj = (ObjectNode) node; + + if (obj.size() == 1) { + if (obj.has("$date") && obj.get("$date").isTextual()) { + return TextNode.valueOf(obj.get("$date").asText()); + } +// if (obj.has("$oid") && obj.get("$oid").isTextual()) { +// return TextNode.valueOf(obj.get("$oid").asText()); +// } + } + + ObjectNode copy = obj.objectNode(); + Iterator> it = obj.fields(); + while (it.hasNext()) { + Map.Entry e = it.next(); + copy.set(e.getKey(), normalize(e.getValue())); + } + return copy; + } + + if (node.isArray()) { + ArrayNode src = (ArrayNode) node; + ArrayNode dst = src.arrayNode(); + for (JsonNode child : src) { + dst.add(normalize(child)); + } + return dst; + } + return node; + } +} diff --git a/src/main/java/fr/insee/genesis/controller/utils/JsonSchemaValidator.java b/src/main/java/fr/insee/genesis/controller/utils/JsonSchemaValidator.java new file mode 100644 index 00000000..68c8df0a --- /dev/null +++ b/src/main/java/fr/insee/genesis/controller/utils/JsonSchemaValidator.java @@ -0,0 +1,64 @@ +package fr.insee.genesis.controller.utils; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.networknt.schema.JsonSchema; +import com.networknt.schema.JsonSchemaFactory; +import com.networknt.schema.SpecVersion; +import com.networknt.schema.ValidationMessage; +import fr.insee.genesis.exceptions.SchemaValidationException; +import lombok.extern.slf4j.Slf4j; + +import java.io.IOException; +import java.io.InputStream; + +@Slf4j +public class JsonSchemaValidator { + + private JsonSchemaValidator() {} + + public static T readAndValidate(JsonNode root, + JsonSchema schema, + Class targetType, + ObjectMapper mapper) + throws SchemaValidationException, JsonProcessingException { + ensureValid(root, schema); + return mapper.treeToValue(root, targetType); + } + + public static T readAndValidateFromClasspath(JsonNode root, + String schemaResourcePath, + Class targetType, + ObjectMapper mapper) + throws SchemaValidationException, IOException { + JsonSchema schema = loadSchemaFromClasspath(schemaResourcePath, mapper); + return readAndValidate(root, schema, targetType, mapper); + } + + public static JsonSchema loadSchemaFromClasspath(String resourcePath, ObjectMapper mapper) throws IOException { + if (resourcePath == null || resourcePath.isBlank()) { + throw new IOException("Schema resource path is null or blank"); + } + String cp = resourcePath.startsWith("/") ? resourcePath.substring(1) : resourcePath; + try (InputStream in = Thread.currentThread().getContextClassLoader().getResourceAsStream(cp)) { + if (in == null) throw new IOException("Schema not found on classpath: " + resourcePath); + JsonNode schemaNode = mapper.readTree(in); + return JsonSchemaFactory.getInstance(SpecVersion.VersionFlag.V202012).getSchema(schemaNode); + } + } + + private static void ensureValid(JsonNode root, JsonSchema schema) throws SchemaValidationException { + log.info("Schéma Apply : " + schema.getSchemaNode().get("title")); + java.util.Set errors = schema.validate(root); + if (!errors.isEmpty()) { + String formatted = errors.stream() + .sorted(java.util.Comparator.comparing(ValidationMessage::getEvaluationPath)) + .map(err -> err.getMessage()) + .collect(java.util.stream.Collectors.joining("\n")); + throw new SchemaValidationException( + "Uploaded JSON is not correct according to the json-schema:\n" + formatted, errors); + } + log.info("Schema-compliant JSON : " + schema.getSchemaNode().get("title")); + } +} diff --git a/src/main/java/fr/insee/genesis/controller/utils/SchemaType.java b/src/main/java/fr/insee/genesis/controller/utils/SchemaType.java new file mode 100644 index 00000000..16171879 --- /dev/null +++ b/src/main/java/fr/insee/genesis/controller/utils/SchemaType.java @@ -0,0 +1,26 @@ +package fr.insee.genesis.controller.utils; + +import lombok.Getter; + +@Getter +public enum SchemaType { + + // PROCESS_MESSAGE(Names.PROCESS_MESSAGE), + INTERROGATION(Names.INTERROGATION), + RAW_RESPONSE(Names.RAW_RESPONSE); + + public static class Names { +// public static final String PROCESS_MESSAGE = "/modele-filiere-spec/Command.json"; + public static final String INTERROGATION = "/modele-filiere-spec/Interrogation.json"; + public static final String RAW_RESPONSE = "/modele-filiere-spec/RawResponse.json"; + + private Names() { + + } + } + private final String schemaFileName; + + SchemaType(String schemaFileName) { + this.schemaFileName = schemaFileName; + } +} diff --git a/src/main/java/fr/insee/genesis/exceptions/SchemaValidationException.java b/src/main/java/fr/insee/genesis/exceptions/SchemaValidationException.java new file mode 100644 index 00000000..37313109 --- /dev/null +++ b/src/main/java/fr/insee/genesis/exceptions/SchemaValidationException.java @@ -0,0 +1,16 @@ +package fr.insee.genesis.exceptions; + +import com.networknt.schema.ValidationMessage; +import lombok.AllArgsConstructor; +import lombok.Getter; + +import java.util.Set; + +@Getter +@AllArgsConstructor +public class SchemaValidationException extends Exception{ + + private final String message; + + private Set errors; +} diff --git a/src/main/java/fr/insee/genesis/infrastructure/repository/RawResponseInputRepository.java b/src/main/java/fr/insee/genesis/infrastructure/repository/RawResponseInputRepository.java new file mode 100644 index 00000000..96242a0b --- /dev/null +++ b/src/main/java/fr/insee/genesis/infrastructure/repository/RawResponseInputRepository.java @@ -0,0 +1,41 @@ +package fr.insee.genesis.infrastructure.repository; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import fr.insee.modelefiliere.RawResponseDto; +import lombok.RequiredArgsConstructor; +import org.bson.Document; +import org.springframework.data.mongodb.core.MongoTemplate; +import org.springframework.stereotype.Repository; + +import java.time.Instant; +import java.util.Map; + +@Repository +@RequiredArgsConstructor +public class RawResponseInputRepository { + + private final MongoTemplate mongoTemplate; + private final ObjectMapper objectMapper; + + public void saveAsRawJson(RawResponseDto dto) { + // Convertir le DTO en JSON brut + String json = null; + + try { + json = objectMapper.writeValueAsString(dto); + // Parser en Document BSON (Mongo gère automatiquement les types) + Document payload = Document.parse(json); + + Map document = new java.util.HashMap<>(); + document.put("interrogationId", dto.getInterrogationId()); + document.put("collectionInstrumentId", dto.getCollectionInstrumentId()); + document.put("recordDate", Instant.now()); + document.put("payload", payload); + mongoTemplate.save(document, "rawResponses"); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + + } +} diff --git a/src/test/java/cucumber/functional_tests/RawDataDefinitions.java b/src/test/java/cucumber/functional_tests/RawDataDefinitions.java index 137ffd4a..05479ba3 100644 --- a/src/test/java/cucumber/functional_tests/RawDataDefinitions.java +++ b/src/test/java/cucumber/functional_tests/RawDataDefinitions.java @@ -15,6 +15,7 @@ import fr.insee.genesis.domain.service.surveyunit.SurveyUnitQualityService; import fr.insee.genesis.domain.service.surveyunit.SurveyUnitService; import fr.insee.genesis.domain.utils.JsonUtils; +import fr.insee.genesis.infrastructure.repository.RawResponseInputRepository; import fr.insee.genesis.infrastructure.utils.FileUtils; import fr.insee.genesis.stubs.ConfigStub; import fr.insee.genesis.stubs.DataProcessingContextPersistancePortStub; @@ -22,6 +23,7 @@ import fr.insee.genesis.stubs.QuestionnaireMetadataPersistancePortStub; import fr.insee.genesis.stubs.SurveyUnitPersistencePortStub; import fr.insee.genesis.stubs.SurveyUnitQualityToolPerretAdapterStub; +import fr.insee.modelefiliere.RawResponseDto; import io.cucumber.java.Before; import io.cucumber.java.en.Given; import io.cucumber.java.en.Then; @@ -88,8 +90,14 @@ public class RawDataDefinitions { config, dataProcessingContextPersistancePortStub); + RawResponseInputRepository rawResponseInputRepositoryStub = new RawResponseInputRepository(null, null) { + @Override + public void saveAsRawJson(RawResponseDto dto) { + // Ne rien faire — stub pour les tests + } + }; RawResponseController rawResponseController = new RawResponseController( - lunaticJsonRawDataService + lunaticJsonRawDataService, rawResponseInputRepositoryStub ); Path rawDataFilePath; String rawJsonData; diff --git a/src/test/java/fr/insee/genesis/controller/rest/responses/RawResponseControllerTest.java b/src/test/java/fr/insee/genesis/controller/rest/responses/RawResponseControllerTest.java index 26d93a9f..0da8a3c1 100644 --- a/src/test/java/fr/insee/genesis/controller/rest/responses/RawResponseControllerTest.java +++ b/src/test/java/fr/insee/genesis/controller/rest/responses/RawResponseControllerTest.java @@ -16,6 +16,7 @@ import fr.insee.genesis.domain.utils.JsonUtils; import fr.insee.genesis.infrastructure.document.rawdata.LunaticJsonRawDataDocument; import fr.insee.genesis.infrastructure.mappers.DataProcessingContextMapper; +import fr.insee.genesis.infrastructure.repository.RawResponseInputRepository; import fr.insee.genesis.infrastructure.utils.FileUtils; import fr.insee.genesis.stubs.ConfigStub; import fr.insee.genesis.stubs.DataProcessingContextPersistancePortStub; @@ -23,6 +24,7 @@ import fr.insee.genesis.stubs.QuestionnaireMetadataPersistancePortStub; import fr.insee.genesis.stubs.SurveyUnitPersistencePortStub; import fr.insee.genesis.stubs.SurveyUnitQualityToolPerretAdapterStub; +import fr.insee.modelefiliere.RawResponseDto; import org.assertj.core.api.Assertions; import org.junit.jupiter.api.Test; import org.springframework.data.web.PagedModel; @@ -58,7 +60,15 @@ class RawResponseControllerTest { new DataProcessingContextPersistancePortStub() ); - private final RawResponseController rawResponseController = new RawResponseController(lunaticJsonRawDataApiPort); + // TODO: change this + RawResponseInputRepository rawResponseInputRepositoryStub = new RawResponseInputRepository(null, null) { + @Override + public void saveAsRawJson(RawResponseDto dto) { + // Ne rien faire — stub pour les tests + } + }; + + private final RawResponseController rawResponseController = new RawResponseController(lunaticJsonRawDataApiPort, rawResponseInputRepositoryStub); @Test From 3ec96002fc13fd6a6de4454a1f7493aa8fbbf2ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Henninger?= Date: Wed, 12 Nov 2025 10:58:08 +0100 Subject: [PATCH 02/42] fix: remove unnecessary bracket --- .../controller/rest/responses/RawResponseController.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseController.java b/src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseController.java index baf0cbb0..c5e1ad88 100644 --- a/src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseController.java +++ b/src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseController.java @@ -171,8 +171,6 @@ public ResponseEntity saveRawResponsesFromJsonBodyWithValidation( return ResponseEntity.ok("Change this when ready"); } - } - //GET unprocessed @Operation(summary = "Get campaign id and interrogationId from all unprocessed raw json data") @GetMapping(path = "/responses/raw/lunatic-json/get/unprocessed") From d5e54649923659b4c04ecde24a425bd19552f28e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Henninger?= Date: Wed, 12 Nov 2025 11:45:17 +0100 Subject: [PATCH 03/42] chore: add log --- .../controller/rest/responses/RawResponseController.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseController.java b/src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseController.java index c5e1ad88..c6319548 100644 --- a/src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseController.java +++ b/src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseController.java @@ -1,5 +1,6 @@ package fr.insee.genesis.controller.rest.responses; +import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.SerializationFeature; @@ -150,16 +151,18 @@ public ResponseEntity saveRawResponsesFromJsonBodyWithValidationDeprecat @PreAuthorize("hasRole('COLLECT_PLATFORM')") public ResponseEntity saveRawResponsesFromJsonBodyWithValidation( @RequestBody Map body - ) { + ) throws JsonProcessingException { + ObjectMapper objectMapperLocal = new ObjectMapper(); objectMapperLocal.registerModule(new JavaTimeModule()); objectMapperLocal.disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS); // ISO-8601 objectMapperLocal.disable(DeserializationFeature.ADJUST_DATES_TO_CONTEXT_TIME_ZONE); + log.info(objectMapperLocal.writeValueAsString(body)); try { RawResponseDto rawResponseDto = JsonSchemaValidator.readAndValidateFromClasspath( - ExtendedJsonNormalizer.normalize(new ObjectMapper().readTree( - new ObjectMapper().writeValueAsString(body))), + ExtendedJsonNormalizer.normalize(objectMapperLocal.readTree( + objectMapperLocal.writeValueAsString(body))), SchemaType.RAW_RESPONSE.getSchemaFileName(), RawResponseDto.class, objectMapperLocal From 4f4a9d719bfbbf98ba41a643200b686ef3e0fa5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Henninger?= Date: Wed, 12 Nov 2025 13:18:41 +0100 Subject: [PATCH 04/42] test: add new endpoint for testing purpose --- .../rest/responses/RawResponseController.java | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseController.java b/src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseController.java index c6319548..3527b4a3 100644 --- a/src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseController.java +++ b/src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseController.java @@ -147,7 +147,7 @@ public ResponseEntity saveRawResponsesFromJsonBodyWithValidationDeprecat @Operation(summary = "Save lunatic json data from one interrogation in Genesis Database (with json " + "schema validation)") - @PutMapping(path="/raw-responses") + @PostMapping(path="/raw-responses") @PreAuthorize("hasRole('COLLECT_PLATFORM')") public ResponseEntity saveRawResponsesFromJsonBodyWithValidation( @RequestBody Map body @@ -174,6 +174,17 @@ public ResponseEntity saveRawResponsesFromJsonBodyWithValidation( return ResponseEntity.ok("Change this when ready"); } + @Operation(summary = "Save lunatic json data from one interrogation in Genesis Database (with json " + + "schema validation)") + @PostMapping(path="/raw-responses/dto") + @PreAuthorize("hasRole('COLLECT_PLATFORM')") + public ResponseEntity saveRawResponsesFromRawResponseDto( + @RequestBody RawResponseDto dto + ) { + rawRepository.saveAsRawJson(dto); + return ResponseEntity.ok("Change this when ready"); + } + //GET unprocessed @Operation(summary = "Get campaign id and interrogationId from all unprocessed raw json data") @GetMapping(path = "/responses/raw/lunatic-json/get/unprocessed") From 71f342957b74e1e5cf475a8762b1880c3fcb7d84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Henninger?= Date: Wed, 12 Nov 2025 16:16:57 +0100 Subject: [PATCH 05/42] feat: add @valid on endpoint --- .../controller/rest/responses/RawResponseController.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseController.java b/src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseController.java index 3527b4a3..f991a0dd 100644 --- a/src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseController.java +++ b/src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseController.java @@ -20,6 +20,7 @@ import fr.insee.modelefiliere.RawResponseDto; import io.swagger.v3.oas.annotations.Hidden; import io.swagger.v3.oas.annotations.Operation; +import jakarta.validation.Valid; import lombok.extern.slf4j.Slf4j; import org.springframework.data.domain.Page; import org.springframework.data.domain.PageRequest; @@ -179,7 +180,7 @@ public ResponseEntity saveRawResponsesFromJsonBodyWithValidation( @PostMapping(path="/raw-responses/dto") @PreAuthorize("hasRole('COLLECT_PLATFORM')") public ResponseEntity saveRawResponsesFromRawResponseDto( - @RequestBody RawResponseDto dto + @Valid @RequestBody RawResponseDto dto ) { rawRepository.saveAsRawJson(dto); return ResponseEntity.ok("Change this when ready"); From 8d7eec123afeb3f7532e3e4fc648580ab4cd25b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Henninger?= Date: Wed, 12 Nov 2025 17:23:48 +0100 Subject: [PATCH 06/42] feat: change raw responses urls and status --- .../controller/rest/responses/RawResponseController.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseController.java b/src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseController.java index f991a0dd..9174c257 100644 --- a/src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseController.java +++ b/src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseController.java @@ -148,7 +148,7 @@ public ResponseEntity saveRawResponsesFromJsonBodyWithValidationDeprecat @Operation(summary = "Save lunatic json data from one interrogation in Genesis Database (with json " + "schema validation)") - @PostMapping(path="/raw-responses") + @PostMapping(path="/raw-responses/debug") @PreAuthorize("hasRole('COLLECT_PLATFORM')") public ResponseEntity saveRawResponsesFromJsonBodyWithValidation( @RequestBody Map body @@ -172,18 +172,18 @@ public ResponseEntity saveRawResponsesFromJsonBodyWithValidation( } catch (SchemaValidationException | IOException e) { return ResponseEntity.status(400).body(e.toString()); } - return ResponseEntity.ok("Change this when ready"); + return ResponseEntity.status(201).body(String.format(SUCCESS_MESSAGE, body.get(INTERROGATION_ID).toString())); } @Operation(summary = "Save lunatic json data from one interrogation in Genesis Database (with json " + "schema validation)") - @PostMapping(path="/raw-responses/dto") + @PostMapping(path="/raw-responses") @PreAuthorize("hasRole('COLLECT_PLATFORM')") public ResponseEntity saveRawResponsesFromRawResponseDto( @Valid @RequestBody RawResponseDto dto ) { rawRepository.saveAsRawJson(dto); - return ResponseEntity.ok("Change this when ready"); + return ResponseEntity.status(201).body(String.format(SUCCESS_MESSAGE, dto.getInterrogationId())); } //GET unprocessed From eee89db5937a11e001f2387273f593dede42f0fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Henninger?= Date: Fri, 14 Nov 2025 16:20:00 +0100 Subject: [PATCH 07/42] feat: WIP implementing new model of raw responses --- .../genesis/controller/dto/ScheduleDto.java | 1 + .../rest/responses/RawResponseController.java | 27 ++- .../context/DataProcessingContextModel.java | 3 + .../model/surveyunit/SurveyUnitModel.java | 6 + .../model/surveyunit/rawdata/RawResponse.java | 18 ++ .../api/DataProcessingContextApiPort.java | 3 + .../domain/ports/api/RawResponseApiPort.java | 19 ++ .../DataProcessingContextPersistancePort.java | 2 + .../ports/spi/RawResponsePersistencePort.java | 15 ++ .../context/DataProcessingContextService.java | 7 + .../service/rawdata/RawResponseService.java | 225 ++++++++++++++++++ .../DataProcessingContextMongoAdapter.java | 7 + .../adapter/RawResponseMongoAdapter.java | 49 ++++ .../DataProcessingContextDocument.java | 1 + .../document/rawdata/RawResponseDocument.java | 26 ++ .../mappers/RawResponseDocumentMapper.java | 19 ++ ...ataProcessingContextMongoDBRepository.java | 3 + .../RawResponseInputRepository.java | 1 + .../repository/RawResponseRepository.java | 16 ++ .../functional_tests/RawDataDefinitions.java | 31 ++- .../responses/RawResponseControllerTest.java | 31 ++- 21 files changed, 507 insertions(+), 3 deletions(-) create mode 100644 src/main/java/fr/insee/genesis/domain/model/surveyunit/rawdata/RawResponse.java create mode 100644 src/main/java/fr/insee/genesis/domain/ports/api/RawResponseApiPort.java create mode 100644 src/main/java/fr/insee/genesis/domain/ports/spi/RawResponsePersistencePort.java create mode 100644 src/main/java/fr/insee/genesis/domain/service/rawdata/RawResponseService.java create mode 100644 src/main/java/fr/insee/genesis/infrastructure/adapter/RawResponseMongoAdapter.java create mode 100644 src/main/java/fr/insee/genesis/infrastructure/document/rawdata/RawResponseDocument.java create mode 100644 src/main/java/fr/insee/genesis/infrastructure/mappers/RawResponseDocumentMapper.java create mode 100644 src/main/java/fr/insee/genesis/infrastructure/repository/RawResponseRepository.java diff --git a/src/main/java/fr/insee/genesis/controller/dto/ScheduleDto.java b/src/main/java/fr/insee/genesis/controller/dto/ScheduleDto.java index 976bfe7e..bf67427e 100644 --- a/src/main/java/fr/insee/genesis/controller/dto/ScheduleDto.java +++ b/src/main/java/fr/insee/genesis/controller/dto/ScheduleDto.java @@ -8,6 +8,7 @@ @Builder public record ScheduleDto (String surveyName, + String collectionInstrumentId, LocalDateTime lastExecution, List kraftwerkExecutionScheduleList ){} diff --git a/src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseController.java b/src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseController.java index 9174c257..cd13c1de 100644 --- a/src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseController.java +++ b/src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseController.java @@ -13,6 +13,7 @@ import fr.insee.genesis.domain.model.surveyunit.rawdata.DataProcessResult; import fr.insee.genesis.domain.model.surveyunit.rawdata.LunaticJsonRawDataModel; import fr.insee.genesis.domain.ports.api.LunaticJsonRawDataApiPort; +import fr.insee.genesis.domain.ports.api.RawResponseApiPort; import fr.insee.genesis.exceptions.GenesisError; import fr.insee.genesis.exceptions.GenesisException; import fr.insee.genesis.exceptions.SchemaValidationException; @@ -54,11 +55,13 @@ public class RawResponseController { private static final String PARTITION_ID = "partitionId"; private static final String INTERROGATION_ID = "interrogationId"; private final LunaticJsonRawDataApiPort lunaticJsonRawDataApiPort; + private final RawResponseApiPort rawResponseApiPort; private final RawResponseInputRepository rawRepository; - public RawResponseController(LunaticJsonRawDataApiPort lunaticJsonRawDataApiPort, RawResponseInputRepository rawRepository) { + public RawResponseController(LunaticJsonRawDataApiPort lunaticJsonRawDataApiPort, RawResponseApiPort rawResponseApiPort, RawResponseInputRepository rawRepository) { this.lunaticJsonRawDataApiPort = lunaticJsonRawDataApiPort; + this.rawResponseApiPort = rawResponseApiPort; this.rawRepository = rawRepository; } @@ -186,6 +189,28 @@ public ResponseEntity saveRawResponsesFromRawResponseDto( return ResponseEntity.status(201).body(String.format(SUCCESS_MESSAGE, dto.getInterrogationId())); } + //PROCESS + @Operation(summary = "Process raw data of a campaign") + @PostMapping(path = "/raw-responses/process") + @PreAuthorize("hasRole('SCHEDULER')") + public ResponseEntity processRawResponses( + @RequestParam("questionnaireId") String questionnaireId, + @RequestBody List interrogationIdList + ) { + log.info("Try to process raw responses for questionnaireId {} and {} interrogationIds", interrogationIdList.size()); + List errors = new ArrayList<>(); + + try { + DataProcessResult result = rawResponseApiPort.processRawResponses(questionnaireId, interrogationIdList, errors); + return result.formattedDataCount() == 0 ? + ResponseEntity.ok("%d document(s) processed".formatted(result.dataCount())) + : ResponseEntity.ok("%d document(s) processed, including %d FORMATTED after data verification" + .formatted(result.dataCount(), result.formattedDataCount())); + } catch (GenesisException e) { + return ResponseEntity.status(e.getStatus()).body(e.getMessage()); + } + } + //GET unprocessed @Operation(summary = "Get campaign id and interrogationId from all unprocessed raw json data") @GetMapping(path = "/responses/raw/lunatic-json/get/unprocessed") diff --git a/src/main/java/fr/insee/genesis/domain/model/context/DataProcessingContextModel.java b/src/main/java/fr/insee/genesis/domain/model/context/DataProcessingContextModel.java index 10de6526..58a19c62 100644 --- a/src/main/java/fr/insee/genesis/domain/model/context/DataProcessingContextModel.java +++ b/src/main/java/fr/insee/genesis/domain/model/context/DataProcessingContextModel.java @@ -22,6 +22,8 @@ public class DataProcessingContextModel { private String partitionId; + private String collectionInstrumentId; //QuestionnaireId + private LocalDateTime lastExecution; List kraftwerkExecutionScheduleList; @@ -31,6 +33,7 @@ public class DataProcessingContextModel { public ScheduleDto toScheduleDto(){ return ScheduleDto.builder() .surveyName(partitionId) + .collectionInstrumentId(collectionInstrumentId) .lastExecution(lastExecution) .kraftwerkExecutionScheduleList(kraftwerkExecutionScheduleList) .build(); diff --git a/src/main/java/fr/insee/genesis/domain/model/surveyunit/SurveyUnitModel.java b/src/main/java/fr/insee/genesis/domain/model/surveyunit/SurveyUnitModel.java index d1fc8bd5..cac82c6d 100644 --- a/src/main/java/fr/insee/genesis/domain/model/surveyunit/SurveyUnitModel.java +++ b/src/main/java/fr/insee/genesis/domain/model/surveyunit/SurveyUnitModel.java @@ -10,12 +10,18 @@ import java.util.List; import java.util.Objects; +/* + * This model class should be in equation with the model of our information system ("modèle filière) + * Its up to the adapter and mappers to deal with old fields + * + */ @Builder @Data @NoArgsConstructor @AllArgsConstructor public class SurveyUnitModel { + private String questionnaireId; private String campaignId; private String interrogationId; diff --git a/src/main/java/fr/insee/genesis/domain/model/surveyunit/rawdata/RawResponse.java b/src/main/java/fr/insee/genesis/domain/model/surveyunit/rawdata/RawResponse.java new file mode 100644 index 00000000..7f50b76c --- /dev/null +++ b/src/main/java/fr/insee/genesis/domain/model/surveyunit/rawdata/RawResponse.java @@ -0,0 +1,18 @@ +package fr.insee.genesis.domain.model.surveyunit.rawdata; + +import fr.insee.genesis.domain.model.surveyunit.Mode; +import org.bson.types.ObjectId; + +import java.time.LocalDateTime; +import java.util.Map; + +public record RawResponse( + ObjectId id, + String interrogationId, + String collectionInstrumentId, + Mode mode, + Map payload, + LocalDateTime recordDate, + LocalDateTime processDate +) +{} diff --git a/src/main/java/fr/insee/genesis/domain/ports/api/DataProcessingContextApiPort.java b/src/main/java/fr/insee/genesis/domain/ports/api/DataProcessingContextApiPort.java index f26ad129..0fc9f0d8 100644 --- a/src/main/java/fr/insee/genesis/domain/ports/api/DataProcessingContextApiPort.java +++ b/src/main/java/fr/insee/genesis/domain/ports/api/DataProcessingContextApiPort.java @@ -32,6 +32,9 @@ void saveKraftwerkExecutionSchedule(String partitionId, DataProcessingContextModel getContext(String interrogationId) throws GenesisException; DataProcessingContextModel getContextByPartitionId(String partitionId) throws GenesisException; + + DataProcessingContextModel getContextByCollectionInstrumentId(String collectionInstrumentId); + List getPartitionIds(boolean withReview); /** diff --git a/src/main/java/fr/insee/genesis/domain/ports/api/RawResponseApiPort.java b/src/main/java/fr/insee/genesis/domain/ports/api/RawResponseApiPort.java new file mode 100644 index 00000000..1d318538 --- /dev/null +++ b/src/main/java/fr/insee/genesis/domain/ports/api/RawResponseApiPort.java @@ -0,0 +1,19 @@ +package fr.insee.genesis.domain.ports.api; + +import fr.insee.bpm.metadata.model.VariablesMap; +import fr.insee.genesis.domain.model.surveyunit.Mode; +import fr.insee.genesis.domain.model.surveyunit.SurveyUnitModel; +import fr.insee.genesis.domain.model.surveyunit.rawdata.DataProcessResult; +import fr.insee.genesis.domain.model.surveyunit.rawdata.RawResponse; +import fr.insee.genesis.exceptions.GenesisError; +import fr.insee.genesis.exceptions.GenesisException; + +import java.util.List; + +public interface RawResponseApiPort { + + List getRawResponses(String questionnaireModelId, Mode mode, List interrogationIdList); + DataProcessResult processRawResponses(String questionnaireId, List interrogationIdList, List errors) throws GenesisException; + List convertRawResponse(List rawResponses, VariablesMap variablesMap); + void updateProcessDates(List surveyUnitModels); +} diff --git a/src/main/java/fr/insee/genesis/domain/ports/spi/DataProcessingContextPersistancePort.java b/src/main/java/fr/insee/genesis/domain/ports/spi/DataProcessingContextPersistancePort.java index eb579253..69d3d6ba 100644 --- a/src/main/java/fr/insee/genesis/domain/ports/spi/DataProcessingContextPersistancePort.java +++ b/src/main/java/fr/insee/genesis/domain/ports/spi/DataProcessingContextPersistancePort.java @@ -11,6 +11,8 @@ public interface DataProcessingContextPersistancePort { List findByPartitionIds(List partitionIds); + List findByCollectionInstrumentIds(List collectionInstrumentIds); + void save(DataProcessingContextDocument dataProcessingContextDocument); void saveAll(List dataProcessingContextDocuments); diff --git a/src/main/java/fr/insee/genesis/domain/ports/spi/RawResponsePersistencePort.java b/src/main/java/fr/insee/genesis/domain/ports/spi/RawResponsePersistencePort.java new file mode 100644 index 00000000..5057cac2 --- /dev/null +++ b/src/main/java/fr/insee/genesis/domain/ports/spi/RawResponsePersistencePort.java @@ -0,0 +1,15 @@ +package fr.insee.genesis.domain.ports.spi; + +import fr.insee.genesis.domain.model.surveyunit.Mode; +import fr.insee.genesis.domain.model.surveyunit.rawdata.RawResponse; + +import java.util.List; +import java.util.Set; + +public interface RawResponsePersistencePort { + + List findRawResponses(String questionnaireId, Mode mode, List interrogationIdList); + + + void updateProcessDates(String collectionInstrumentId, Set interrogationIds); +} diff --git a/src/main/java/fr/insee/genesis/domain/service/context/DataProcessingContextService.java b/src/main/java/fr/insee/genesis/domain/service/context/DataProcessingContextService.java index 9688a84a..9ea59dfa 100644 --- a/src/main/java/fr/insee/genesis/domain/service/context/DataProcessingContextService.java +++ b/src/main/java/fr/insee/genesis/domain/service/context/DataProcessingContextService.java @@ -174,6 +174,13 @@ public DataProcessingContextModel getContextByPartitionId(String partitionId){ ); } + @Override + public DataProcessingContextModel getContextByCollectionInstrumentId(String collectionInstrumentId){ + return DataProcessingContextMapper.INSTANCE.documentToModel( + dataProcessingContextPersistancePort.findByPartitionId(collectionInstrumentId) + ); + } + @Override public List getPartitionIds(boolean withReview){ List partitionIds = new ArrayList<>(); diff --git a/src/main/java/fr/insee/genesis/domain/service/rawdata/RawResponseService.java b/src/main/java/fr/insee/genesis/domain/service/rawdata/RawResponseService.java new file mode 100644 index 00000000..9f890b89 --- /dev/null +++ b/src/main/java/fr/insee/genesis/domain/service/rawdata/RawResponseService.java @@ -0,0 +1,225 @@ +package fr.insee.genesis.domain.service.rawdata; + +import fr.insee.bpm.metadata.model.VariablesMap; +import fr.insee.genesis.configuration.Config; +import fr.insee.genesis.controller.utils.ControllerUtils; +import fr.insee.genesis.domain.model.context.DataProcessingContextModel; +import fr.insee.genesis.domain.model.surveyunit.DataState; +import fr.insee.genesis.domain.model.surveyunit.Mode; +import fr.insee.genesis.domain.model.surveyunit.SurveyUnitModel; +import fr.insee.genesis.domain.model.surveyunit.rawdata.DataProcessResult; +import fr.insee.genesis.domain.model.surveyunit.rawdata.RawResponse; +import fr.insee.genesis.domain.ports.api.RawResponseApiPort; +import fr.insee.genesis.domain.ports.spi.RawResponsePersistencePort; +import fr.insee.genesis.domain.ports.spi.SurveyUnitQualityToolPort; +import fr.insee.genesis.domain.service.context.DataProcessingContextService; +import fr.insee.genesis.domain.service.metadata.QuestionnaireMetadataService; +import fr.insee.genesis.domain.service.surveyunit.SurveyUnitQualityService; +import fr.insee.genesis.domain.service.surveyunit.SurveyUnitService; +import fr.insee.genesis.exceptions.GenesisError; +import fr.insee.genesis.exceptions.GenesisException; +import fr.insee.genesis.infrastructure.utils.FileUtils; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.http.ResponseEntity; +import org.springframework.stereotype.Service; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +@Service +@Slf4j +public class RawResponseService implements RawResponseApiPort { + + private final ControllerUtils controllerUtils; + private final QuestionnaireMetadataService metadataService; + private final SurveyUnitService surveyUnitService; + private final SurveyUnitQualityService surveyUnitQualityService; + private final SurveyUnitQualityToolPort surveyUnitQualityToolPort; + private final DataProcessingContextService dataProcessingContextService; + private final FileUtils fileUtils; + private final Config config; + + @Qualifier("rawResponseMongoAdapter") + private final RawResponsePersistencePort rawResponsePersistencePort; + + public RawResponseService(ControllerUtils controllerUtils, QuestionnaireMetadataService metadataService, SurveyUnitService surveyUnitService, SurveyUnitQualityService surveyUnitQualityService, SurveyUnitQualityToolPort surveyUnitQualityToolPort, DataProcessingContextService dataProcessingContextService, FileUtils fileUtils, Config config, RawResponsePersistencePort rawResponsePersistencePort) { + this.controllerUtils = controllerUtils; + this.metadataService = metadataService; + this.surveyUnitService = surveyUnitService; + this.surveyUnitQualityService = surveyUnitQualityService; + this.surveyUnitQualityToolPort = surveyUnitQualityToolPort; + this.dataProcessingContextService = dataProcessingContextService; + this.fileUtils = fileUtils; + this.config = config; + this.rawResponsePersistencePort = rawResponsePersistencePort; + } + + @Override + public List getRawResponses(String questionnaireModelId, Mode mode, List interrogationIdList) { + return List.of(); + } + + @Override + public DataProcessResult processRawResponses(String collectionInstrumentId, List interrogationIdList, List errors) throws GenesisException { + int dataCount=0; + int formattedDataCount=0; + DataProcessingContextModel dataProcessingContext = + dataProcessingContextService.getContextByCollectionInstrumentId(collectionInstrumentId); + List modesList = controllerUtils.getModesList(collectionInstrumentId, null); + for (Mode mode : modesList) { + //Load and save metadata into database, throw exception if none + VariablesMap variablesMap = metadataService.loadAndSaveIfNotExists(collectionInstrumentId, collectionInstrumentId, mode, fileUtils, + errors).getVariables(); + if (variablesMap == null) { + throw new GenesisException(400, + "Error during metadata parsing for mode %s :%n%s" + .formatted(mode, errors.getLast().getMessage()) + ); + } + int totalBatchs = Math.ceilDiv(interrogationIdList.size() , config.getRawDataProcessingBatchSize()); + int batchNumber = 1; + List interrogationIdListForMode = new ArrayList<>(interrogationIdList); + while(!interrogationIdListForMode.isEmpty()){ + log.info("Processing raw data batch {}/{}", batchNumber, totalBatchs); + int maxIndex = Math.min(interrogationIdListForMode.size(), config.getRawDataProcessingBatchSize()); + List interrogationIdToProcess = interrogationIdListForMode.subList(0, maxIndex); + + List rawResponses = getRawResponses(collectionInstrumentId, mode, interrogationIdToProcess); + + List surveyUnitModels = convertRawResponse( + rawResponses, + variablesMap + ); + + //Save converted data + surveyUnitQualityService.verifySurveyUnits(surveyUnitModels, variablesMap); + surveyUnitService.saveSurveyUnits(surveyUnitModels); + + //Update process dates + updateProcessDates(surveyUnitModels); + + //Increment data count + dataCount += surveyUnitModels.size(); + formattedDataCount += surveyUnitModels.stream() + .filter(surveyUnitModel -> surveyUnitModel.getState().equals(DataState.FORMATTED)) + .toList() + .size(); + + //Send processed ids grouped by questionnaire (if review activated) + if(dataProcessingContext != null && dataProcessingContext.isWithReview()) { + sendProcessedIdsToQualityTool(surveyUnitModels); + } + + //Remove processed ids from list + interrogationIdListForMode = interrogationIdListForMode.subList(maxIndex, interrogationIdListForMode.size()); + + batchNumber++; + } + } + return null; + } + + @Override + public List convertRawResponse(List rawResponses, VariablesMap variablesMap) { +/* //Convert to genesis model + List surveyUnitModels = new ArrayList<>(); + //For each possible data state (we receive COLLECTED or EDITED) + for(DataState dataState : List.of(DataState.COLLECTED,DataState.EDITED)){ + for (RawResponse rawResponse : rawResponses) { + //Get optional fields + String contextualId = getContextualId(rawData); + Boolean isCapturedIndirectly = getIsCapturedIndirectly(rawData); + LocalDateTime validationDate = getValidationDate(rawData); + + SurveyUnitModel surveyUnitModel = SurveyUnitModel.builder() + .campaignId(rawData.campaignId()) + .questionnaireId(rawData.questionnaireId()) + .mode(rawData.mode()) + .interrogationId(rawData.interrogationId()) + .idUE(rawData.idUE()) + .contextualId(contextualId) + .validationDate(validationDate) + .isCapturedIndirectly(isCapturedIndirectly) + .state(dataState) + .fileDate(rawData.recordDate()) + .recordDate(LocalDateTime.now()) + .collectedVariables(new ArrayList<>()) + .externalVariables(new ArrayList<>()) + .build(); + + //Data collected variables conversion + convertRawDataCollectedVariables(rawData, surveyUnitModel, dataState, rawDataModelType, variablesMap); + + //External variables conversion into COLLECTED document + if(dataState == DataState.COLLECTED){ + convertRawDataExternalVariables(rawData, surveyUnitModel, rawDataModelType, variablesMap); + } + + boolean hasNoVariable = surveyUnitModel.getCollectedVariables().isEmpty() + && surveyUnitModel.getExternalVariables().isEmpty(); + + if(hasNoVariable){ + if(surveyUnitModel.getState() == DataState.COLLECTED){ + log.warn("No collected or external variable for interrogation {}, raw data is ignored.", rawData.interrogationId()); + } + continue;// don't add suModel + } + surveyUnitModels.add(surveyUnitModel); + } + } + + return surveyUnitModels;*/ + return List.of(); + } + + @Override + public void updateProcessDates(List surveyUnitModels) { + Set collectionInstrumentIds = new HashSet<>(); + for (SurveyUnitModel surveyUnitModel : surveyUnitModels) { + collectionInstrumentIds.add(surveyUnitModel.getQuestionnaireId()); + } + + for (String collectionInstrumentId : collectionInstrumentIds) { + Set interrogationIds = new HashSet<>(); + for (SurveyUnitModel surveyUnitModel : + surveyUnitModels.stream().filter( + surveyUnitModel -> surveyUnitModel.getQuestionnaireId().equals(collectionInstrumentId) + ).toList()) { + interrogationIds.add(surveyUnitModel.getInterrogationId()); + } + rawResponsePersistencePort.updateProcessDates(collectionInstrumentId, interrogationIds); + } + } + + private Map> getProcessedIdsMap(List surveyUnitModels) { + Map> processedInterrogationIdsPerQuestionnaire = new HashMap<>(); + surveyUnitModels.forEach(model -> + processedInterrogationIdsPerQuestionnaire + .computeIfAbsent(model.getQuestionnaireId(), k -> new HashSet<>()) + .add(model.getInterrogationId()) + ); + return processedInterrogationIdsPerQuestionnaire; + } + + private void sendProcessedIdsToQualityTool(List surveyUnitModels) { + try { + ResponseEntity response = + surveyUnitQualityToolPort.sendProcessedIds(getProcessedIdsMap(surveyUnitModels)); + + if (response.getStatusCode().is2xxSuccessful()) { + log.info("Successfully sent {} ids to quality tool", getProcessedIdsMap(surveyUnitModels).size()); + }else{ + log.warn("Survey unit quality tool responded non-2xx code {} and body {}", + response.getStatusCode(), response.getBody()); + } + }catch (IOException e){ + log.error("Error during Perret call request building : {}", e.toString()); + } + } +} diff --git a/src/main/java/fr/insee/genesis/infrastructure/adapter/DataProcessingContextMongoAdapter.java b/src/main/java/fr/insee/genesis/infrastructure/adapter/DataProcessingContextMongoAdapter.java index f104965a..930135dc 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/adapter/DataProcessingContextMongoAdapter.java +++ b/src/main/java/fr/insee/genesis/infrastructure/adapter/DataProcessingContextMongoAdapter.java @@ -49,6 +49,13 @@ public List findByPartitionIds(List partitio return DataProcessingContextMapper.INSTANCE.listDocumentToListModel(ContextDedupUtils.deduplicateContexts(existingDocuments)); } + @Override + public List findByCollectionInstrumentIds(List collectionInstrumentIds) { + List existingDocuments = + dataProcessingContextMongoDBRepository.findByCollectionInstrumentIdList(collectionInstrumentIds); + return DataProcessingContextMapper.INSTANCE.listDocumentToListModel(ContextDedupUtils.deduplicateContexts(existingDocuments)); + } + @Override public void save(DataProcessingContextDocument dataProcessingContextDocument) { dataProcessingContextMongoDBRepository.save(dataProcessingContextDocument); diff --git a/src/main/java/fr/insee/genesis/infrastructure/adapter/RawResponseMongoAdapter.java b/src/main/java/fr/insee/genesis/infrastructure/adapter/RawResponseMongoAdapter.java new file mode 100644 index 00000000..1b1b35b7 --- /dev/null +++ b/src/main/java/fr/insee/genesis/infrastructure/adapter/RawResponseMongoAdapter.java @@ -0,0 +1,49 @@ +package fr.insee.genesis.infrastructure.adapter; + +import fr.insee.genesis.Constants; +import fr.insee.genesis.domain.model.surveyunit.Mode; +import fr.insee.genesis.domain.model.surveyunit.rawdata.RawResponse; +import fr.insee.genesis.domain.ports.spi.RawResponsePersistencePort; +import fr.insee.genesis.infrastructure.document.rawdata.RawResponseDocument; +import fr.insee.genesis.infrastructure.mappers.RawResponseDocumentMapper; +import fr.insee.genesis.infrastructure.repository.RawResponseRepository; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.data.mongodb.core.MongoTemplate; +import org.springframework.data.mongodb.core.query.Criteria; +import org.springframework.data.mongodb.core.query.Query; +import org.springframework.data.mongodb.core.query.Update; +import org.springframework.stereotype.Service; + +import java.time.LocalDateTime; +import java.util.List; +import java.util.Set; + +@Service +@Slf4j +@Qualifier("rawResponseMongoAdapter") +public class RawResponseMongoAdapter implements RawResponsePersistencePort { + + private final RawResponseRepository repository; + private final MongoTemplate mongoTemplate; + + public RawResponseMongoAdapter(RawResponseRepository repository, MongoTemplate mongoTemplate) { + this.repository = repository; + this.mongoTemplate = mongoTemplate; + } + + @Override + public List findRawResponses(String questionnaireId, Mode mode, List interrogationIdList) { + List rawDataDocs = repository.findByCollectionInstrumentIdAndModeAndInterrogationIdList(questionnaireId, mode, interrogationIdList); + return RawResponseDocumentMapper.INSTANCE.listDocumentToListModel(rawDataDocs); + } + + @Override + public void updateProcessDates(String collectionInstrumentId, Set interrogationIds) { + mongoTemplate.updateMulti( + Query.query(Criteria.where("collectionInstrumentId").is(collectionInstrumentId).and("interrogationId").in(interrogationIds)) + , new Update().set("processDate", LocalDateTime.now()) + , Constants.MONGODB_LUNATIC_RAWDATA_COLLECTION_NAME + ); + } +} diff --git a/src/main/java/fr/insee/genesis/infrastructure/document/context/DataProcessingContextDocument.java b/src/main/java/fr/insee/genesis/infrastructure/document/context/DataProcessingContextDocument.java index cdfda072..37a43cfd 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/document/context/DataProcessingContextDocument.java +++ b/src/main/java/fr/insee/genesis/infrastructure/document/context/DataProcessingContextDocument.java @@ -24,6 +24,7 @@ public DataProcessingContextDocument(String partitionId, @Id private ObjectId id; private String partitionId; //ex Survey Name, campaignId + private String collectionInstrumentId; // QuestionnaireId private LocalDateTime lastExecution; private List kraftwerkExecutionScheduleList; private boolean withReview; diff --git a/src/main/java/fr/insee/genesis/infrastructure/document/rawdata/RawResponseDocument.java b/src/main/java/fr/insee/genesis/infrastructure/document/rawdata/RawResponseDocument.java new file mode 100644 index 00000000..82eb6fe5 --- /dev/null +++ b/src/main/java/fr/insee/genesis/infrastructure/document/rawdata/RawResponseDocument.java @@ -0,0 +1,26 @@ +package fr.insee.genesis.infrastructure.document.rawdata; + +import fr.insee.genesis.domain.model.surveyunit.Mode; +import lombok.Builder; +import org.bson.types.ObjectId; +import org.springframework.data.annotation.Id; +import org.springframework.data.mongodb.core.index.IndexDirection; +import org.springframework.data.mongodb.core.index.Indexed; +import org.springframework.data.mongodb.core.mapping.Document; + +import java.time.LocalDateTime; +import java.util.Map; + +@Builder +@Document(collection = "rawResponses") +public record RawResponseDocument ( + @Id + ObjectId id, + String interrogationId, + String collectionInstrumentId, + Mode mode, + Map payload, + LocalDateTime recordDate, + @Indexed(direction = IndexDirection.DESCENDING) + LocalDateTime processDate +){} diff --git a/src/main/java/fr/insee/genesis/infrastructure/mappers/RawResponseDocumentMapper.java b/src/main/java/fr/insee/genesis/infrastructure/mappers/RawResponseDocumentMapper.java new file mode 100644 index 00000000..b72f6021 --- /dev/null +++ b/src/main/java/fr/insee/genesis/infrastructure/mappers/RawResponseDocumentMapper.java @@ -0,0 +1,19 @@ +package fr.insee.genesis.infrastructure.mappers; + +import fr.insee.genesis.domain.model.surveyunit.rawdata.RawResponse; +import fr.insee.genesis.infrastructure.document.rawdata.RawResponseDocument; +import org.mapstruct.Mapper; +import org.mapstruct.factory.Mappers; + +import java.util.List; + +@Mapper +public interface RawResponseDocumentMapper { + + RawResponseDocumentMapper INSTANCE = Mappers.getMapper(RawResponseDocumentMapper.class); + + RawResponse documentToModel(RawResponseDocument document); + RawResponseDocument modelToDocument(RawResponse model); + List listDocumentToListModel(List documentList); + List listModelToListDocument(List modelList); +} diff --git a/src/main/java/fr/insee/genesis/infrastructure/repository/DataProcessingContextMongoDBRepository.java b/src/main/java/fr/insee/genesis/infrastructure/repository/DataProcessingContextMongoDBRepository.java index 35eb37a8..b0fe29fd 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/repository/DataProcessingContextMongoDBRepository.java +++ b/src/main/java/fr/insee/genesis/infrastructure/repository/DataProcessingContextMongoDBRepository.java @@ -12,6 +12,9 @@ public interface DataProcessingContextMongoDBRepository extends MongoRepository< @Query(value = "{ 'partitionId' : {$in: ?0} }") List findByPartitionIdList(List partitionIds); + @Query(value = "{ 'collectionInstrumentId' : {$in: ?0} }") + List findByCollectionInstrumentIdList(List collectionInstrumentIds); + @Query(value = "{ 'partitionId' : ?0 }", delete = true) void deleteByPartitionId(String partitionId); } diff --git a/src/main/java/fr/insee/genesis/infrastructure/repository/RawResponseInputRepository.java b/src/main/java/fr/insee/genesis/infrastructure/repository/RawResponseInputRepository.java index 96242a0b..85a34ec5 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/repository/RawResponseInputRepository.java +++ b/src/main/java/fr/insee/genesis/infrastructure/repository/RawResponseInputRepository.java @@ -30,6 +30,7 @@ public void saveAsRawJson(RawResponseDto dto) { Map document = new java.util.HashMap<>(); document.put("interrogationId", dto.getInterrogationId()); document.put("collectionInstrumentId", dto.getCollectionInstrumentId()); + document.put("mode", dto.getMode()); document.put("recordDate", Instant.now()); document.put("payload", payload); mongoTemplate.save(document, "rawResponses"); diff --git a/src/main/java/fr/insee/genesis/infrastructure/repository/RawResponseRepository.java b/src/main/java/fr/insee/genesis/infrastructure/repository/RawResponseRepository.java new file mode 100644 index 00000000..d2f9d1e0 --- /dev/null +++ b/src/main/java/fr/insee/genesis/infrastructure/repository/RawResponseRepository.java @@ -0,0 +1,16 @@ +package fr.insee.genesis.infrastructure.repository; + +import fr.insee.genesis.domain.model.surveyunit.Mode; +import fr.insee.genesis.infrastructure.document.rawdata.RawResponseDocument; +import org.springframework.data.mongodb.repository.MongoRepository; +import org.springframework.data.mongodb.repository.Query; +import org.springframework.stereotype.Repository; + +import java.util.List; + +@Repository +public interface RawResponseRepository extends MongoRepository { + + @Query(value = "{ 'collectionInstrumentId' : ?0, 'mode' : ?1, 'interrogationId': {$in: ?2} }") + List findByCollectionInstrumentIdAndModeAndInterrogationIdList(String questionnaireId, Mode mode, List interrogationIdList); +} diff --git a/src/test/java/cucumber/functional_tests/RawDataDefinitions.java b/src/test/java/cucumber/functional_tests/RawDataDefinitions.java index 05479ba3..8fe9a8cd 100644 --- a/src/test/java/cucumber/functional_tests/RawDataDefinitions.java +++ b/src/test/java/cucumber/functional_tests/RawDataDefinitions.java @@ -1,6 +1,7 @@ package cucumber.functional_tests; import cucumber.functional_tests.config.CucumberSpringConfiguration; +import fr.insee.bpm.metadata.model.VariablesMap; import fr.insee.genesis.TestConstants; import fr.insee.genesis.configuration.Config; import fr.insee.genesis.controller.rest.responses.RawResponseController; @@ -9,12 +10,17 @@ import fr.insee.genesis.domain.model.surveyunit.Mode; import fr.insee.genesis.domain.model.surveyunit.SurveyUnitModel; import fr.insee.genesis.domain.model.surveyunit.VariableModel; +import fr.insee.genesis.domain.model.surveyunit.rawdata.DataProcessResult; +import fr.insee.genesis.domain.model.surveyunit.rawdata.RawResponse; +import fr.insee.genesis.domain.ports.api.RawResponseApiPort; import fr.insee.genesis.domain.service.context.DataProcessingContextService; import fr.insee.genesis.domain.service.metadata.QuestionnaireMetadataService; import fr.insee.genesis.domain.service.rawdata.LunaticJsonRawDataService; import fr.insee.genesis.domain.service.surveyunit.SurveyUnitQualityService; import fr.insee.genesis.domain.service.surveyunit.SurveyUnitService; import fr.insee.genesis.domain.utils.JsonUtils; +import fr.insee.genesis.exceptions.GenesisError; +import fr.insee.genesis.exceptions.GenesisException; import fr.insee.genesis.infrastructure.repository.RawResponseInputRepository; import fr.insee.genesis.infrastructure.utils.FileUtils; import fr.insee.genesis.stubs.ConfigStub; @@ -90,14 +96,37 @@ public class RawDataDefinitions { config, dataProcessingContextPersistancePortStub); + // TODO : implements real stubs RawResponseInputRepository rawResponseInputRepositoryStub = new RawResponseInputRepository(null, null) { @Override public void saveAsRawJson(RawResponseDto dto) { // Ne rien faire — stub pour les tests } }; + + RawResponseApiPort rawResponseApiPortStub = new RawResponseApiPort() { + @Override + public List getRawResponses(String questionnaireModelId, Mode mode, List interrogationIdList) { + return List.of(); + } + + @Override + public DataProcessResult processRawResponses(String questionnaireId, List interrogationIdList, List errors) throws GenesisException { + return null; + } + + @Override + public List convertRawResponse(List rawResponses, VariablesMap variablesMap) { + return List.of(); + } + + @Override + public void updateProcessDates(List surveyUnitModels) { + + } + }; RawResponseController rawResponseController = new RawResponseController( - lunaticJsonRawDataService, rawResponseInputRepositoryStub + lunaticJsonRawDataService, rawResponseApiPortStub, rawResponseInputRepositoryStub ); Path rawDataFilePath; String rawJsonData; diff --git a/src/test/java/fr/insee/genesis/controller/rest/responses/RawResponseControllerTest.java b/src/test/java/fr/insee/genesis/controller/rest/responses/RawResponseControllerTest.java index 0da8a3c1..54780413 100644 --- a/src/test/java/fr/insee/genesis/controller/rest/responses/RawResponseControllerTest.java +++ b/src/test/java/fr/insee/genesis/controller/rest/responses/RawResponseControllerTest.java @@ -1,19 +1,26 @@ package fr.insee.genesis.controller.rest.responses; +import fr.insee.bpm.metadata.model.VariablesMap; import fr.insee.genesis.controller.dto.rawdata.LunaticJsonRawDataUnprocessedDto; import fr.insee.genesis.controller.utils.ControllerUtils; import fr.insee.genesis.domain.model.context.DataProcessingContextModel; import fr.insee.genesis.domain.model.surveyunit.DataState; import fr.insee.genesis.domain.model.surveyunit.Mode; +import fr.insee.genesis.domain.model.surveyunit.SurveyUnitModel; +import fr.insee.genesis.domain.model.surveyunit.rawdata.DataProcessResult; import fr.insee.genesis.domain.model.surveyunit.rawdata.LunaticJsonRawDataModel; +import fr.insee.genesis.domain.model.surveyunit.rawdata.RawResponse; import fr.insee.genesis.domain.ports.api.LunaticJsonRawDataApiPort; +import fr.insee.genesis.domain.ports.api.RawResponseApiPort; import fr.insee.genesis.domain.service.context.DataProcessingContextService; import fr.insee.genesis.domain.service.metadata.QuestionnaireMetadataService; import fr.insee.genesis.domain.service.rawdata.LunaticJsonRawDataService; import fr.insee.genesis.domain.service.surveyunit.SurveyUnitQualityService; import fr.insee.genesis.domain.service.surveyunit.SurveyUnitService; import fr.insee.genesis.domain.utils.JsonUtils; +import fr.insee.genesis.exceptions.GenesisError; +import fr.insee.genesis.exceptions.GenesisException; import fr.insee.genesis.infrastructure.document.rawdata.LunaticJsonRawDataDocument; import fr.insee.genesis.infrastructure.mappers.DataProcessingContextMapper; import fr.insee.genesis.infrastructure.repository.RawResponseInputRepository; @@ -68,7 +75,29 @@ public void saveAsRawJson(RawResponseDto dto) { } }; - private final RawResponseController rawResponseController = new RawResponseController(lunaticJsonRawDataApiPort, rawResponseInputRepositoryStub); + RawResponseApiPort rawResponseApiPortStub = new RawResponseApiPort() { + @Override + public List getRawResponses(String questionnaireModelId, Mode mode, List interrogationIdList) { + return List.of(); + } + + @Override + public DataProcessResult processRawResponses(String questionnaireId, List interrogationIdList, List errors) throws GenesisException { + return null; + } + + @Override + public List convertRawResponse(List rawResponses, VariablesMap variablesMap) { + return List.of(); + } + + @Override + public void updateProcessDates(List surveyUnitModels) { + + } + }; + + private final RawResponseController rawResponseController = new RawResponseController(lunaticJsonRawDataApiPort, rawResponseApiPortStub, rawResponseInputRepositoryStub); @Test From 4bb92718c00dd2c1937e4dea9278915eb398f901 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Henninger?= Date: Mon, 17 Nov 2025 17:55:30 +0100 Subject: [PATCH 08/42] feat: WIP new modele filiere --- .../adapter/LunaticJsonAdapter.java | 2 +- .../controller/adapter/LunaticXmlAdapter.java | 2 +- .../rest/responses/RawResponseController.java | 11 +- .../rest/responses/ResponseController.java | 10 +- .../genesis/domain/model/surveyunit/Mode.java | 20 +- .../model/surveyunit/SurveyUnitModel.java | 12 +- .../ports/spi/RawResponsePersistencePort.java | 4 +- .../rawdata/LunaticJsonRawDataService.java | 7 +- .../service/rawdata/RawResponseService.java | 196 ++++++++++++++++-- .../service/surveyunit/SurveyUnitService.java | 6 +- .../genesis/domain/utils/DataVerifier.java | 4 +- .../adapter/RawResponseMongoAdapter.java | 4 +- .../document/rawdata/RawResponseDocument.java | 3 +- .../surveyunit/SurveyUnitDocument.java | 23 ++ .../mappers/RawResponseDocumentMapper.java | 11 + .../mappers/SurveyUnitDocumentMapper.java | 16 ++ .../repository/RawResponseRepository.java | 3 +- .../LunaticModelDefinitions.java | 2 +- .../functional_tests/RawDataDefinitions.java | 5 +- .../adapter/LunaticXmlAdapterTest.java | 2 +- .../controller/rest/ControllerAccessTest.java | 10 +- .../DataProcessingContextControllerTest.java | 3 + .../rest/HealthCheckControllerTest.java | 2 +- .../controller/rest/UtilsControllerTest.java | 4 +- .../responses/RawResponseControllerTest.java | 4 +- .../responses/ResponseControllerTest.java | 32 +-- .../controller/rest/responses/Utils.java | 8 +- .../surveyunit/SurveyUnitServiceTest.java | 24 +-- .../domain/utils/DataVerifierTest.java | 14 +- .../SurveyUnitDocumentMapperImplTest.java | 12 +- ...aProcessingContextPersistancePortStub.java | 5 + .../stubs/SurveyUnitPersistencePortStub.java | 16 +- .../features/do_we_process_raw_data.feature | 2 +- 33 files changed, 359 insertions(+), 120 deletions(-) diff --git a/src/main/java/fr/insee/genesis/controller/adapter/LunaticJsonAdapter.java b/src/main/java/fr/insee/genesis/controller/adapter/LunaticJsonAdapter.java index edded928..33178f1c 100644 --- a/src/main/java/fr/insee/genesis/controller/adapter/LunaticJsonAdapter.java +++ b/src/main/java/fr/insee/genesis/controller/adapter/LunaticJsonAdapter.java @@ -13,7 +13,7 @@ public class LunaticJsonAdapter { public SurveyUnitModel convert(LunaticJsonSurveyUnit su){ return SurveyUnitModel.builder() - .questionnaireId(su.getQuestionnaireId()) + .collectionInstrumentId(su.getQuestionnaireId()) .campaignId("") .interrogationId(su.getInterrogationId()) .state(DataState.COLLECTED) diff --git a/src/main/java/fr/insee/genesis/controller/adapter/LunaticXmlAdapter.java b/src/main/java/fr/insee/genesis/controller/adapter/LunaticXmlAdapter.java index 143965f5..ee78e099 100644 --- a/src/main/java/fr/insee/genesis/controller/adapter/LunaticXmlAdapter.java +++ b/src/main/java/fr/insee/genesis/controller/adapter/LunaticXmlAdapter.java @@ -69,7 +69,7 @@ public static List convert(LunaticXmlSurveyUnit su, VariablesMa */ private static SurveyUnitModel getStateDataFromSurveyUnit(LunaticXmlSurveyUnit su, VariablesMap variablesMap, String campaignId, DataState dataState, Mode mode) { SurveyUnitModel surveyUnitModel = SurveyUnitModel.builder() - .questionnaireId(su.getQuestionnaireModelId().toUpperCase()) + .collectionInstrumentId(su.getQuestionnaireModelId().toUpperCase()) .campaignId(campaignId) .interrogationId(su.getId()) .state(dataState) diff --git a/src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseController.java b/src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseController.java index cd13c1de..31432317 100644 --- a/src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseController.java +++ b/src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseController.java @@ -21,6 +21,7 @@ import fr.insee.modelefiliere.RawResponseDto; import io.swagger.v3.oas.annotations.Hidden; import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.Parameter; import jakarta.validation.Valid; import lombok.extern.slf4j.Slf4j; import org.springframework.data.domain.Page; @@ -194,14 +195,18 @@ public ResponseEntity saveRawResponsesFromRawResponseDto( @PostMapping(path = "/raw-responses/process") @PreAuthorize("hasRole('SCHEDULER')") public ResponseEntity processRawResponses( - @RequestParam("questionnaireId") String questionnaireId, + @Parameter( + description = "Id of the collection instrument (old questionnaireId)", + example = "ENQTEST2025X00" + ) + @RequestParam("collectionInstrumentId") String collectionInstrumentId, @RequestBody List interrogationIdList ) { - log.info("Try to process raw responses for questionnaireId {} and {} interrogationIds", interrogationIdList.size()); + log.info("Try to process raw responses for questionnaireId {} and {} interrogationIds", collectionInstrumentId, interrogationIdList.size()); List errors = new ArrayList<>(); try { - DataProcessResult result = rawResponseApiPort.processRawResponses(questionnaireId, interrogationIdList, errors); + DataProcessResult result = rawResponseApiPort.processRawResponses(collectionInstrumentId, interrogationIdList, errors); return result.formattedDataCount() == 0 ? ResponseEntity.ok("%d document(s) processed".formatted(result.dataCount())) : ResponseEntity.ok("%d document(s) processed, including %d FORMATTED after data verification" diff --git a/src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java b/src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java index c228535b..1627c318 100644 --- a/src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java +++ b/src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java @@ -248,10 +248,10 @@ public ResponseEntity getLatestByInterrogationOneObject(@R outputExternalVariables.addAll(response.getExternalVariables()); }); return ResponseEntity.ok(SurveyUnitSimplified.builder() - .questionnaireId(responses.getFirst().getQuestionnaireId()) + .questionnaireId(responses.getFirst().getCollectionInstrumentId()) .campaignId(responses.getFirst().getCampaignId()) .interrogationId(responses.getFirst().getInterrogationId()) - .surveyUnitId(responses.getFirst().getIdUE()) + .surveyUnitId(responses.getFirst().getUsualSurveyUnitId()) .variablesUpdate(outputVariables) .externalVariables(outputExternalVariables) .build()); @@ -278,10 +278,10 @@ public ResponseEntity> getLatestForInterrogationList( }); if (!outputVariables.isEmpty() || !outputExternalVariables.isEmpty()) { results.add(SurveyUnitSimplified.builder() - .questionnaireId(responses.getFirst().getQuestionnaireId()) + .questionnaireId(responses.getFirst().getCollectionInstrumentId()) .campaignId(responses.getFirst().getCampaignId()) .interrogationId(responses.getFirst().getInterrogationId()) - .surveyUnitId(responses.getFirst().getIdUE()) + .surveyUnitId(responses.getFirst().getUsualSurveyUnitId()) .mode(mode) .variablesUpdate(outputVariables) .externalVariables(outputExternalVariables) @@ -358,7 +358,7 @@ private SurveyUnitSimplified fusionWithLastUpdated(List respons Mode modeWrapped = Mode.getEnumFromModeName(mode); simplifiedResponse = SurveyUnitSimplified.builder() - .questionnaireId(responsesForSingleInterrId.getFirst().getQuestionnaireId()) + .questionnaireId(responsesForSingleInterrId.getFirst().getCollectionInstrumentId()) .campaignId(responsesForSingleInterrId.getFirst().getCampaignId()) .interrogationId(responsesForSingleInterrId.getFirst().getInterrogationId()) .mode(modeWrapped) diff --git a/src/main/java/fr/insee/genesis/domain/model/surveyunit/Mode.java b/src/main/java/fr/insee/genesis/domain/model/surveyunit/Mode.java index cbf45b86..ad25a45b 100644 --- a/src/main/java/fr/insee/genesis/domain/model/surveyunit/Mode.java +++ b/src/main/java/fr/insee/genesis/domain/model/surveyunit/Mode.java @@ -1,5 +1,6 @@ package fr.insee.genesis.domain.model.surveyunit; +import com.fasterxml.jackson.annotation.JsonCreator; import io.swagger.v3.oas.annotations.media.Schema; import lombok.Getter; import org.springframework.lang.Nullable; @@ -7,7 +8,11 @@ @Getter public enum Mode { - WEB("WEB", "WEB", "CAWI"),TEL("TEL", "ENQ", "CATI"), F2F("F2F", "ENQ", "CAPI"),OTHER("OTHER", "", ""),PAPER("PAPER", "", "PAPI"); + WEB("WEB", "WEB", "CAWI"), + TEL("TEL", "ENQ", "CATI"), + F2F("F2F", "ENQ", "CAPI"), + OTHER("OTHER", "", ""), + PAPER("PAPER", "", "PAPI"); @Nullable @Schema(nullable = true, type = "string", allowableValues = { "WEB", "TEL", "F2F", "PAPER", "OTHER" }) @@ -23,6 +28,19 @@ public enum Mode { this.jsonName = jsonName; } + @JsonCreator + public static Mode fromString(String value) { + if (value == null) return null; + + for (Mode m : values()) { + if (value.equalsIgnoreCase(m.modeName) || + value.equalsIgnoreCase(m.jsonName)) { + return m; + } + } + throw new IllegalArgumentException("Invalid Mode: " + value); + } + public static Mode getEnumFromModeName(String modeName) { if (modeName == null){ return null; diff --git a/src/main/java/fr/insee/genesis/domain/model/surveyunit/SurveyUnitModel.java b/src/main/java/fr/insee/genesis/domain/model/surveyunit/SurveyUnitModel.java index cac82c6d..307d592d 100644 --- a/src/main/java/fr/insee/genesis/domain/model/surveyunit/SurveyUnitModel.java +++ b/src/main/java/fr/insee/genesis/domain/model/surveyunit/SurveyUnitModel.java @@ -21,14 +21,18 @@ @AllArgsConstructor public class SurveyUnitModel { - - private String questionnaireId; + // New name of questionnaireId + private String collectionInstrumentId; + // To be removed private String campaignId; private String interrogationId; - private String idUE; + // New name of idUE + private String usualSurveyUnitId; + private String technicalSurveyUnitId; + // Represents the major version of the "modèle filière" + private String majorModelVersion; private DataState state; private Mode mode; - private String contextualId; private Boolean isCapturedIndirectly; private LocalDateTime validationDate; @JsonFormat(shape = JsonFormat.Shape.STRING, pattern = "yyyy-MM-dd'T'hh:mm") diff --git a/src/main/java/fr/insee/genesis/domain/ports/spi/RawResponsePersistencePort.java b/src/main/java/fr/insee/genesis/domain/ports/spi/RawResponsePersistencePort.java index 5057cac2..3d79e88c 100644 --- a/src/main/java/fr/insee/genesis/domain/ports/spi/RawResponsePersistencePort.java +++ b/src/main/java/fr/insee/genesis/domain/ports/spi/RawResponsePersistencePort.java @@ -8,8 +8,6 @@ public interface RawResponsePersistencePort { - List findRawResponses(String questionnaireId, Mode mode, List interrogationIdList); - - + List findRawResponses(String collectionInstrumentId, Mode mode, List interrogationIdList); void updateProcessDates(String collectionInstrumentId, Set interrogationIds); } diff --git a/src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java b/src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java index 07aa8b67..64d25d00 100644 --- a/src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java +++ b/src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java @@ -178,7 +178,7 @@ private Map> getProcessedIdsMap(List survey Map> processedInterrogationIdsPerQuestionnaire = new HashMap<>(); surveyUnitModels.forEach(model -> processedInterrogationIdsPerQuestionnaire - .computeIfAbsent(model.getQuestionnaireId(), k -> new HashSet<>()) + .computeIfAbsent(model.getCollectionInstrumentId(), k -> new HashSet<>()) .add(model.getInterrogationId()) ); return processedInterrogationIdsPerQuestionnaire; @@ -200,11 +200,10 @@ public List convertRawData(List rawDat SurveyUnitModel surveyUnitModel = SurveyUnitModel.builder() .campaignId(rawData.campaignId()) - .questionnaireId(rawData.questionnaireId()) + .collectionInstrumentId(rawData.questionnaireId()) .mode(rawData.mode()) .interrogationId(rawData.interrogationId()) - .idUE(rawData.idUE()) - .contextualId(contextualId) + .usualSurveyUnitId(rawData.idUE()) .validationDate(validationDate) .isCapturedIndirectly(isCapturedIndirectly) .state(dataState) diff --git a/src/main/java/fr/insee/genesis/domain/service/rawdata/RawResponseService.java b/src/main/java/fr/insee/genesis/domain/service/rawdata/RawResponseService.java index 9f890b89..bfa7e924 100644 --- a/src/main/java/fr/insee/genesis/domain/service/rawdata/RawResponseService.java +++ b/src/main/java/fr/insee/genesis/domain/service/rawdata/RawResponseService.java @@ -1,12 +1,14 @@ package fr.insee.genesis.domain.service.rawdata; import fr.insee.bpm.metadata.model.VariablesMap; +import fr.insee.genesis.Constants; import fr.insee.genesis.configuration.Config; import fr.insee.genesis.controller.utils.ControllerUtils; import fr.insee.genesis.domain.model.context.DataProcessingContextModel; import fr.insee.genesis.domain.model.surveyunit.DataState; import fr.insee.genesis.domain.model.surveyunit.Mode; import fr.insee.genesis.domain.model.surveyunit.SurveyUnitModel; +import fr.insee.genesis.domain.model.surveyunit.VariableModel; import fr.insee.genesis.domain.model.surveyunit.rawdata.DataProcessResult; import fr.insee.genesis.domain.model.surveyunit.rawdata.RawResponse; import fr.insee.genesis.domain.ports.api.RawResponseApiPort; @@ -16,6 +18,8 @@ import fr.insee.genesis.domain.service.metadata.QuestionnaireMetadataService; import fr.insee.genesis.domain.service.surveyunit.SurveyUnitQualityService; import fr.insee.genesis.domain.service.surveyunit.SurveyUnitService; +import fr.insee.genesis.domain.utils.GroupUtils; +import fr.insee.genesis.domain.utils.JsonUtils; import fr.insee.genesis.exceptions.GenesisError; import fr.insee.genesis.exceptions.GenesisException; import fr.insee.genesis.infrastructure.utils.FileUtils; @@ -25,6 +29,7 @@ import org.springframework.stereotype.Service; import java.io.IOException; +import java.time.LocalDateTime; import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; @@ -61,8 +66,8 @@ public RawResponseService(ControllerUtils controllerUtils, QuestionnaireMetadata } @Override - public List getRawResponses(String questionnaireModelId, Mode mode, List interrogationIdList) { - return List.of(); + public List getRawResponses(String collectionInstrumentId, Mode mode, List interrogationIdList) { + return rawResponsePersistencePort.findRawResponses(collectionInstrumentId,mode,interrogationIdList); } @Override @@ -122,43 +127,43 @@ public DataProcessResult processRawResponses(String collectionInstrumentId, List batchNumber++; } } - return null; + return new DataProcessResult(dataCount, formattedDataCount); } @Override public List convertRawResponse(List rawResponses, VariablesMap variablesMap) { -/* //Convert to genesis model + //Convert to genesis model List surveyUnitModels = new ArrayList<>(); //For each possible data state (we receive COLLECTED or EDITED) for(DataState dataState : List.of(DataState.COLLECTED,DataState.EDITED)){ for (RawResponse rawResponse : rawResponses) { //Get optional fields - String contextualId = getContextualId(rawData); - Boolean isCapturedIndirectly = getIsCapturedIndirectly(rawData); - LocalDateTime validationDate = getValidationDate(rawData); + Boolean isCapturedIndirectly = getIsCapturedIndirectly(rawResponse); + LocalDateTime validationDate = getValidationDate(rawResponse); + String usualSurveyUnitId = getStringFieldInPayload(rawResponse,"usualSurveyUnitId"); + String majorModelVersion = getStringFieldInPayload(rawResponse, "majorModelVersion"); SurveyUnitModel surveyUnitModel = SurveyUnitModel.builder() - .campaignId(rawData.campaignId()) - .questionnaireId(rawData.questionnaireId()) - .mode(rawData.mode()) - .interrogationId(rawData.interrogationId()) - .idUE(rawData.idUE()) - .contextualId(contextualId) + .collectionInstrumentId(rawResponse.collectionInstrumentId()) + .majorModelVersion(majorModelVersion) + .mode(rawResponse.mode()) + .interrogationId(rawResponse.interrogationId()) + .usualSurveyUnitId(usualSurveyUnitId) .validationDate(validationDate) .isCapturedIndirectly(isCapturedIndirectly) .state(dataState) - .fileDate(rawData.recordDate()) + .fileDate(rawResponse.recordDate()) .recordDate(LocalDateTime.now()) .collectedVariables(new ArrayList<>()) .externalVariables(new ArrayList<>()) .build(); //Data collected variables conversion - convertRawDataCollectedVariables(rawData, surveyUnitModel, dataState, rawDataModelType, variablesMap); + convertRawDataCollectedVariables(rawResponse, surveyUnitModel, dataState, variablesMap); //External variables conversion into COLLECTED document if(dataState == DataState.COLLECTED){ - convertRawDataExternalVariables(rawData, surveyUnitModel, rawDataModelType, variablesMap); + convertRawDataExternalVariables(rawResponse, surveyUnitModel, variablesMap); } boolean hasNoVariable = surveyUnitModel.getCollectedVariables().isEmpty() @@ -166,30 +171,29 @@ public List convertRawResponse(List rawResponses, if(hasNoVariable){ if(surveyUnitModel.getState() == DataState.COLLECTED){ - log.warn("No collected or external variable for interrogation {}, raw data is ignored.", rawData.interrogationId()); + log.warn("No collected or external variable for interrogation {}, raw data is ignored.", rawResponse.interrogationId()); } continue;// don't add suModel } surveyUnitModels.add(surveyUnitModel); } } - - return surveyUnitModels;*/ - return List.of(); + return surveyUnitModels; + //return List.of(); } @Override public void updateProcessDates(List surveyUnitModels) { Set collectionInstrumentIds = new HashSet<>(); for (SurveyUnitModel surveyUnitModel : surveyUnitModels) { - collectionInstrumentIds.add(surveyUnitModel.getQuestionnaireId()); + collectionInstrumentIds.add(surveyUnitModel.getCollectionInstrumentId()); } for (String collectionInstrumentId : collectionInstrumentIds) { Set interrogationIds = new HashSet<>(); for (SurveyUnitModel surveyUnitModel : surveyUnitModels.stream().filter( - surveyUnitModel -> surveyUnitModel.getQuestionnaireId().equals(collectionInstrumentId) + surveyUnitModel -> surveyUnitModel.getCollectionInstrumentId().equals(collectionInstrumentId) ).toList()) { interrogationIds.add(surveyUnitModel.getInterrogationId()); } @@ -201,7 +205,7 @@ private Map> getProcessedIdsMap(List survey Map> processedInterrogationIdsPerQuestionnaire = new HashMap<>(); surveyUnitModels.forEach(model -> processedInterrogationIdsPerQuestionnaire - .computeIfAbsent(model.getQuestionnaireId(), k -> new HashSet<>()) + .computeIfAbsent(model.getCollectionInstrumentId(), k -> new HashSet<>()) .add(model.getInterrogationId()) ); return processedInterrogationIdsPerQuestionnaire; @@ -222,4 +226,150 @@ private void sendProcessedIdsToQualityTool(List surveyUnitModel log.error("Error during Perret call request building : {}", e.toString()); } } + + private static Boolean getIsCapturedIndirectly(RawResponse rawResponse) { + try{ + return rawResponse.payload().get("isCapturedIndirectly") == null ? null : + Boolean.parseBoolean(rawResponse.payload().get("isCapturedIndirectly").toString()); + }catch(Exception e){ + log.warn("Exception when parsing isCapturedIndirectly : {}}",e.toString()); + return Boolean.FALSE; + } + } + + private static LocalDateTime getValidationDate(RawResponse rawResponse) { + try{ + return rawResponse.payload().get("validationDate") == null ? null : + LocalDateTime.parse(rawResponse.payload().get("validationDate").toString()); + }catch(Exception e){ + log.warn("Exception when parsing validation date : {}}",e.toString()); + return null; + } + } + + private static String getStringFieldInPayload(RawResponse rawResponse, String field) { + try{ + return rawResponse.payload().get(field).toString(); + }catch(Exception e){ + log.warn("Exception when parsing {} : {}",field, e.toString()); + return null; + } + } + + @SuppressWarnings("unchecked") + private void convertRawDataCollectedVariables( + RawResponse rawResponse, + SurveyUnitModel dstSurveyUnitModel, + DataState dataState, + VariablesMap variablesMap + ) { + Map dataMap = rawResponse.payload(); + dataMap = (Map) dataMap.get("data"); + + dataMap = (Map)dataMap.get("COLLECTED"); + + + Map collectedMap = JsonUtils.asMap(dataMap); + if (collectedMap == null || collectedMap.isEmpty()){ + if(dataState.equals(DataState.COLLECTED)) { + log.warn("No collected data for interrogation {}", rawResponse.interrogationId()); + } + return; + } + convertToCollectedVar(dstSurveyUnitModel, dataState, variablesMap, collectedMap); + } + + private static void convertToCollectedVar( + SurveyUnitModel dstSurveyUnitModel, + DataState dataState, + VariablesMap variablesMap, + Map collectedMap + ) { + final String stateKey = dataState.toString(); + final var dest = dstSurveyUnitModel.getCollectedVariables(); + + for (Map.Entry collectedVariable : collectedMap.entrySet()) { + // Map for this variable (COLLECTED/EDITED -> value) + Map states = JsonUtils.asMap(collectedVariable.getValue()); + + // nothing if no state + if (states.containsKey(stateKey)) { + Object value = states.get(stateKey); + + // liste ? + if (value instanceof List) { + // on garde exactement ta signature existante + convertListVar(value, collectedVariable, variablesMap, dest); + } + + // scalaire non null ? + if (value != null && !(value instanceof List)) { + // idem: on garde convertOneVar(entry, String, ...) + convertOneVar(collectedVariable, String.valueOf(value), variablesMap, 1, dest); + } + } + } + } + + private static void convertListVar(Object valuesForState, Map.Entry collectedVariable, VariablesMap variablesMap, List dstSurveyUnitModel) { + List values = JsonUtils.asStringList(valuesForState); + if (!values.isEmpty()) { + int iteration = 1; + for (String value : values) { + convertOneVar(collectedVariable, value, variablesMap, iteration, dstSurveyUnitModel); + iteration++; + } + } + } + + private static void convertOneVar(Map.Entry externalVariableEntry, String valueObject, VariablesMap variablesMap, int iteration, List dstSurveyUnitModel) { + VariableModel externalVariableModel = VariableModel.builder() + .varId(externalVariableEntry.getKey()) + .value(valueObject) + .scope(getIdLoop(variablesMap, externalVariableEntry.getKey())) + .iteration(iteration) + .parentId(GroupUtils.getParentGroupName(externalVariableEntry.getKey(), variablesMap)) + .build(); + dstSurveyUnitModel.add(externalVariableModel); + } + + private static String getIdLoop(VariablesMap variablesMap, String variableName) { + if (variablesMap.getVariable(variableName) == null) { + log.warn("Variable {} not present in metadatas, assigning to {}", variableName, Constants.ROOT_GROUP_NAME); + return Constants.ROOT_GROUP_NAME; + } + return variablesMap.getVariable(variableName).getGroupName(); + } + + @SuppressWarnings("unchecked") + private static void convertRawDataExternalVariables( + RawResponse rawResponse, + SurveyUnitModel dstSurveyUnitModel, + VariablesMap variablesMap + ) { + Map dataMap = rawResponse.payload(); + dataMap = (Map) dataMap.get("data"); + + + dataMap = (Map)dataMap.get("EXTERNAL"); + Map externalMap = JsonUtils.asMap(dataMap); + if (externalMap != null && !externalMap.isEmpty()){ + convertToExternalVar(dstSurveyUnitModel, variablesMap, externalMap); + } + } + + private static void convertToExternalVar(SurveyUnitModel dstSurveyUnitModel, VariablesMap variablesMap, Map externalMap) { + for(Map.Entry externalVariableEntry : externalMap.entrySet()){ + Object valueObject = externalVariableEntry.getValue(); + if (valueObject instanceof List){ + //Array of values + convertListVar(valueObject, externalVariableEntry, variablesMap, dstSurveyUnitModel.getExternalVariables()); + continue; + } + //Value + if (valueObject != null) { + convertOneVar(externalVariableEntry, valueObject.toString(), variablesMap, 1, dstSurveyUnitModel.getExternalVariables()); + } + } + } } diff --git a/src/main/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitService.java b/src/main/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitService.java index 91b35e0c..de36d001 100644 --- a/src/main/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitService.java +++ b/src/main/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitService.java @@ -253,7 +253,7 @@ public SurveyUnitDto findLatestValuesByStateByIdAndByQuestionnaireId(String inte if (variablesMap == null) { variablesMap = metadataService.loadAndSaveIfNotExists( surveyUnitModel.getCampaignId(), - surveyUnitModel.getQuestionnaireId(), + surveyUnitModel.getCollectionInstrumentId(), surveyUnitModel.getMode(), fileUtils, new ArrayList<>()).getVariables(); @@ -453,7 +453,7 @@ public List parseEditedVariables( SurveyUnitModel surveyUnitModel = SurveyUnitModel.builder() .campaignId(surveyUnitInputDto.getCampaignId()) .mode(surveyUnitInputDto.getMode()) - .questionnaireId(surveyUnitInputDto.getQuestionnaireId().toUpperCase()) + .collectionInstrumentId(surveyUnitInputDto.getQuestionnaireId().toUpperCase()) .interrogationId(surveyUnitInputDto.getInterrogationId()) .state(state) .recordDate(LocalDateTime.now()) @@ -496,7 +496,7 @@ public String findQuestionnaireIdByInterrogationId(String interrogationId) throw } Set questionnaireIds = new HashSet<>(); for(SurveyUnitModel surveyUnitModel : surveyUnitModels){ - questionnaireIds.add(surveyUnitModel.getQuestionnaireId()); + questionnaireIds.add(surveyUnitModel.getCollectionInstrumentId()); } if(questionnaireIds.size() > 1){ throw new GenesisException(207,String.format("Multiple questionnaires for %s :%n%s", diff --git a/src/main/java/fr/insee/genesis/domain/utils/DataVerifier.java b/src/main/java/fr/insee/genesis/domain/utils/DataVerifier.java index 55a7abfd..45a30d7e 100644 --- a/src/main/java/fr/insee/genesis/domain/utils/DataVerifier.java +++ b/src/main/java/fr/insee/genesis/domain/utils/DataVerifier.java @@ -73,10 +73,10 @@ private static SurveyUnitModel createFormattedSurveyUnitModel( ) { SurveyUnitModel sampleSurveyUnitModel = surveyUnitModelsList.stream().filter(element -> element.getInterrogationId().equals(interrogationId)).toList().getFirst(); SurveyUnitModel newFormattedSurveyUnitModel = SurveyUnitModel.builder() - .questionnaireId(sampleSurveyUnitModel.getQuestionnaireId()) + .collectionInstrumentId(sampleSurveyUnitModel.getCollectionInstrumentId()) .campaignId(sampleSurveyUnitModel.getCampaignId()) .interrogationId(interrogationId) - .idUE(sampleSurveyUnitModel.getIdUE()) + .usualSurveyUnitId(sampleSurveyUnitModel.getUsualSurveyUnitId()) .state(DataState.FORMATTED) .mode(sampleSurveyUnitModel.getMode()) .recordDate(LocalDateTime.now().plusSeconds(1)) // Add 1 second to avoid same recordDate as COLLECTED diff --git a/src/main/java/fr/insee/genesis/infrastructure/adapter/RawResponseMongoAdapter.java b/src/main/java/fr/insee/genesis/infrastructure/adapter/RawResponseMongoAdapter.java index 1b1b35b7..ad7b7bee 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/adapter/RawResponseMongoAdapter.java +++ b/src/main/java/fr/insee/genesis/infrastructure/adapter/RawResponseMongoAdapter.java @@ -33,8 +33,8 @@ public RawResponseMongoAdapter(RawResponseRepository repository, MongoTemplate m } @Override - public List findRawResponses(String questionnaireId, Mode mode, List interrogationIdList) { - List rawDataDocs = repository.findByCollectionInstrumentIdAndModeAndInterrogationIdList(questionnaireId, mode, interrogationIdList); + public List findRawResponses(String collectionInstrumentId, Mode mode, List interrogationIdList) { + List rawDataDocs = repository.findByCollectionInstrumentIdAndModeAndInterrogationIdList(collectionInstrumentId, mode.getJsonName(), interrogationIdList); return RawResponseDocumentMapper.INSTANCE.listDocumentToListModel(rawDataDocs); } diff --git a/src/main/java/fr/insee/genesis/infrastructure/document/rawdata/RawResponseDocument.java b/src/main/java/fr/insee/genesis/infrastructure/document/rawdata/RawResponseDocument.java index 82eb6fe5..e62e5a4c 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/document/rawdata/RawResponseDocument.java +++ b/src/main/java/fr/insee/genesis/infrastructure/document/rawdata/RawResponseDocument.java @@ -1,6 +1,5 @@ package fr.insee.genesis.infrastructure.document.rawdata; -import fr.insee.genesis.domain.model.surveyunit.Mode; import lombok.Builder; import org.bson.types.ObjectId; import org.springframework.data.annotation.Id; @@ -18,7 +17,7 @@ public record RawResponseDocument ( ObjectId id, String interrogationId, String collectionInstrumentId, - Mode mode, + String mode, Map payload, LocalDateTime recordDate, @Indexed(direction = IndexDirection.DESCENDING) diff --git a/src/main/java/fr/insee/genesis/infrastructure/document/surveyunit/SurveyUnitDocument.java b/src/main/java/fr/insee/genesis/infrastructure/document/surveyunit/SurveyUnitDocument.java index 1977c949..0eaae474 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/document/surveyunit/SurveyUnitDocument.java +++ b/src/main/java/fr/insee/genesis/infrastructure/document/surveyunit/SurveyUnitDocument.java @@ -1,5 +1,6 @@ package fr.insee.genesis.infrastructure.document.surveyunit; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import fr.insee.genesis.Constants; import lombok.Data; import org.springframework.data.mongodb.core.index.CompoundIndex; @@ -15,12 +16,34 @@ @CompoundIndex(name = "questionnaireId_1_campaignId_1", def = "{'questionnaireId': 1, 'campaignId': 1}") @CompoundIndex(name = "questionnaireId_1_interrogationId_1", def = "{'questionnaireId': 1, 'interrogationId': 1}") @CompoundIndex(name = "interrogationId_1_questionnaireId_1", def = "{'interrogationId': 1, 'questionnaireId': 1}") +@JsonIgnoreProperties(ignoreUnknown = true) public class SurveyUnitDocument { + + /** + * @deprecated This piece of information will not be available anymore in the raw responses + */ + @Deprecated(forRemoval = true) private String campaignId; @Indexed private String interrogationId; + + /** + * @deprecated It will be replaced by usualSurveyUnitId + */ + @Deprecated(forRemoval = true) private String idUE; + + private String usualSurveyUnitId; + + /** + * @deprecated It will be replaced by collectionInstrumentId + */ + @Deprecated(forRemoval = true) private String questionnaireId; + + private String collectionInstrumentId; + + private String majorModelVersion; private String state; @Indexed private String mode; diff --git a/src/main/java/fr/insee/genesis/infrastructure/mappers/RawResponseDocumentMapper.java b/src/main/java/fr/insee/genesis/infrastructure/mappers/RawResponseDocumentMapper.java index b72f6021..96e5e073 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/mappers/RawResponseDocumentMapper.java +++ b/src/main/java/fr/insee/genesis/infrastructure/mappers/RawResponseDocumentMapper.java @@ -1,5 +1,6 @@ package fr.insee.genesis.infrastructure.mappers; +import fr.insee.genesis.domain.model.surveyunit.Mode; import fr.insee.genesis.domain.model.surveyunit.rawdata.RawResponse; import fr.insee.genesis.infrastructure.document.rawdata.RawResponseDocument; import org.mapstruct.Mapper; @@ -16,4 +17,14 @@ public interface RawResponseDocumentMapper { RawResponseDocument modelToDocument(RawResponse model); List listDocumentToListModel(List documentList); List listModelToListDocument(List modelList); + + // --- Custom mapping: String -> Mode + default Mode stringToMode(String value) { + return Mode.fromString(value); // réutilise ton JsonCreator (parfait) + } + + // --- Custom mapping: Mode -> String + default String modeToString(Mode mode) { + return mode != null ? mode.getJsonName() : null; + } } diff --git a/src/main/java/fr/insee/genesis/infrastructure/mappers/SurveyUnitDocumentMapper.java b/src/main/java/fr/insee/genesis/infrastructure/mappers/SurveyUnitDocumentMapper.java index fb2df28a..16fe8618 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/mappers/SurveyUnitDocumentMapper.java +++ b/src/main/java/fr/insee/genesis/infrastructure/mappers/SurveyUnitDocumentMapper.java @@ -2,7 +2,9 @@ import fr.insee.genesis.domain.model.surveyunit.SurveyUnitModel; import fr.insee.genesis.infrastructure.document.surveyunit.SurveyUnitDocument; +import org.mapstruct.AfterMapping; import org.mapstruct.Mapper; +import org.mapstruct.MappingTarget; import org.mapstruct.factory.Mappers; import java.util.List; @@ -19,4 +21,18 @@ public interface SurveyUnitDocumentMapper { List listModelToListDocument(List surveyUnitModels); + @AfterMapping + default void fillModelAfterRead(SurveyUnitDocument doc, + @MappingTarget SurveyUnitModel model) { + + if (model.getUsualSurveyUnitId() == null) { + model.setUsualSurveyUnitId(doc.getIdUE()); + } + + if (model.getCollectionInstrumentId() == null) { + model.setCollectionInstrumentId(doc.getQuestionnaireId()); + } + + } + } diff --git a/src/main/java/fr/insee/genesis/infrastructure/repository/RawResponseRepository.java b/src/main/java/fr/insee/genesis/infrastructure/repository/RawResponseRepository.java index d2f9d1e0..40e23381 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/repository/RawResponseRepository.java +++ b/src/main/java/fr/insee/genesis/infrastructure/repository/RawResponseRepository.java @@ -1,6 +1,5 @@ package fr.insee.genesis.infrastructure.repository; -import fr.insee.genesis.domain.model.surveyunit.Mode; import fr.insee.genesis.infrastructure.document.rawdata.RawResponseDocument; import org.springframework.data.mongodb.repository.MongoRepository; import org.springframework.data.mongodb.repository.Query; @@ -12,5 +11,5 @@ public interface RawResponseRepository extends MongoRepository { @Query(value = "{ 'collectionInstrumentId' : ?0, 'mode' : ?1, 'interrogationId': {$in: ?2} }") - List findByCollectionInstrumentIdAndModeAndInterrogationIdList(String questionnaireId, Mode mode, List interrogationIdList); + List findByCollectionInstrumentIdAndModeAndInterrogationIdList(String questionnaireId, String mode, List interrogationIdList); } diff --git a/src/test/java/cucumber/functional_tests/LunaticModelDefinitions.java b/src/test/java/cucumber/functional_tests/LunaticModelDefinitions.java index efb5916c..b785d234 100644 --- a/src/test/java/cucumber/functional_tests/LunaticModelDefinitions.java +++ b/src/test/java/cucumber/functional_tests/LunaticModelDefinitions.java @@ -108,7 +108,7 @@ public void load_response(String campaignId, String questionnaireId, String inte surveyUnitPersistencePortStub.getMongoStub().add( SurveyUnitModel.builder() .campaignId(campaignId) - .questionnaireId(questionnaireId) + .collectionInstrumentId(questionnaireId) .interrogationId(interrogationId) .build() ); diff --git a/src/test/java/cucumber/functional_tests/RawDataDefinitions.java b/src/test/java/cucumber/functional_tests/RawDataDefinitions.java index 8fe9a8cd..e4dbbfa9 100644 --- a/src/test/java/cucumber/functional_tests/RawDataDefinitions.java +++ b/src/test/java/cucumber/functional_tests/RawDataDefinitions.java @@ -323,9 +323,9 @@ public void check_external_variable_content_in_mongo( } - @Then("In surveyUnit {string} of the campaign {string} we must have {string} as contextualId, " + + @Then("In surveyUnit {string} of the campaign {string} we must have " + "isCapturedIndirectly to {string} and validationDate null") - public void check_optional_values(String interrogationId, String campaignId, String expectedContextualId, + public void check_optional_values(String interrogationId, String campaignId, String expectedCapturedIndirectly) { //Get SurveyUnitModel List concernedSurveyUnitModels = surveyUnitPersistencePortStub.getMongoStub().stream().filter(surveyUnitModel -> @@ -335,7 +335,6 @@ public void check_optional_values(String interrogationId, String campaignId, Str ).toList(); Assertions.assertThat(concernedSurveyUnitModels).hasSize(1); - Assertions.assertThat(concernedSurveyUnitModels.getFirst().getContextualId()).isEqualTo(expectedContextualId); Assertions.assertThat(concernedSurveyUnitModels.getFirst().getIsCapturedIndirectly()).isEqualTo(Boolean.parseBoolean(expectedCapturedIndirectly)); Assertions.assertThat(concernedSurveyUnitModels.getFirst().getValidationDate()).isNull(); } diff --git a/src/test/java/fr/insee/genesis/controller/adapter/LunaticXmlAdapterTest.java b/src/test/java/fr/insee/genesis/controller/adapter/LunaticXmlAdapterTest.java index 727756cd..c3d8f3c9 100644 --- a/src/test/java/fr/insee/genesis/controller/adapter/LunaticXmlAdapterTest.java +++ b/src/test/java/fr/insee/genesis/controller/adapter/LunaticXmlAdapterTest.java @@ -234,7 +234,7 @@ void test02() { // When List surveyUnitModels = LunaticXmlAdapter.convert(lunaticXmlSurveyUnit1, metadataModel.getVariables(), CAMPAIGN_ID, Mode.WEB); // Then - Assertions.assertThat(surveyUnitModels.getFirst().getQuestionnaireId()).isEqualTo("questionnaireId1".toUpperCase()); + Assertions.assertThat(surveyUnitModels.getFirst().getCollectionInstrumentId()).isEqualTo("questionnaireId1".toUpperCase()); } @Test diff --git a/src/test/java/fr/insee/genesis/controller/rest/ControllerAccessTest.java b/src/test/java/fr/insee/genesis/controller/rest/ControllerAccessTest.java index 1d29e9fa..448d4aba 100644 --- a/src/test/java/fr/insee/genesis/controller/rest/ControllerAccessTest.java +++ b/src/test/java/fr/insee/genesis/controller/rest/ControllerAccessTest.java @@ -2,6 +2,7 @@ import fr.insee.genesis.domain.ports.api.DataProcessingContextApiPort; import fr.insee.genesis.domain.ports.api.LunaticJsonRawDataApiPort; +import fr.insee.genesis.domain.ports.api.RawResponseApiPort; import fr.insee.genesis.domain.ports.api.SurveyUnitApiPort; import fr.insee.genesis.infrastructure.repository.ContextualExternalVariableMongoDBRepository; import fr.insee.genesis.infrastructure.repository.ContextualPreviousVariableMongoDBRepository; @@ -10,6 +11,8 @@ import fr.insee.genesis.infrastructure.repository.LunaticJsonMongoDBRepository; import fr.insee.genesis.infrastructure.repository.LunaticModelMongoDBRepository; import fr.insee.genesis.infrastructure.repository.QuestionnaireMetadataMongoDBRepository; +import fr.insee.genesis.infrastructure.repository.RawResponseInputRepository; +import fr.insee.genesis.infrastructure.repository.RawResponseRepository; import fr.insee.genesis.infrastructure.repository.RundeckExecutionDBRepository; import fr.insee.genesis.infrastructure.repository.SurveyUnitMongoDBRepository; import fr.insee.genesis.infrastructure.repository.VariableTypeMongoDBRepository; @@ -77,6 +80,10 @@ class ControllerAccessTest { @MockitoBean private LunaticJsonRawDataApiPort lunaticJsonRawDataApiPort; @MockitoBean + private RawResponseApiPort rawResponseApiPort; + @MockitoBean + private RawResponseInputRepository rawRepository; + @MockitoBean private SurveyUnitMongoDBRepository surveyUnitMongoDBRepository; @MockitoBean private LastJsonExtractionMongoDBRepository lastJsonExtractionMongoDBRepository; @@ -94,9 +101,10 @@ class ControllerAccessTest { private ContextualPreviousVariableMongoDBRepository contextualPreviousVariableMongoDBRepository; @MockitoBean private ContextualExternalVariableMongoDBRepository contextualExternalVariableMongoDBRepository; - @MockitoBean private QuestionnaireMetadataMongoDBRepository questionnaireMetadataMongoDBRepository; + @MockitoBean + private RawResponseRepository rawResponseRepository; /** * Provides a stream of URIs that are allowed for reader. diff --git a/src/test/java/fr/insee/genesis/controller/rest/DataProcessingContextControllerTest.java b/src/test/java/fr/insee/genesis/controller/rest/DataProcessingContextControllerTest.java index 7a0c9e19..f4594a98 100644 --- a/src/test/java/fr/insee/genesis/controller/rest/DataProcessingContextControllerTest.java +++ b/src/test/java/fr/insee/genesis/controller/rest/DataProcessingContextControllerTest.java @@ -296,6 +296,7 @@ void deleteExpiredScheduleTest_execution() throws IOException, GenesisException DataProcessingContextModel dataProcessingContextModel = new DataProcessingContextModel( null, "TESTSURVEYADDED", + "TESTSURVEYADDED", null, new ArrayList<>(), false @@ -343,6 +344,7 @@ void deleteExpiredScheduleTest_wholeSurvey() throws IOException, GenesisExceptio DataProcessingContextModel dataProcessingContextModel = new DataProcessingContextModel( null, "TESTSURVEYADDED", + "TESTSURVEYADDED", null, new ArrayList<>(), false @@ -391,6 +393,7 @@ void deleteExpiredScheduleTest_appendLog() throws IOException, GenesisException DataProcessingContextModel dataProcessingContextModel = new DataProcessingContextModel( null, "TESTSURVEYADDED2", + "TESTSURVEYADDED2", null, new ArrayList<>(), false diff --git a/src/test/java/fr/insee/genesis/controller/rest/HealthCheckControllerTest.java b/src/test/java/fr/insee/genesis/controller/rest/HealthCheckControllerTest.java index f3a5dc57..31d1aa4d 100644 --- a/src/test/java/fr/insee/genesis/controller/rest/HealthCheckControllerTest.java +++ b/src/test/java/fr/insee/genesis/controller/rest/HealthCheckControllerTest.java @@ -72,7 +72,7 @@ static void init() { .campaignId("TESTCAMPAIGNID") .mode(Mode.WEB) .interrogationId("TESTINTERROGATIONID") - .questionnaireId("TESTQUESTIONNAIREID") + .collectionInstrumentId("TESTQUESTIONNAIREID") .state(DataState.COLLECTED) .fileDate(LocalDateTime.of(2023, 1, 1, 0, 0 , 0)) .recordDate(LocalDateTime.of(2024, 1, 1, 0, 0, 0)) diff --git a/src/test/java/fr/insee/genesis/controller/rest/UtilsControllerTest.java b/src/test/java/fr/insee/genesis/controller/rest/UtilsControllerTest.java index aad43cb1..5f389189 100644 --- a/src/test/java/fr/insee/genesis/controller/rest/UtilsControllerTest.java +++ b/src/test/java/fr/insee/genesis/controller/rest/UtilsControllerTest.java @@ -132,7 +132,7 @@ void reset() throws IOException { .campaignId("TEST-TABLEAUX") .mode(Mode.WEB) .interrogationId(DEFAULT_INTERROGATION_ID) - .questionnaireId(DEFAULT_QUESTIONNAIRE_ID) + .collectionInstrumentId(DEFAULT_QUESTIONNAIRE_ID) .state(DataState.COLLECTED) .fileDate(LocalDateTime.of(2023, 1, 1, 0, 0, 0)) .recordDate(LocalDateTime.of(2024, 1, 1, 0, 0, 0)) @@ -451,7 +451,7 @@ private void addAdditionalSurveyUnitModelToMongoStub(String campaignId, String q .campaignId(campaignId) .mode(Mode.WEB) .interrogationId(DEFAULT_INTERROGATION_ID) - .questionnaireId(questionnaireId) + .collectionInstrumentId(questionnaireId) .state(DataState.COLLECTED) .fileDate(LocalDateTime.of(2023, 2, 2, 0, 0, 0)) .recordDate(LocalDateTime.of(2024, 2, 2, 0, 0, 0)) diff --git a/src/test/java/fr/insee/genesis/controller/rest/responses/RawResponseControllerTest.java b/src/test/java/fr/insee/genesis/controller/rest/responses/RawResponseControllerTest.java index 54780413..1b0545ca 100644 --- a/src/test/java/fr/insee/genesis/controller/rest/responses/RawResponseControllerTest.java +++ b/src/test/java/fr/insee/genesis/controller/rest/responses/RawResponseControllerTest.java @@ -215,10 +215,10 @@ void processJsonRawDataTest(){ Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub()).isNotNull().isNotEmpty().hasSize(1); Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getFirst()).isNotNull(); Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getFirst().getCampaignId()).isEqualTo(campaignId); - Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getFirst().getQuestionnaireId()).isNotNull().isEqualTo(questionnaireId); + Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getFirst().getCollectionInstrumentId()).isNotNull().isEqualTo(questionnaireId); Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getFirst().getMode()).isNotNull().isEqualTo(Mode.WEB); Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getFirst().getInterrogationId()).isEqualTo(interrogationId); - Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getFirst().getIdUE()).isEqualTo(idUE); + Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getFirst().getUsualSurveyUnitId()).isEqualTo(idUE); Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getFirst().getFileDate()).isNotNull(); Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getFirst().getRecordDate()).isNotNull(); Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getFirst().getCollectedVariables()).isNotNull().isNotEmpty().hasSize(1); diff --git a/src/test/java/fr/insee/genesis/controller/rest/responses/ResponseControllerTest.java b/src/test/java/fr/insee/genesis/controller/rest/responses/ResponseControllerTest.java index 0b3b8b71..22cfd4a0 100644 --- a/src/test/java/fr/insee/genesis/controller/rest/responses/ResponseControllerTest.java +++ b/src/test/java/fr/insee/genesis/controller/rest/responses/ResponseControllerTest.java @@ -157,8 +157,8 @@ void findResponsesByUEAndQuestionnaireTest() { Assertions.assertThat(response.getStatusCode().is2xxSuccessful()).isTrue(); Assertions.assertThat(response.getBody()).isNotNull().isNotEmpty(); Assertions.assertThat(response.getBody().getFirst().getInterrogationId()).isEqualTo(DEFAULT_INTERROGATION_ID); - Assertions.assertThat(response.getBody().getFirst().getIdUE()).isEqualTo(DEFAULT_ID_UE); - Assertions.assertThat(response.getBody().getFirst().getQuestionnaireId()).isEqualTo(DEFAULT_QUESTIONNAIRE_ID); + Assertions.assertThat(response.getBody().getFirst().getUsualSurveyUnitId()).isEqualTo(DEFAULT_ID_UE); + Assertions.assertThat(response.getBody().getFirst().getCollectionInstrumentId()).isEqualTo(DEFAULT_QUESTIONNAIRE_ID); } @Test @@ -170,8 +170,8 @@ void getLatestByUETest() { Assertions.assertThat(response.getStatusCode().is2xxSuccessful()).isTrue(); Assertions.assertThat(response.getBody()).isNotNull().isNotEmpty(); Assertions.assertThat(response.getBody().getFirst().getInterrogationId()).isEqualTo(DEFAULT_INTERROGATION_ID); - Assertions.assertThat(response.getBody().getFirst().getIdUE()).isEqualTo(DEFAULT_ID_UE); - Assertions.assertThat(response.getBody().getFirst().getQuestionnaireId()).isEqualTo(DEFAULT_QUESTIONNAIRE_ID); + Assertions.assertThat(response.getBody().getFirst().getUsualSurveyUnitId()).isEqualTo(DEFAULT_ID_UE); + Assertions.assertThat(response.getBody().getFirst().getCollectionInstrumentId()).isEqualTo(DEFAULT_QUESTIONNAIRE_ID); Assertions.assertThat(response.getBody().getFirst().getFileDate()).hasMonth(Month.FEBRUARY); } @@ -393,7 +393,7 @@ void saveEditedTest() { .campaignId(campaignId) .state(DataState.COLLECTED) .mode(Mode.WEB) - .questionnaireId(questionnaireId) + .collectionInstrumentId(questionnaireId) .interrogationId(DEFAULT_INTERROGATION_ID) .collectedVariables(List.of()) .build(); @@ -406,7 +406,7 @@ void saveEditedTest() { SurveyUnitModel docSaved = surveyUnitPersistencePortStub.getMongoStub().get(1); Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub()).hasSize(2); Assertions.assertThat(docSaved.getCampaignId()).isEqualTo(campaignId); - Assertions.assertThat(docSaved.getQuestionnaireId()).isEqualTo(questionnaireId); + Assertions.assertThat(docSaved.getCollectionInstrumentId()).isEqualTo(questionnaireId); Assertions.assertThat(docSaved.getMode()).isEqualTo(Mode.WEB); Assertions.assertThat(docSaved.getState()).isEqualTo(DataState.EDITED); Assertions.assertThat(docSaved.getFileDate()).isNull(); @@ -469,7 +469,7 @@ void saveEditedTest_DocumentEdited() { .campaignId(campaignId) .state(DataState.COLLECTED) .mode(Mode.WEB) - .questionnaireId(questionnaireId) + .collectionInstrumentId(questionnaireId) .interrogationId(DEFAULT_INTERROGATION_ID) .collectedVariables(List.of()) .build(); @@ -482,7 +482,7 @@ void saveEditedTest_DocumentEdited() { //EDITED document assertions Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub()).hasSize(3); Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().get(1).getCampaignId()).isEqualTo(campaignId); - Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().get(1).getQuestionnaireId()).isEqualTo(questionnaireId); + Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().get(1).getCollectionInstrumentId()).isEqualTo(questionnaireId); Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().get(1).getState()).isEqualTo(DataState.EDITED); Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().get(1).getMode()).isEqualTo(Mode.WEB); Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().get(1).getFileDate()).isNull(); @@ -544,7 +544,7 @@ void saveEditedTest_DocumentFormatted() { .campaignId(campaignId) .state(DataState.COLLECTED) .mode(Mode.WEB) - .questionnaireId(questionnaireId) + .collectionInstrumentId(questionnaireId) .interrogationId(DEFAULT_INTERROGATION_ID) .collectedVariables(List.of()) .build(); @@ -558,7 +558,7 @@ void saveEditedTest_DocumentFormatted() { //FORMATTED document assertions Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getLast().getCampaignId()).isEqualTo(campaignId); - Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getLast().getQuestionnaireId()).isEqualTo(questionnaireId); + Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getLast().getCollectionInstrumentId()).isEqualTo(questionnaireId); Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getLast().getState()).isEqualTo(DataState.FORMATTED); Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getLast().getMode()).isEqualTo(Mode.WEB); Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getLast().getFileDate()).isNull(); @@ -605,7 +605,7 @@ void saveEditedTest_No_Metadata_Error() { .campaignId(campaignId) .state(DataState.COLLECTED) .mode(Mode.WEB) - .questionnaireId(campaignId) + .collectionInstrumentId(campaignId) .interrogationId(DEFAULT_INTERROGATION_ID) .collectedVariables(List.of()) .build(); @@ -650,7 +650,7 @@ void saveTest_With_Collected_State_Error(){ .campaignId(CAMPAIGN_ID_WITH_DDI) .state(DataState.COLLECTED) .mode(Mode.WEB) - .questionnaireId(DEFAULT_QUESTIONNAIRE_ID) + .collectionInstrumentId(DEFAULT_QUESTIONNAIRE_ID) .interrogationId(DEFAULT_INTERROGATION_ID) .collectedVariables(List.of()) .build(); @@ -695,7 +695,7 @@ void saveEditedTest_int() { .campaignId(campaignId) .state(DataState.COLLECTED) .mode(Mode.WEB) - .questionnaireId(questionnaireId) + .collectionInstrumentId(questionnaireId) .interrogationId(DEFAULT_INTERROGATION_ID) .collectedVariables(List.of()) .build(); @@ -708,7 +708,7 @@ void saveEditedTest_int() { SurveyUnitModel docSaved = surveyUnitPersistencePortStub.getMongoStub().get(1); Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub()).hasSize(2); Assertions.assertThat(docSaved.getCampaignId()).isEqualTo(campaignId); - Assertions.assertThat(docSaved.getQuestionnaireId()).isEqualTo(questionnaireId); + Assertions.assertThat(docSaved.getCollectionInstrumentId()).isEqualTo(questionnaireId); Assertions.assertThat(docSaved.getMode()).isEqualTo(Mode.WEB); Assertions.assertThat(docSaved.getState()).isEqualTo(DataState.EDITED); Assertions.assertThat(docSaved.getFileDate()).isNull(); @@ -760,7 +760,7 @@ void saveEditedTest_null() { .campaignId(campaignId) .state(DataState.COLLECTED) .mode(Mode.WEB) - .questionnaireId(questionnaireId) + .collectionInstrumentId(questionnaireId) .interrogationId(DEFAULT_INTERROGATION_ID) .collectedVariables(List.of()) .build(); @@ -773,7 +773,7 @@ void saveEditedTest_null() { SurveyUnitModel docSaved = surveyUnitPersistencePortStub.getMongoStub().get(1); Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub()).hasSize(2); Assertions.assertThat(docSaved.getCampaignId()).isEqualTo(campaignId); - Assertions.assertThat(docSaved.getQuestionnaireId()).isEqualTo(questionnaireId); + Assertions.assertThat(docSaved.getCollectionInstrumentId()).isEqualTo(questionnaireId); Assertions.assertThat(docSaved.getMode()).isEqualTo(Mode.WEB); Assertions.assertThat(docSaved.getState()).isEqualTo(DataState.EDITED); Assertions.assertThat(docSaved.getFileDate()).isNull(); diff --git a/src/test/java/fr/insee/genesis/controller/rest/responses/Utils.java b/src/test/java/fr/insee/genesis/controller/rest/responses/Utils.java index 3ce6d67f..bdb914f5 100644 --- a/src/test/java/fr/insee/genesis/controller/rest/responses/Utils.java +++ b/src/test/java/fr/insee/genesis/controller/rest/responses/Utils.java @@ -221,8 +221,8 @@ static void addAdditionalSurveyUnitModelToMongoStub(String campaignId, String qu .campaignId(campaignId) .mode(Mode.WEB) .interrogationId(DEFAULT_INTERROGATION_ID) - .idUE(idUE) - .questionnaireId(questionnaireId) + .usualSurveyUnitId(idUE) + .collectionInstrumentId(questionnaireId) .state(DataState.COLLECTED) .fileDate(fileDate) .recordDate(recordDate) @@ -263,7 +263,7 @@ static void addAdditionalSurveyUnitModelToMongoStub(DataState state, .campaignId("TEST-TABLEAUX") .mode(Mode.WEB) .interrogationId(DEFAULT_INTERROGATION_ID) - .questionnaireId(DEFAULT_QUESTIONNAIRE_ID) + .collectionInstrumentId(DEFAULT_QUESTIONNAIRE_ID) .state(state) .fileDate(fileDate) .recordDate(recordDate) @@ -303,7 +303,7 @@ static void addAdditionalSurveyUnitModelToMongoStub(DataState state, .campaignId("TEST-TABLEAUX") .mode(Mode.WEB) .interrogationId(DEFAULT_INTERROGATION_ID) - .questionnaireId(DEFAULT_QUESTIONNAIRE_ID) + .collectionInstrumentId(DEFAULT_QUESTIONNAIRE_ID) .state(state) .fileDate(fileDate) .recordDate(recordDate) diff --git a/src/test/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitServiceTest.java b/src/test/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitServiceTest.java index 34c7eb69..e1de9ce0 100644 --- a/src/test/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitServiceTest.java +++ b/src/test/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitServiceTest.java @@ -80,7 +80,7 @@ void reset(){ .campaignId("TEST-TABLEAUX") .mode(Mode.WEB) .interrogationId(DEFAULT_INTERROGATION_ID) - .questionnaireId(DEFAULT_QUESTIONNAIRE_ID) + .collectionInstrumentId(DEFAULT_QUESTIONNAIRE_ID) .state(DataState.COLLECTED) .fileDate(LocalDateTime.of(2023,1,1,0,0,0)) .recordDate(LocalDateTime.of(2024,1,1,0,0,0)) @@ -133,7 +133,7 @@ void saveAllTest(){ .campaignId("TEST-TABLEAUX") .mode(Mode.WEB) .interrogationId("TESTINTERROGATIONID2") - .questionnaireId(DEFAULT_QUESTIONNAIRE_ID) + .collectionInstrumentId(DEFAULT_QUESTIONNAIRE_ID) .state(DataState.COLLECTED) .fileDate(LocalDateTime.of(2023,1,1,0,0,0)) .recordDate(LocalDateTime.of(2024,1,1,0,0,0)) @@ -148,7 +148,7 @@ void saveAllTest(){ surveyUnitModel.getCampaignId().equals("TEST-TABLEAUX") && surveyUnitModel.getMode().equals(Mode.WEB) && surveyUnitModel.getInterrogationId().equals("TESTINTERROGATIONID2") - && surveyUnitModel.getQuestionnaireId().equals(DEFAULT_QUESTIONNAIRE_ID) + && surveyUnitModel.getCollectionInstrumentId().equals(DEFAULT_QUESTIONNAIRE_ID) && surveyUnitModel.getState().equals(DataState.COLLECTED) && surveyUnitModel.getFileDate().equals(LocalDateTime.of(2023,1,1,0,0,0)) && surveyUnitModel.getRecordDate().equals(LocalDateTime.of(2024,1,1,0,0,0)) @@ -170,7 +170,7 @@ void findByIdsUEAndQuestionnaireTest(){ Assertions.assertThat(surveyUnitServiceStatic.findByIdsInterrogationAndQuestionnaire(DEFAULT_INTERROGATION_ID, DEFAULT_QUESTIONNAIRE_ID)).filteredOn( surveyUnitModel -> surveyUnitModel.getInterrogationId().equals(DEFAULT_INTERROGATION_ID) - && surveyUnitModel.getQuestionnaireId().equals(DEFAULT_QUESTIONNAIRE_ID) + && surveyUnitModel.getCollectionInstrumentId().equals(DEFAULT_QUESTIONNAIRE_ID) ).isNotEmpty(); } @@ -185,7 +185,7 @@ void findByInterrogationIdTest(){ @Test void findByQuestionnaireIdTest(){ Assertions.assertThat(surveyUnitServiceStatic.findByQuestionnaireId(DEFAULT_QUESTIONNAIRE_ID)).filteredOn( - surveyUnitModel -> surveyUnitModel.getQuestionnaireId().equals(DEFAULT_QUESTIONNAIRE_ID) + surveyUnitModel -> surveyUnitModel.getCollectionInstrumentId().equals(DEFAULT_QUESTIONNAIRE_ID) ).isNotEmpty(); } @@ -195,7 +195,7 @@ void findLatestByIdAndByModeTest(){ Assertions.assertThat(surveyUnitServiceStatic.findLatestByIdAndByQuestionnaireId(DEFAULT_INTERROGATION_ID, DEFAULT_QUESTIONNAIRE_ID)).filteredOn( surveyUnitModel -> surveyUnitModel.getInterrogationId().equals(DEFAULT_INTERROGATION_ID) - && surveyUnitModel.getQuestionnaireId().equals(DEFAULT_QUESTIONNAIRE_ID) + && surveyUnitModel.getCollectionInstrumentId().equals(DEFAULT_QUESTIONNAIRE_ID) && surveyUnitModel.getFileDate().getMonth().equals(Month.FEBRUARY) ).isNotEmpty(); } @@ -212,7 +212,7 @@ void findResponsesByUEAndQuestionnaireTest_null_collectedVariables() { Assertions.assertThat(surveyUnitServiceStatic.findLatestByIdAndByQuestionnaireId(DEFAULT_INTERROGATION_ID, DEFAULT_QUESTIONNAIRE_ID)).filteredOn( surveyUnitModel -> surveyUnitModel.getInterrogationId().equals(DEFAULT_INTERROGATION_ID) - && surveyUnitModel.getQuestionnaireId().equals(DEFAULT_QUESTIONNAIRE_ID) + && surveyUnitModel.getCollectionInstrumentId().equals(DEFAULT_QUESTIONNAIRE_ID) && surveyUnitModel.getFileDate().getMonth().equals(Month.FEBRUARY) ).isNotEmpty(); } @@ -228,7 +228,7 @@ void findResponsesByUEAndQuestionnaireTest_null_externalVariables() { Assertions.assertThat(surveyUnitServiceStatic.findLatestByIdAndByQuestionnaireId(DEFAULT_INTERROGATION_ID, DEFAULT_QUESTIONNAIRE_ID)).filteredOn( surveyUnitModel -> surveyUnitModel.getInterrogationId().equals(DEFAULT_INTERROGATION_ID) - && surveyUnitModel.getQuestionnaireId().equals(DEFAULT_QUESTIONNAIRE_ID) + && surveyUnitModel.getCollectionInstrumentId().equals(DEFAULT_QUESTIONNAIRE_ID) && surveyUnitModel.getFileDate().getMonth().equals(Month.FEBRUARY) ).isNotEmpty(); } @@ -612,7 +612,7 @@ private void addAdditionnalSurveyUnitModelToMongoStub(){ .campaignId("TEST-TABLEAUX") .mode(Mode.WEB) .interrogationId(DEFAULT_INTERROGATION_ID) - .questionnaireId(DEFAULT_QUESTIONNAIRE_ID) + .collectionInstrumentId(DEFAULT_QUESTIONNAIRE_ID) .state(DataState.COLLECTED) .fileDate(LocalDateTime.of(2023,2,2,0,0,0)) .recordDate(LocalDateTime.of(2024,2,2,0,0,0)) @@ -659,7 +659,7 @@ private void addAdditionnalSurveyUnitModelToMongoStub(String questionnaireId) { .campaignId("TEST-TABLEAUX") .mode(Mode.WEB) .interrogationId(DEFAULT_INTERROGATION_ID) - .questionnaireId(questionnaireId) + .collectionInstrumentId(questionnaireId) .state(DataState.COLLECTED) .fileDate(LocalDateTime.of(2023, 2, 2, 0, 0, 0)) .recordDate(LocalDateTime.of(2024, 2, 2, 0, 0, 0)) @@ -697,7 +697,7 @@ private void addAdditionnalSurveyUnitModelToMongoStub(DataState state, .campaignId("TEST-TABLEAUX") .mode(Mode.WEB) .interrogationId(DEFAULT_INTERROGATION_ID) - .questionnaireId(DEFAULT_QUESTIONNAIRE_ID) + .collectionInstrumentId(DEFAULT_QUESTIONNAIRE_ID) .state(state) .fileDate(fileDate) .recordDate(recordDate) @@ -736,7 +736,7 @@ private void addAdditionnalSurveyUnitModelToMongoStub(DataState state, .campaignId("TEST-TABLEAUX") .mode(Mode.WEB) .interrogationId(DEFAULT_INTERROGATION_ID) - .questionnaireId(DEFAULT_QUESTIONNAIRE_ID) + .collectionInstrumentId(DEFAULT_QUESTIONNAIRE_ID) .state(state) .fileDate(fileDate) .recordDate(recordDate) diff --git a/src/test/java/fr/insee/genesis/domain/utils/DataVerifierTest.java b/src/test/java/fr/insee/genesis/domain/utils/DataVerifierTest.java index addb8aba..3024d349 100644 --- a/src/test/java/fr/insee/genesis/domain/utils/DataVerifierTest.java +++ b/src/test/java/fr/insee/genesis/domain/utils/DataVerifierTest.java @@ -60,7 +60,7 @@ void setUp() { SurveyUnitModel surveyUnit = SurveyUnitModel.builder() .interrogationId("UE1100000001") - .questionnaireId("Quest1") + .collectionInstrumentId("Quest1") .campaignId("Camp1") .state(DataState.COLLECTED) .collectedVariables(List.of(collectedVariable1, collectedVariable2)) @@ -102,7 +102,7 @@ void shouldAddFormattedSurveyUnit_WhenInvalidCollectedVariable() { .build(); SurveyUnitModel surveyUnit = SurveyUnitModel.builder() .interrogationId("UE1100000001") - .questionnaireId("Quest1") + .collectionInstrumentId("Quest1") .campaignId("Camp1") .state(DataState.COLLECTED) .collectedVariables(List.of(collectedVariable1, collectedVariable2)) @@ -140,7 +140,7 @@ void shouldAddFormattedSurveyUnit_WhenInvalidExternalVariable() { SurveyUnitModel surveyUnitWithInvalidExt = SurveyUnitModel.builder() .interrogationId("UE1100000002") - .questionnaireId("Quest1") + .collectionInstrumentId("Quest1") .campaignId("Camp1") .state(DataState.COLLECTED) .collectedVariables(List.of()) @@ -186,7 +186,7 @@ void shouldCorrectInvalidValuesInFormattedSurveyUnit() { .build(); SurveyUnitModel surveyUnit = SurveyUnitModel.builder() .interrogationId("UE1100000001") - .questionnaireId("Quest1") + .collectionInstrumentId("Quest1") .campaignId("Camp1") .state(DataState.COLLECTED) .collectedVariables(List.of(collectedVariable1, collectedVariable2, collectedVariable3)) @@ -239,7 +239,7 @@ void shouldCorrectInvalidIterationOnFormattedSurveyUnit() { .build(); SurveyUnitModel surveyUnit = SurveyUnitModel.builder() .interrogationId("UE1100000001") - .questionnaireId("Quest1") + .collectionInstrumentId("Quest1") .campaignId("Camp1") .state(DataState.COLLECTED) .collectedVariables(List.of(collectedVariable1, collectedVariable2, collectedVariable3, collectedVariable4)) @@ -287,7 +287,7 @@ void shouldCreateFormattedIfNull(DataState dataState) { .build(); SurveyUnitModel surveyUnit = SurveyUnitModel.builder() .interrogationId("UE1100000001") - .questionnaireId("Quest1") + .collectionInstrumentId("Quest1") .campaignId("Camp1") .state(dataState) .collectedVariables(List.of(collectedVariable1)) @@ -340,7 +340,7 @@ void shouldNotCreateFormattedIfNull(DataState dataState) { .build(); SurveyUnitModel surveyUnit = SurveyUnitModel.builder() .interrogationId("UE1100000001") - .questionnaireId("Quest1") + .collectionInstrumentId("Quest1") .campaignId("Camp1") .state(dataState) .collectedVariables(List.of(collectedVariable1)) diff --git a/src/test/java/fr/insee/genesis/infrastructure/mapper/SurveyUnitDocumentMapperImplTest.java b/src/test/java/fr/insee/genesis/infrastructure/mapper/SurveyUnitDocumentMapperImplTest.java index b5500369..238c9103 100644 --- a/src/test/java/fr/insee/genesis/infrastructure/mapper/SurveyUnitDocumentMapperImplTest.java +++ b/src/test/java/fr/insee/genesis/infrastructure/mapper/SurveyUnitDocumentMapperImplTest.java @@ -25,6 +25,8 @@ class SurveyUnitDocumentMapperImplTest { static SurveyUnitModel surveyUnitStatic; + // TODO : make different tests for document to model transformation, for old documents and new ones + @BeforeAll static void init(){ surveyUnitDocumentMapperImplStatic = new SurveyUnitDocumentMapperImpl(); @@ -71,7 +73,7 @@ static void init(){ .campaignId("TESTCAMPAIGNID") .mode(Mode.WEB) .interrogationId("TESTINTERROGATIONID") - .questionnaireId("TESTQUESTIONNAIREID") + .collectionInstrumentId("TESTQUESTIONNAIREID") .state(DataState.COLLECTED) .fileDate(LocalDateTime.of(2023,1,1,0,0,0)) .recordDate(LocalDateTime.of(2024,1,1,0,0,0)) @@ -99,7 +101,7 @@ void shouldReturnModelFromDocument(){ Assertions.assertThat(surveyUnit.getCampaignId()).isEqualTo("TESTCAMPAIGNID"); Assertions.assertThat(surveyUnit.getMode()).isEqualTo(Mode.WEB); Assertions.assertThat(surveyUnit.getInterrogationId()).isEqualTo("TESTINTERROGATIONID"); - Assertions.assertThat(surveyUnit.getQuestionnaireId()).isEqualTo("TESTQUESTIONNAIREID"); + Assertions.assertThat(surveyUnit.getCollectionInstrumentId()).isEqualTo("TESTQUESTIONNAIREID"); Assertions.assertThat(surveyUnit.getState()).isEqualTo(DataState.COLLECTED); Assertions.assertThat(surveyUnit.getFileDate()).isEqualTo(LocalDateTime.of(2023,1,1,0,0,0)); @@ -123,7 +125,7 @@ void shouldReturnDocumentFromModel(){ Assertions.assertThat(surveyUnitDocument.getCampaignId()).isEqualTo("TESTCAMPAIGNID"); Assertions.assertThat(surveyUnitDocument.getMode()).isEqualTo("WEB"); Assertions.assertThat(surveyUnitDocument.getInterrogationId()).isEqualTo("TESTINTERROGATIONID"); - Assertions.assertThat(surveyUnitDocument.getQuestionnaireId()).isEqualTo("TESTQUESTIONNAIREID"); + Assertions.assertThat(surveyUnitDocument.getCollectionInstrumentId()).isEqualTo("TESTQUESTIONNAIREID"); Assertions.assertThat(surveyUnitDocument.getState()).isEqualTo("COLLECTED"); Assertions.assertThat(surveyUnitDocument.getFileDate()).isEqualTo(LocalDateTime.of(2023,1,1,0,0,0)); @@ -151,7 +153,7 @@ void shouldReturnModelListFromDocumentList(){ Assertions.assertThat(surveyUnitList.getFirst().getCampaignId()).isEqualTo("TESTCAMPAIGNID"); Assertions.assertThat(surveyUnitList.getFirst().getMode()).isEqualTo(Mode.WEB); Assertions.assertThat(surveyUnitList.getFirst().getInterrogationId()).isEqualTo("TESTINTERROGATIONID"); - Assertions.assertThat(surveyUnitList.getFirst().getQuestionnaireId()).isEqualTo("TESTQUESTIONNAIREID"); + Assertions.assertThat(surveyUnitList.getFirst().getCollectionInstrumentId()).isEqualTo("TESTQUESTIONNAIREID"); Assertions.assertThat(surveyUnitList.getFirst().getState()).isEqualTo(DataState.COLLECTED); Assertions.assertThat(surveyUnitList.getFirst().getFileDate()).isEqualTo(LocalDateTime.of(2023,1,1,0,0,0)); @@ -177,7 +179,7 @@ void shouldReturnDocumentListFromModelList(){ Assertions.assertThat(surveyUnitDocumentList.getFirst().getCampaignId()).isEqualTo("TESTCAMPAIGNID"); Assertions.assertThat(surveyUnitDocumentList.getFirst().getMode()).isEqualTo("WEB"); Assertions.assertThat(surveyUnitDocumentList.getFirst().getInterrogationId()).isEqualTo("TESTINTERROGATIONID"); - Assertions.assertThat(surveyUnitDocumentList.getFirst().getQuestionnaireId()).isEqualTo("TESTQUESTIONNAIREID"); + Assertions.assertThat(surveyUnitDocumentList.getFirst().getCollectionInstrumentId()).isEqualTo("TESTQUESTIONNAIREID"); Assertions.assertThat(surveyUnitDocumentList.getFirst().getState()).isEqualTo("COLLECTED"); Assertions.assertThat(surveyUnitDocumentList.getFirst().getFileDate()).isEqualTo(LocalDateTime.of(2023,1,1,0,0,0)); diff --git a/src/test/java/fr/insee/genesis/stubs/DataProcessingContextPersistancePortStub.java b/src/test/java/fr/insee/genesis/stubs/DataProcessingContextPersistancePortStub.java index 77a93995..6326e619 100644 --- a/src/test/java/fr/insee/genesis/stubs/DataProcessingContextPersistancePortStub.java +++ b/src/test/java/fr/insee/genesis/stubs/DataProcessingContextPersistancePortStub.java @@ -38,6 +38,11 @@ public List findByPartitionIds(List partitio ); } + @Override + public List findByCollectionInstrumentIds(List collectionInstrumentIds) { + return List.of(); + } + @Override public void save(DataProcessingContextDocument dataProcessingContextDocument) { mongoStub.removeIf(existingDoc -> existingDoc.getPartitionId().equals( diff --git a/src/test/java/fr/insee/genesis/stubs/SurveyUnitPersistencePortStub.java b/src/test/java/fr/insee/genesis/stubs/SurveyUnitPersistencePortStub.java index a0be834b..c7ca0ad0 100644 --- a/src/test/java/fr/insee/genesis/stubs/SurveyUnitPersistencePortStub.java +++ b/src/test/java/fr/insee/genesis/stubs/SurveyUnitPersistencePortStub.java @@ -24,7 +24,7 @@ public void saveAll(List suList) { public List findByIds(String interrogationId, String questionnaireId) { List surveyUnitModelList = new ArrayList<>(); for(SurveyUnitModel SurveyUnitModel : mongoStub){ - if(SurveyUnitModel.getInterrogationId().equals(interrogationId) && SurveyUnitModel.getQuestionnaireId().equals(questionnaireId)) + if(SurveyUnitModel.getInterrogationId().equals(interrogationId) && SurveyUnitModel.getCollectionInstrumentId().equals(questionnaireId)) surveyUnitModelList.add(SurveyUnitModel); } @@ -57,7 +57,7 @@ public List findByInterrogationIdsAndQuestionnaireId(List surveyUnitModelList = new ArrayList<>(); for(SurveyUnitModel surveyUnitModel : interrogationIds) { for (SurveyUnitModel document : mongoStub) { - if (surveyUnitModel.getInterrogationId().equals(document.getInterrogationId()) && document.getQuestionnaireId().equals(questionnaireId)) + if (surveyUnitModel.getInterrogationId().equals(document.getInterrogationId()) && document.getCollectionInstrumentId().equals(questionnaireId)) surveyUnitModelList.add(document); } } @@ -69,7 +69,7 @@ public List findByInterrogationIdsAndQuestionnaireId(List findByQuestionnaireId(String questionnaireId) { List surveyUnitModelList = new ArrayList<>(); for(SurveyUnitModel SurveyUnitModel : mongoStub){ - if(SurveyUnitModel.getQuestionnaireId().equals(questionnaireId)) + if(SurveyUnitModel.getCollectionInstrumentId().equals(questionnaireId)) surveyUnitModelList.add(SurveyUnitModel); } @@ -80,7 +80,7 @@ public Stream findByQuestionnaireId(String questionnaireId) { public List findInterrogationIdsByQuestionnaireId(String questionnaireId) { List surveyUnitModelList = new ArrayList<>(); for(SurveyUnitModel SurveyUnitModel : mongoStub){ - if(SurveyUnitModel.getQuestionnaireId().equals(questionnaireId)) + if(SurveyUnitModel.getCollectionInstrumentId().equals(questionnaireId)) surveyUnitModelList.add( new SurveyUnitModel(SurveyUnitModel.getInterrogationId(), SurveyUnitModel.getMode()) ); @@ -115,7 +115,7 @@ public List findModesByCampaignIdV2(String campaignId) { public List findInterrogationIdsByQuestionnaireIdAndDateAfter(String questionnaireId, LocalDateTime since) { List surveyUnitModelList = new ArrayList<>(); for(SurveyUnitModel surveyUnitModel : mongoStub){ - if(surveyUnitModel.getQuestionnaireId().equals(questionnaireId) && surveyUnitModel.getRecordDate().isAfter(since)) + if(surveyUnitModel.getCollectionInstrumentId().equals(questionnaireId) && surveyUnitModel.getRecordDate().isAfter(since)) surveyUnitModelList.add( new SurveyUnitModel(surveyUnitModel.getInterrogationId(), surveyUnitModel.getMode()) ); @@ -163,7 +163,7 @@ public Set findQuestionnaireIdsByCampaignId(String campaignId) { Set questionnaireIdSet = new HashSet<>(); for(SurveyUnitModel SurveyUnitModel : mongoStub){ if(SurveyUnitModel.getCampaignId().equals(campaignId)) - questionnaireIdSet.add(SurveyUnitModel.getQuestionnaireId()); + questionnaireIdSet.add(SurveyUnitModel.getCollectionInstrumentId()); } return questionnaireIdSet; @@ -196,7 +196,7 @@ public long countByCampaignId(String campaignId) { public Set findDistinctQuestionnaireIds() { Set questionnaireIds = new HashSet<>(); for(SurveyUnitModel SurveyUnitModel : mongoStub){ - questionnaireIds.add(SurveyUnitModel.getQuestionnaireId()); + questionnaireIds.add(SurveyUnitModel.getCollectionInstrumentId()); } return questionnaireIds; } @@ -205,7 +205,7 @@ public Set findDistinctQuestionnaireIds() { public Set findCampaignIdsByQuestionnaireId(String questionnaireId) { Set campaignIdSet = new HashSet<>(); for(SurveyUnitModel SurveyUnitModel : mongoStub){ - if(SurveyUnitModel.getQuestionnaireId().equals(questionnaireId)) + if(SurveyUnitModel.getCollectionInstrumentId().equals(questionnaireId)) campaignIdSet.add(SurveyUnitModel.getCampaignId()); } diff --git a/src/test/resources/fr/insee/genesis/features/do_we_process_raw_data.feature b/src/test/resources/fr/insee/genesis/features/do_we_process_raw_data.feature index 00e1424c..53ca94c7 100644 --- a/src/test/resources/fr/insee/genesis/features/do_we_process_raw_data.feature +++ b/src/test/resources/fr/insee/genesis/features/do_we_process_raw_data.feature @@ -16,7 +16,7 @@ Feature: Raw data processing And We process raw data for campaign "", questionnaire "" and interrogation "" Then For collected variable "" in survey unit "" we should have "" for iteration 1 And For external variable "" in survey unit "" we should have "" for iteration 1 - And In surveyUnit "" of the campaign "" we must have "TESTCONTEXT" as contextualId, isCapturedIndirectly to "true" and validationDate null + And In surveyUnit "" of the campaign "" we must have isCapturedIndirectly to "true" and validationDate null Examples: | JsonFile | CampaignId | QuestionnaireId | InterrogationId | CollectedVariableName | ExpectedCollectedValue | ExternalVariableName | ExpectedExternalValue | | raw_data/rawdatasample_filieremodel_optionals.json | RAWDATATESTCAMPAIGN | TESTQUEST | TESTUE00001 | PRENOM_PAR1 | Farid | RPPRENOM | Robert | From bc6c80f9c9207e906c5aad479d41c4b703bb43af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Henninger?= Date: Thu, 20 Nov 2025 15:11:27 +0100 Subject: [PATCH 09/42] feat: WIP still moving to modelefiliere --- .../rest/responses/ModeController.java | 6 +- .../rest/responses/ResponseController.java | 93 +++++++++++++------ .../domain/ports/api/SurveyUnitApiPort.java | 15 ++- .../ports/spi/SurveyUnitPersistencePort.java | 4 +- .../service/surveyunit/SurveyUnitService.java | 36 ++++--- .../adapter/SurveyUnitMongoAdapter.java | 22 ++++- .../SurveyUnitMongoDBRepository.java | 5 + .../functional_tests/MainDefinitions.java | 2 +- .../controller/rest/ControllerAccessTest.java | 2 +- .../responses/ResponseControllerTest.java | 6 +- .../surveyunit/SurveyUnitServiceTest.java | 67 ++++++------- .../stubs/SurveyUnitPersistencePortStub.java | 26 ++++-- 12 files changed, 167 insertions(+), 117 deletions(-) diff --git a/src/main/java/fr/insee/genesis/controller/rest/responses/ModeController.java b/src/main/java/fr/insee/genesis/controller/rest/responses/ModeController.java index d7fef093..efb6400c 100644 --- a/src/main/java/fr/insee/genesis/controller/rest/responses/ModeController.java +++ b/src/main/java/fr/insee/genesis/controller/rest/responses/ModeController.java @@ -26,10 +26,10 @@ public ModeController(SurveyUnitApiPort surveyUnitService) { } - @Operation(summary = "List sources/modes used for a given questionnaire") + @Operation(summary = "List sources/modes used for a given collection instrument (ex questionnaire)") @GetMapping(path = "/by-questionnaire") - public ResponseEntity> getModesByQuestionnaire(@RequestParam("questionnaireId") String questionnaireId) { - List modes = surveyUnitService.findModesByQuestionnaireId(questionnaireId); + public ResponseEntity> getModesByQuestionnaire(@RequestParam("collectionInstrumentId") String collectionInstrumentId) { + List modes = surveyUnitService.findModesByCollectionInstrumentId(collectionInstrumentId); return ResponseEntity.ok(modes); } diff --git a/src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java b/src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java index 1627c318..21a009d0 100644 --- a/src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java +++ b/src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java @@ -185,27 +185,42 @@ public ResponseEntity saveResponsesFromAllCampaignFolders(){ //DELETE - @Operation(summary = "Delete all responses associated with a questionnaire") - @DeleteMapping(path = "/delete/by-questionnaire") + @Operation(summary = "Delete all responses associated with a collection instrument (formerly questionnaire)") + @DeleteMapping(path = "/delete/by-collection-instrument") @PreAuthorize("hasRole('ADMIN')") - public ResponseEntity deleteAllResponsesByQuestionnaire(@RequestParam("questionnaireId") String questionnaireId) { - log.info("Try to delete all responses of questionnaire : {}", questionnaireId); - Long ndDocuments = surveyUnitService.deleteByQuestionnaireId(questionnaireId); + public ResponseEntity deleteAllResponsesByCollectionInstrument(@RequestParam("collectionInstrumentId") String collectionInstrumentId) { + log.info("Try to delete all responses of collection instrument : {}", collectionInstrumentId); + Long ndDocuments = surveyUnitService.deleteByCollectionInstrumentId(collectionInstrumentId); log.info("{} responses deleted", ndDocuments); return ResponseEntity.ok(String.format("%d responses deleted", ndDocuments)); } //GET - @Operation(summary = "Retrieve responses for an interrogation, using interrogationId and questionnaireId from Genesis Database") - @GetMapping(path = "/by-ue-and-questionnaire") + @Operation(summary = "Retrieve responses for an interrogation, using interrogationId and collectionInstrumentId (formerly questionnaireId) from Genesis Database") + @GetMapping(path = "/by-interrogation-and-collection-instrument") @PreAuthorize("hasRole('ADMIN')") - public ResponseEntity> findResponsesByInterrogationAndQuestionnaire(@RequestParam("interrogationId") String interrogationId, - @RequestParam("questionnaireId") String questionnaireId) { - List responses = surveyUnitService.findByIdsInterrogationAndQuestionnaire(interrogationId, questionnaireId); + public ResponseEntity> findResponsesByInterrogationAndCollectionInstrument( + @RequestParam("interrogationId") String interrogationId, + @RequestParam("collectionInstrumentId") String collectionInstrumentId) + { + List responses = surveyUnitService.findByIdsInterrogationAndCollectionInstrument(interrogationId, collectionInstrumentId); return ResponseEntity.ok(responses); } - @Operation(summary = "Retrieve responses for an interrogation, using interrogationId and questionnaireId from Genesis Database with the latest value for each available state of every variable") + /** + * @deprecated + * This endpoint is deprecated because the parameter `questionnaireId` has been renamed + * to `collectionInstrumentId` in the Information System (modeled in the modelefiliere library). + * + * A new endpoint using the updated parameter names will be provided to remain compliant with + * the current data model. This endpoint will be removed once all dependent APIs have adopted + * the new naming convention. + * + * Use the new endpoint with `collectionInstrumentId` for future implementations. + */ + @Deprecated(forRemoval = true) + @Operation(summary = "Retrieve responses for an interrogation, using interrogationId and questionnaireId from Genesis Database with the latest value for each available state of every variable", + description = "use /by-interrogation-and-collection-instrument/latest-states instead") @GetMapping(path = "/by-ue-and-questionnaire/latest-states", produces = MediaType.APPLICATION_JSON_VALUE) @PreAuthorize("hasAnyRole('USER_PLATINE','SCHEDULER')") @@ -220,27 +235,47 @@ public ResponseEntity findResponsesByInterrogationAndQuestionnaireLatest return ResponseEntity.status(403).body(new ApiError("Review is disabled for that partition")); } - SurveyUnitDto response = surveyUnitService.findLatestValuesByStateByIdAndByQuestionnaireId(interrogationId, questionnaireId); + SurveyUnitDto response = surveyUnitService.findLatestValuesByStateByIdAndByCollectionInstrumentId(interrogationId, questionnaireId); SurveyUnitQualityToolDto responseQualityToolDto = DataTransformer.transformSurveyUnitDto(response); return ResponseEntity.ok(responseQualityToolDto); } - @Operation(summary = "Retrieve responses for an interrogation, using interrogationId and questionnaireId from Genesis Database. It returns only the latest value of each variable regardless of the state.") - @GetMapping(path = "/by-ue-and-questionnaire/latest") + @Operation(summary = "Retrieve the latest available values for each variable state for a given interrogation and collection instrument (formerly questionnaire).") + @GetMapping(path = "/by-interrogation-and-collection-instrument/latest-states", + produces = MediaType.APPLICATION_JSON_VALUE) + @PreAuthorize("hasAnyRole('USER_PLATINE','SCHEDULER')") + public ResponseEntity findResponsesByInterrogationAndCollectionInstrumentLatestStates( + @RequestParam("interrogationId") String interrogationId, + @RequestParam("collectionInstrumentId") String collectionInstrumentId) throws GenesisException { + //Check context + DataProcessingContextModel dataProcessingContextModel = + contextService.getContext(interrogationId); + + if(dataProcessingContextModel == null || !dataProcessingContextModel.isWithReview()){ + return ResponseEntity.status(403).body(new ApiError("Review is disabled for that partition")); + } + + SurveyUnitDto response = surveyUnitService.findLatestValuesByStateByIdAndByCollectionInstrumentId(interrogationId, collectionInstrumentId); + SurveyUnitQualityToolDto responseQualityToolDto = DataTransformer.transformSurveyUnitDto(response); + return ResponseEntity.ok(responseQualityToolDto); + } + + @Operation(summary = "Retrieve responses for an interrogation, using interrogationId and collectionInstrumentId from Genesis Database. It returns only the latest value of each variable regardless of the state.") + @GetMapping(path = "/by-interrogation-and-collection-instrument/latest") @PreAuthorize("hasRole('ADMIN')") - public ResponseEntity> getLatestByInterrogation(@RequestParam("interrogationId") String interrogationId, - @RequestParam("questionnaireId") String questionnaireId) { - List responses = surveyUnitService.findLatestByIdAndByQuestionnaireId(interrogationId, questionnaireId); + public ResponseEntity> getLatestByInterrogationAndCollectionInstrument(@RequestParam("interrogationId") String interrogationId, + @RequestParam("collectionInstrumentId") String collectionInstrumentId) { + List responses = surveyUnitService.findLatestByIdAndByCollectionInstrumentId(interrogationId, collectionInstrumentId); return ResponseEntity.ok(responses); } - @Operation(summary = "Retrieve responses for an interrogation, using interrogationId and questionnaireId from Genesis Database. For a given mode, it returns only the latest value of each variable regardless of the state. The result is one object by mode in the output") - @GetMapping(path = "/simplified/by-ue-questionnaire-and-mode/latest") + @Operation(summary = "Retrieve responses for an interrogation, using interrogationId and collectionInstrumentId from Genesis Database. For a given mode, it returns only the latest value of each variable regardless of the state. The result is one object by mode in the output") + @GetMapping(path = "/simplified/by-interrogation-collection-instrument-and-mode/latest") @PreAuthorize("hasRole('USER_KRAFTWERK')") public ResponseEntity getLatestByInterrogationOneObject(@RequestParam("interrogationId") String interrogationId, - @RequestParam("questionnaireId") String questionnaireId, + @RequestParam("collectionInstrumentId") String collectionInstrumentId, @RequestParam("mode") Mode mode) { - List responses = surveyUnitService.findLatestByIdAndByQuestionnaireId(interrogationId, questionnaireId); + List responses = surveyUnitService.findLatestByIdAndByCollectionInstrumentId(interrogationId, collectionInstrumentId); List outputVariables = new ArrayList<>(); List outputExternalVariables = new ArrayList<>(); responses.stream().filter(rep -> rep.getMode().equals(mode)).forEach(response -> { @@ -258,17 +293,19 @@ public ResponseEntity getLatestByInterrogationOneObject(@R } - @Operation(summary = "Retrieve all responses for a questionnaire and a list of UE", - description = "Return the latest state for each variable for the given ids and a given questionnaire.
" + + @Operation(summary = "Retrieve all responses for a collection instrument and a list of interrogations", + description = "Return the latest state for each variable for the given interrogationIds and a given collection instrument (formerly questionnaire).
" + "For a given id, the endpoint returns a document by collection mode (if there is more than one).") - @PostMapping(path = "/simplified/by-list-interrogation-and-questionnaire/latest") + @PostMapping(path = "/simplified/by-list-interrogation-and-collection-instrument/latest") @PreAuthorize("hasRole('USER_KRAFTWERK')") - public ResponseEntity> getLatestForInterrogationList(@RequestParam("questionnaireId") String questionnaireId, - @RequestBody List interrogationIds) { + public ResponseEntity> getLatestForInterrogationListAndCollectionInstrument( + @RequestParam("collectionInstrumentId") String collectionInstrumentId, + @RequestBody List interrogationIds) + { List results = new ArrayList<>(); - List modes = surveyUnitService.findModesByQuestionnaireId(questionnaireId); + List modes = surveyUnitService.findModesByCollectionInstrumentId(collectionInstrumentId); interrogationIds.forEach(interrogationId -> { - List responses = surveyUnitService.findLatestByIdAndByQuestionnaireId(interrogationId.getInterrogationId(), questionnaireId); + List responses = surveyUnitService.findLatestByIdAndByCollectionInstrumentId(interrogationId.getInterrogationId(), collectionInstrumentId); modes.forEach(mode -> { List outputVariables = new ArrayList<>(); List outputExternalVariables = new ArrayList<>(); diff --git a/src/main/java/fr/insee/genesis/domain/ports/api/SurveyUnitApiPort.java b/src/main/java/fr/insee/genesis/domain/ports/api/SurveyUnitApiPort.java index 9465a321..7a65a943 100644 --- a/src/main/java/fr/insee/genesis/domain/ports/api/SurveyUnitApiPort.java +++ b/src/main/java/fr/insee/genesis/domain/ports/api/SurveyUnitApiPort.java @@ -2,10 +2,10 @@ import fr.insee.bpm.metadata.model.VariablesMap; import fr.insee.genesis.controller.dto.CampaignWithQuestionnaire; -import fr.insee.genesis.domain.model.surveyunit.InterrogationId; import fr.insee.genesis.controller.dto.QuestionnaireWithCampaign; import fr.insee.genesis.controller.dto.SurveyUnitDto; import fr.insee.genesis.controller.dto.SurveyUnitInputDto; +import fr.insee.genesis.domain.model.surveyunit.InterrogationId; import fr.insee.genesis.domain.model.surveyunit.Mode; import fr.insee.genesis.domain.model.surveyunit.SurveyUnitModel; import fr.insee.genesis.exceptions.GenesisException; @@ -13,26 +13,23 @@ import java.time.LocalDateTime; import java.util.List; import java.util.Set; -import java.util.stream.Stream; public interface SurveyUnitApiPort { void saveSurveyUnits(List suList); - List findByIdsInterrogationAndQuestionnaire(String interrogationId, String questionnaireId); + List findByIdsInterrogationAndCollectionInstrument(String interrogationId, String questionnaireId); List findByInterrogationId(String interrogationId); - Stream findByQuestionnaireId(String questionnaireId); - - List findLatestByIdAndByQuestionnaireId(String interrogationId, String questionnaireId); + List findLatestByIdAndByCollectionInstrumentId(String interrogationId, String collectionInstrumentId); //========= OPTIMISATIONS PERFS (START) ========== List> findLatestByIdAndByQuestionnaireIdAndModeOrdered(String questionnaireId, String mode, List interrogationIds); //========= OPTIMISATIONS PERFS (END) ========== - SurveyUnitDto findLatestValuesByStateByIdAndByQuestionnaireId(String interrogationId, String questionnaireId) throws GenesisException; + SurveyUnitDto findLatestValuesByStateByIdAndByCollectionInstrumentId(String interrogationId, String collectionInstrumentId) throws GenesisException; List findInterrogationIdsAndModesByQuestionnaireId(String questionnaireId); @@ -49,7 +46,7 @@ List findDistinctPageableInterrogationIdsByQuestionnaireId(Stri List findModesByQuestionnaireIdV2(String questionnaireId); //========= OPTIMISATIONS PERFS (END) ========== - List findModesByQuestionnaireId(String questionnaireId); + List findModesByCollectionInstrumentId(String collectionInstrumentId); List findModesByCampaignId(String campaignId); @@ -57,7 +54,7 @@ List findDistinctPageableInterrogationIdsByQuestionnaireId(Stri List findModesByCampaignIdV2(String campaignId); //========= OPTIMISATIONS PERFS (END) ========== - Long deleteByQuestionnaireId(String questionnaireId); + Long deleteByCollectionInstrumentId(String collectionInstrumentId); long countResponses(); diff --git a/src/main/java/fr/insee/genesis/domain/ports/spi/SurveyUnitPersistencePort.java b/src/main/java/fr/insee/genesis/domain/ports/spi/SurveyUnitPersistencePort.java index ae1a31b2..e7237830 100644 --- a/src/main/java/fr/insee/genesis/domain/ports/spi/SurveyUnitPersistencePort.java +++ b/src/main/java/fr/insee/genesis/domain/ports/spi/SurveyUnitPersistencePort.java @@ -28,6 +28,8 @@ public interface SurveyUnitPersistencePort { List findInterrogationIdsByQuestionnaireId(String questionnaireId); + List findInterrogationIdsByCollectionInstrumentId(String collectionInstrumentId); + List findInterrogationIdsByQuestionnaireIdAndDateAfter(String questionnaireId, LocalDateTime since); //======== OPTIMISATIONS PERFS (START) ======== @@ -42,7 +44,7 @@ public interface SurveyUnitPersistencePort { List findInterrogationIdsByCampaignId(String campaignId); - Long deleteByQuestionnaireId(String questionnaireId); + Long deleteByCollectionInstrumentId(String collectionInstrumentId); long count(); diff --git a/src/main/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitService.java b/src/main/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitService.java index de36d001..67abea3c 100644 --- a/src/main/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitService.java +++ b/src/main/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitService.java @@ -34,7 +34,6 @@ import java.util.Map; import java.util.Set; import java.util.stream.Collectors; -import java.util.stream.Stream; @Service @Slf4j @@ -60,8 +59,8 @@ public void saveSurveyUnits(List surveyUnitModels) { } @Override - public List findByIdsInterrogationAndQuestionnaire(String interrogationId, String questionnaireId) { - return surveyUnitPersistencePort.findByIds(interrogationId, questionnaireId); + public List findByIdsInterrogationAndCollectionInstrument(String interrogationId, String collectionInstrumentId) { + return surveyUnitPersistencePort.findByIds(interrogationId, collectionInstrumentId); } @Override @@ -69,23 +68,18 @@ public List findByInterrogationId(String interrogationId) { return surveyUnitPersistencePort.findByInterrogationId(interrogationId); } - @Override - public Stream findByQuestionnaireId(String questionnaireId) { - return surveyUnitPersistencePort.findByQuestionnaireId(questionnaireId); - } - /** * In this method we want to get the latest update for each variable of a survey unit * But we need to separate the updates by mode * So we will calculate the latest state for a given collection mode - * @param interrogationId : Survey unit id - * @param questionnaireId : Questionnaire id + * @param interrogationId : Interrogation id + * @param collectionInstrumentId : Collection instrument id * @return the latest update for each variable of a survey unit */ @Override - public List findLatestByIdAndByQuestionnaireId(String interrogationId, String questionnaireId) { + public List findLatestByIdAndByCollectionInstrumentId(String interrogationId, String collectionInstrumentId) { List latestUpdatesbyVariables = new ArrayList<>(); - List surveyUnitModels = surveyUnitPersistencePort.findByIds(interrogationId, questionnaireId); + List surveyUnitModels = surveyUnitPersistencePort.findByIds(interrogationId, collectionInstrumentId); List modes = getDistinctsModes(surveyUnitModels); modes.forEach(mode ->{ List suByMode = surveyUnitModels.stream() @@ -229,8 +223,10 @@ private List extractLatestUpdates(List allResp @Override - public SurveyUnitDto findLatestValuesByStateByIdAndByQuestionnaireId(String interrogationId, - String questionnaireId) throws GenesisException { + public SurveyUnitDto findLatestValuesByStateByIdAndByCollectionInstrumentId( + String interrogationId, + String collectionInstrumentId) throws GenesisException + { SurveyUnitDto surveyUnitDto = SurveyUnitDto.builder() .interrogationId(interrogationId) .collectedVariables(new ArrayList<>()) @@ -240,7 +236,7 @@ public SurveyUnitDto findLatestValuesByStateByIdAndByQuestionnaireId(String inte //Extract variables Map collectedVariableMap = new HashMap<>(); Map externalVariableMap = new HashMap<>(); - List surveyUnitModels = surveyUnitPersistencePort.findByIds(interrogationId, questionnaireId); + List surveyUnitModels = surveyUnitPersistencePort.findByIds(interrogationId, collectionInstrumentId); List modes = getDistinctsModes(surveyUnitModels); for (Mode mode : modes) { List suByMode = surveyUnitModels.stream() @@ -334,8 +330,8 @@ public List findInterrogationIdsAndModesByQuestionnaireId(Strin } @Override - public List findModesByQuestionnaireId(String questionnaireId) { - List surveyUnitModels = surveyUnitPersistencePort.findInterrogationIdsByQuestionnaireId(questionnaireId); + public List findModesByCollectionInstrumentId(String collectionInstrumentId) { + List surveyUnitModels = surveyUnitPersistencePort.findInterrogationIdsByCollectionInstrumentId(collectionInstrumentId); List sources = new ArrayList<>(); surveyUnitModels.forEach(surveyUnitModel -> sources.add(surveyUnitModel.getMode())); return sources.stream().distinct().toList(); @@ -368,8 +364,8 @@ public List findModesByCampaignIdV2(String campaignId) { //========= OPTIMISATIONS PERFS (END) ========== @Override - public Long deleteByQuestionnaireId(String questionnaireId) { - return surveyUnitPersistencePort.deleteByQuestionnaireId(questionnaireId); + public Long deleteByCollectionInstrumentId(String collectionInstrumentId) { + return surveyUnitPersistencePort.deleteByCollectionInstrumentId(collectionInstrumentId); } @Override @@ -510,7 +506,7 @@ public String findQuestionnaireIdByInterrogationId(String interrogationId) throw @Override public Set findCampaignIdsFrom(SurveyUnitInputDto dto) { - List responses = findByIdsInterrogationAndQuestionnaire( + List responses = findByIdsInterrogationAndCollectionInstrument( dto.getInterrogationId(), dto.getQuestionnaireId() ); diff --git a/src/main/java/fr/insee/genesis/infrastructure/adapter/SurveyUnitMongoAdapter.java b/src/main/java/fr/insee/genesis/infrastructure/adapter/SurveyUnitMongoAdapter.java index 3bbbe509..dae26c3e 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/adapter/SurveyUnitMongoAdapter.java +++ b/src/main/java/fr/insee/genesis/infrastructure/adapter/SurveyUnitMongoAdapter.java @@ -46,8 +46,13 @@ public void saveAll(List surveyUnitModels) { } @Override - public List findByIds(String interrogationId, String questionnaireId) { - List surveyUnits = mongoRepository.findByInterrogationIdAndQuestionnaireId(interrogationId, questionnaireId); + public List findByIds(String interrogationId, String collectionInstrumentId) { + List surveyUnits = mongoRepository.findByInterrogationIdAndCollectionInstrumentId(interrogationId, collectionInstrumentId); + // To ensure compatibility with older documents (with questionnaireId instead of collectionInstrumentId) + List surveyUnitsLegacy = mongoRepository.findByInterrogationIdAndQuestionnaireId(interrogationId, collectionInstrumentId); + if(!surveyUnitsLegacy.isEmpty()){ + surveyUnits.addAll(surveyUnitsLegacy); + } return surveyUnits.isEmpty() ? Collections.emptyList() : SurveyUnitDocumentMapper.INSTANCE.listDocumentToListModel(surveyUnits); } @@ -87,8 +92,11 @@ public Stream findByQuestionnaireId(String questionnaireId) { } @Override - public Long deleteByQuestionnaireId(String questionnaireId) { - return mongoRepository.deleteByQuestionnaireId(questionnaireId); + public Long deleteByCollectionInstrumentId(String collectionInstrumentId) { + Long countDeleted = mongoRepository.deleteByCollectionInstrumentId(collectionInstrumentId); + // If the responses are in the old format (previous to modele filiere) + countDeleted += mongoRepository.deleteByQuestionnaireId(collectionInstrumentId); + return countDeleted; } @Override @@ -149,6 +157,12 @@ public List findInterrogationIdsByQuestionnaireId(String questi return surveyUnits.isEmpty() ? Collections.emptyList() : SurveyUnitDocumentMapper.INSTANCE.listDocumentToListModel(surveyUnits); } + @Override + public List findInterrogationIdsByCollectionInstrumentId(String collectionInstrumentId) { + List surveyUnits = mongoRepository.findInterrogationIdsByCollectionInstrumentId(collectionInstrumentId); + return surveyUnits.isEmpty() ? Collections.emptyList() : SurveyUnitDocumentMapper.INSTANCE.listDocumentToListModel(surveyUnits); + } + @Override public List findInterrogationIdsByQuestionnaireIdAndDateAfter(String questionnaireId, LocalDateTime since) { List surveyUnits = mongoRepository.findInterrogationIdsByQuestionnaireIdAndDateAfter(questionnaireId, since); diff --git a/src/main/java/fr/insee/genesis/infrastructure/repository/SurveyUnitMongoDBRepository.java b/src/main/java/fr/insee/genesis/infrastructure/repository/SurveyUnitMongoDBRepository.java index d1c5841d..63999e7c 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/repository/SurveyUnitMongoDBRepository.java +++ b/src/main/java/fr/insee/genesis/infrastructure/repository/SurveyUnitMongoDBRepository.java @@ -18,6 +18,7 @@ public interface SurveyUnitMongoDBRepository extends MongoRepository findByInterrogationId(String interrogationId); List findByInterrogationIdAndQuestionnaireId(String interrogationId, String questionnaireId); + List findByInterrogationIdAndCollectionInstrumentId(String interrogationId, String collectionInstrumentId); //========= OPTIMISATIONS PERFS (START) ========== /** @@ -30,6 +31,9 @@ public interface SurveyUnitMongoDBRepository extends MongoRepository findInterrogationIdsByQuestionnaireId(String questionnaireId); + @Query(value = "{ 'collectionInstrumentId' : ?0 }", fields = "{ 'interrogationId' : 1, 'mode' : 1 }") + List findInterrogationIdsByCollectionInstrumentId(String questionnaireId); + @Query(value = "{ 'questionnaireId' : ?0, 'recordDate': { $gte: ?1 } }", fields = "{ 'interrogationId' : 1, 'mode' : 1 }") List findInterrogationIdsByQuestionnaireIdAndDateAfter(String questionnaireId, LocalDateTime since); @@ -71,6 +75,7 @@ public interface SurveyUnitMongoDBRepository extends MongoRepository findInterrogationIdsByCampaignId(String campaignId); Long deleteByQuestionnaireId(String questionnaireId); + Long deleteByCollectionInstrumentId(String collectionInstrumentId); @Meta(cursorBatchSize = 20) Stream findByQuestionnaireId(String questionnaireId); diff --git a/src/test/java/cucumber/functional_tests/MainDefinitions.java b/src/test/java/cucumber/functional_tests/MainDefinitions.java index f0dbb2dc..b717ad1d 100644 --- a/src/test/java/cucumber/functional_tests/MainDefinitions.java +++ b/src/test/java/cucumber/functional_tests/MainDefinitions.java @@ -210,7 +210,7 @@ public void delete_directory() throws IOException { @When("We extract survey unit data with questionnaireId {string} and interrogationId {string}") public void extract_survey_data(String questionnaireId, String interrogationId) { - this.surveyUnitModelResponse = responseController.getLatestByInterrogation(interrogationId, questionnaireId.toUpperCase()); + this.surveyUnitModelResponse = responseController.getLatestByInterrogationAndCollectionInstrument(interrogationId, questionnaireId.toUpperCase()); } @When("We extract survey unit latest states with questionnaireId {string} and interrogationId {string}") diff --git a/src/test/java/fr/insee/genesis/controller/rest/ControllerAccessTest.java b/src/test/java/fr/insee/genesis/controller/rest/ControllerAccessTest.java index 448d4aba..ef8b422f 100644 --- a/src/test/java/fr/insee/genesis/controller/rest/ControllerAccessTest.java +++ b/src/test/java/fr/insee/genesis/controller/rest/ControllerAccessTest.java @@ -114,7 +114,7 @@ private static Stream endpointsReader() { Arguments.of("/questionnaires/with-campaigns"), Arguments.of("/questionnaires/by-campaign?campaignId=CAMPAIGNTEST"), Arguments.of("/questionnaires/"), - Arguments.of("/modes/by-questionnaire?questionnaireId=QUESTTEST"), + Arguments.of("/modes/by-questionnaire?collectionInstrumentId=QUESTTEST"), Arguments.of("/modes/by-campaign?campaignId=CAMPAIGNTEST"), Arguments.of("/interrogations/by-questionnaire?questionnaireId=QUESTTEST"), Arguments.of("/campaigns/with-questionnaires"), diff --git a/src/test/java/fr/insee/genesis/controller/rest/responses/ResponseControllerTest.java b/src/test/java/fr/insee/genesis/controller/rest/responses/ResponseControllerTest.java index 22cfd4a0..55842089 100644 --- a/src/test/java/fr/insee/genesis/controller/rest/responses/ResponseControllerTest.java +++ b/src/test/java/fr/insee/genesis/controller/rest/responses/ResponseControllerTest.java @@ -152,7 +152,7 @@ void saveResponsesFromAllCampaignFoldersTests(){ //Gets @Test void findResponsesByUEAndQuestionnaireTest() { - ResponseEntity> response = responseControllerStatic.findResponsesByInterrogationAndQuestionnaire(DEFAULT_INTERROGATION_ID, DEFAULT_QUESTIONNAIRE_ID); + ResponseEntity> response = responseControllerStatic.findResponsesByInterrogationAndCollectionInstrument(DEFAULT_INTERROGATION_ID, DEFAULT_QUESTIONNAIRE_ID); Assertions.assertThat(response.getStatusCode().is2xxSuccessful()).isTrue(); Assertions.assertThat(response.getBody()).isNotNull().isNotEmpty(); @@ -165,7 +165,7 @@ void findResponsesByUEAndQuestionnaireTest() { void getLatestByUETest() { Utils.addAdditionalSurveyUnitModelToMongoStub(surveyUnitPersistencePortStub); - ResponseEntity> response = responseControllerStatic.getLatestByInterrogation(DEFAULT_INTERROGATION_ID, DEFAULT_QUESTIONNAIRE_ID); + ResponseEntity> response = responseControllerStatic.getLatestByInterrogationAndCollectionInstrument(DEFAULT_INTERROGATION_ID, DEFAULT_QUESTIONNAIRE_ID); Assertions.assertThat(response.getStatusCode().is2xxSuccessful()).isTrue(); Assertions.assertThat(response.getBody()).isNotNull().isNotEmpty(); @@ -188,7 +188,7 @@ void getLatestByUEOneObjectTest() { @Test void getLatestForUEListTest() { - ResponseEntity> response = responseControllerStatic.getLatestForInterrogationList(DEFAULT_QUESTIONNAIRE_ID, interrogationIdList); + ResponseEntity> response = responseControllerStatic.getLatestForInterrogationListAndCollectionInstrument(DEFAULT_QUESTIONNAIRE_ID, interrogationIdList); Assertions.assertThat(response.getStatusCode().is2xxSuccessful()).isTrue(); Assertions.assertThat(response.getBody()).isNotNull().isNotEmpty(); diff --git a/src/test/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitServiceTest.java b/src/test/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitServiceTest.java index e1de9ce0..fe38f681 100644 --- a/src/test/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitServiceTest.java +++ b/src/test/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitServiceTest.java @@ -31,7 +31,7 @@ class SurveyUnitServiceTest { //Constants static final String DEFAULT_INTERROGATION_ID = "TESTINTERROGATIONID"; - static final String DEFAULT_QUESTIONNAIRE_ID = "TESTQUESTIONNAIREID"; + static final String DEFAULT_COLLECTION_INSTRUMENT_ID = "TESTCOLLECTIONINSTRUMENTID"; @BeforeAll static void init(){ @@ -80,7 +80,7 @@ void reset(){ .campaignId("TEST-TABLEAUX") .mode(Mode.WEB) .interrogationId(DEFAULT_INTERROGATION_ID) - .collectionInstrumentId(DEFAULT_QUESTIONNAIRE_ID) + .collectionInstrumentId(DEFAULT_COLLECTION_INSTRUMENT_ID) .state(DataState.COLLECTED) .fileDate(LocalDateTime.of(2023,1,1,0,0,0)) .recordDate(LocalDateTime.of(2024,1,1,0,0,0)) @@ -133,7 +133,7 @@ void saveAllTest(){ .campaignId("TEST-TABLEAUX") .mode(Mode.WEB) .interrogationId("TESTINTERROGATIONID2") - .collectionInstrumentId(DEFAULT_QUESTIONNAIRE_ID) + .collectionInstrumentId(DEFAULT_COLLECTION_INSTRUMENT_ID) .state(DataState.COLLECTED) .fileDate(LocalDateTime.of(2023,1,1,0,0,0)) .recordDate(LocalDateTime.of(2024,1,1,0,0,0)) @@ -148,7 +148,7 @@ void saveAllTest(){ surveyUnitModel.getCampaignId().equals("TEST-TABLEAUX") && surveyUnitModel.getMode().equals(Mode.WEB) && surveyUnitModel.getInterrogationId().equals("TESTINTERROGATIONID2") - && surveyUnitModel.getCollectionInstrumentId().equals(DEFAULT_QUESTIONNAIRE_ID) + && surveyUnitModel.getCollectionInstrumentId().equals(DEFAULT_COLLECTION_INSTRUMENT_ID) && surveyUnitModel.getState().equals(DataState.COLLECTED) && surveyUnitModel.getFileDate().equals(LocalDateTime.of(2023,1,1,0,0,0)) && surveyUnitModel.getRecordDate().equals(LocalDateTime.of(2024,1,1,0,0,0)) @@ -167,10 +167,10 @@ void saveAllTest(){ @Test void findByIdsUEAndQuestionnaireTest(){ - Assertions.assertThat(surveyUnitServiceStatic.findByIdsInterrogationAndQuestionnaire(DEFAULT_INTERROGATION_ID, DEFAULT_QUESTIONNAIRE_ID)).filteredOn( + Assertions.assertThat(surveyUnitServiceStatic.findByIdsInterrogationAndCollectionInstrument(DEFAULT_INTERROGATION_ID, DEFAULT_COLLECTION_INSTRUMENT_ID)).filteredOn( surveyUnitModel -> surveyUnitModel.getInterrogationId().equals(DEFAULT_INTERROGATION_ID) - && surveyUnitModel.getCollectionInstrumentId().equals(DEFAULT_QUESTIONNAIRE_ID) + && surveyUnitModel.getCollectionInstrumentId().equals(DEFAULT_COLLECTION_INSTRUMENT_ID) ).isNotEmpty(); } @@ -182,20 +182,13 @@ void findByInterrogationIdTest(){ ).isNotEmpty(); } - @Test - void findByQuestionnaireIdTest(){ - Assertions.assertThat(surveyUnitServiceStatic.findByQuestionnaireId(DEFAULT_QUESTIONNAIRE_ID)).filteredOn( - surveyUnitModel -> surveyUnitModel.getCollectionInstrumentId().equals(DEFAULT_QUESTIONNAIRE_ID) - ).isNotEmpty(); - } - @Test void findLatestByIdAndByModeTest(){ addAdditionnalSurveyUnitModelToMongoStub(); - Assertions.assertThat(surveyUnitServiceStatic.findLatestByIdAndByQuestionnaireId(DEFAULT_INTERROGATION_ID, DEFAULT_QUESTIONNAIRE_ID)).filteredOn( + Assertions.assertThat(surveyUnitServiceStatic.findLatestByIdAndByCollectionInstrumentId(DEFAULT_INTERROGATION_ID, DEFAULT_COLLECTION_INSTRUMENT_ID)).filteredOn( surveyUnitModel -> surveyUnitModel.getInterrogationId().equals(DEFAULT_INTERROGATION_ID) - && surveyUnitModel.getCollectionInstrumentId().equals(DEFAULT_QUESTIONNAIRE_ID) + && surveyUnitModel.getCollectionInstrumentId().equals(DEFAULT_COLLECTION_INSTRUMENT_ID) && surveyUnitModel.getFileDate().getMonth().equals(Month.FEBRUARY) ).isNotEmpty(); } @@ -210,9 +203,9 @@ void findResponsesByUEAndQuestionnaireTest_null_collectedVariables() { ); surveyUnitPersistencePortStub.getMongoStub().getLast().setCollectedVariables(null); - Assertions.assertThat(surveyUnitServiceStatic.findLatestByIdAndByQuestionnaireId(DEFAULT_INTERROGATION_ID, DEFAULT_QUESTIONNAIRE_ID)).filteredOn( + Assertions.assertThat(surveyUnitServiceStatic.findLatestByIdAndByCollectionInstrumentId(DEFAULT_INTERROGATION_ID, DEFAULT_COLLECTION_INSTRUMENT_ID)).filteredOn( surveyUnitModel -> surveyUnitModel.getInterrogationId().equals(DEFAULT_INTERROGATION_ID) - && surveyUnitModel.getCollectionInstrumentId().equals(DEFAULT_QUESTIONNAIRE_ID) + && surveyUnitModel.getCollectionInstrumentId().equals(DEFAULT_COLLECTION_INSTRUMENT_ID) && surveyUnitModel.getFileDate().getMonth().equals(Month.FEBRUARY) ).isNotEmpty(); } @@ -226,9 +219,9 @@ void findResponsesByUEAndQuestionnaireTest_null_externalVariables() { ); surveyUnitPersistencePortStub.getMongoStub().getLast().setExternalVariables(null); - Assertions.assertThat(surveyUnitServiceStatic.findLatestByIdAndByQuestionnaireId(DEFAULT_INTERROGATION_ID, DEFAULT_QUESTIONNAIRE_ID)).filteredOn( + Assertions.assertThat(surveyUnitServiceStatic.findLatestByIdAndByCollectionInstrumentId(DEFAULT_INTERROGATION_ID, DEFAULT_COLLECTION_INSTRUMENT_ID)).filteredOn( surveyUnitModel -> surveyUnitModel.getInterrogationId().equals(DEFAULT_INTERROGATION_ID) - && surveyUnitModel.getCollectionInstrumentId().equals(DEFAULT_QUESTIONNAIRE_ID) + && surveyUnitModel.getCollectionInstrumentId().equals(DEFAULT_COLLECTION_INSTRUMENT_ID) && surveyUnitModel.getFileDate().getMonth().equals(Month.FEBRUARY) ).isNotEmpty(); } @@ -237,7 +230,7 @@ void findResponsesByUEAndQuestionnaireTest_null_externalVariables() { void findDistinctInterrogationIdsByQuestionnaireIdTest(){ addAdditionnalSurveyUnitModelToMongoStub(); - Assertions.assertThat(surveyUnitServiceStatic.findDistinctInterrogationIdsByQuestionnaireId(DEFAULT_QUESTIONNAIRE_ID)).filteredOn( + Assertions.assertThat(surveyUnitServiceStatic.findDistinctInterrogationIdsByQuestionnaireId(DEFAULT_COLLECTION_INSTRUMENT_ID)).filteredOn( interrogationId -> interrogationId.getInterrogationId().equals(DEFAULT_INTERROGATION_ID) ).isNotEmpty().hasSize(1); } @@ -246,7 +239,7 @@ void findDistinctInterrogationIdsByQuestionnaireIdTest(){ void findDistinctInterrogationIdsByQuestionnaireIdAndDateAfterTest_no_doc_in_period(){ addAdditionnalSurveyUnitModelToMongoStub(); - Assertions.assertThat(surveyUnitServiceStatic.findDistinctInterrogationIdsByQuestionnaireIdAndDateAfter(DEFAULT_QUESTIONNAIRE_ID,LocalDateTime.of(2025,9,1,0,0,0))).filteredOn( + Assertions.assertThat(surveyUnitServiceStatic.findDistinctInterrogationIdsByQuestionnaireIdAndDateAfter(DEFAULT_COLLECTION_INSTRUMENT_ID,LocalDateTime.of(2025,9,1,0,0,0))).filteredOn( interrogationId -> interrogationId.getInterrogationId().equals(DEFAULT_INTERROGATION_ID) ).isEmpty(); } @@ -255,14 +248,14 @@ void findDistinctInterrogationIdsByQuestionnaireIdAndDateAfterTest_no_doc_in_per void findDistinctInterrogationIdsByQuestionnaireIdAndDateAfterTest_doc_in_period(){ addAdditionnalSurveyUnitModelToMongoStub(); - Assertions.assertThat(surveyUnitServiceStatic.findDistinctInterrogationIdsByQuestionnaireIdAndDateAfter(DEFAULT_QUESTIONNAIRE_ID,LocalDateTime.of(2022,1,1,0,0,0))).filteredOn( + Assertions.assertThat(surveyUnitServiceStatic.findDistinctInterrogationIdsByQuestionnaireIdAndDateAfter(DEFAULT_COLLECTION_INSTRUMENT_ID,LocalDateTime.of(2022,1,1,0,0,0))).filteredOn( interrogationId -> interrogationId.getInterrogationId().equals(DEFAULT_INTERROGATION_ID) ).isNotEmpty().hasSize(1); } @Test void findInterrogationIdsByQuestionnaireIdTest(){ - Assertions.assertThat(surveyUnitServiceStatic.findModesByQuestionnaireId(DEFAULT_QUESTIONNAIRE_ID)).filteredOn( + Assertions.assertThat(surveyUnitServiceStatic.findModesByCollectionInstrumentId(DEFAULT_COLLECTION_INSTRUMENT_ID)).filteredOn( mode -> mode.equals(Mode.WEB) ).isNotEmpty(); } @@ -270,7 +263,7 @@ void findInterrogationIdsByQuestionnaireIdTest(){ //========= OPTIMISATIONS PERFS (START) ========== @Test void findModesByQuestionnaireIdV2Test(){ - Assertions.assertThat(surveyUnitServiceStatic.findModesByQuestionnaireIdV2(DEFAULT_QUESTIONNAIRE_ID)).filteredOn( + Assertions.assertThat(surveyUnitServiceStatic.findModesByQuestionnaireIdV2(DEFAULT_COLLECTION_INSTRUMENT_ID)).filteredOn( mode -> mode.equals(Mode.WEB) ).isNotEmpty(); } @@ -320,9 +313,9 @@ void findLatestByIdAndByQuestionnaireIdPerretTest(){ //When - SurveyUnitDto surveyUnitDto = surveyUnitServiceStatic.findLatestValuesByStateByIdAndByQuestionnaireId( + SurveyUnitDto surveyUnitDto = surveyUnitServiceStatic.findLatestValuesByStateByIdAndByCollectionInstrumentId( DEFAULT_INTERROGATION_ID, - DEFAULT_QUESTIONNAIRE_ID + DEFAULT_COLLECTION_INSTRUMENT_ID ); @@ -396,9 +389,9 @@ void findLatestByIdAndByQuestionnaireIdPerretTest_null_collectedVariables(){ //When - SurveyUnitDto surveyUnitDto = surveyUnitServiceStatic.findLatestValuesByStateByIdAndByQuestionnaireId( + SurveyUnitDto surveyUnitDto = surveyUnitServiceStatic.findLatestValuesByStateByIdAndByCollectionInstrumentId( DEFAULT_INTERROGATION_ID, - DEFAULT_QUESTIONNAIRE_ID + DEFAULT_COLLECTION_INSTRUMENT_ID ); @@ -470,9 +463,9 @@ void findLatestByIdAndByQuestionnaireIdPerretTest_null_externalVariables(){ //When - SurveyUnitDto surveyUnitDto = surveyUnitServiceStatic.findLatestValuesByStateByIdAndByQuestionnaireId( + SurveyUnitDto surveyUnitDto = surveyUnitServiceStatic.findLatestValuesByStateByIdAndByCollectionInstrumentId( DEFAULT_INTERROGATION_ID, - DEFAULT_QUESTIONNAIRE_ID + DEFAULT_COLLECTION_INSTRUMENT_ID ); @@ -540,9 +533,9 @@ void findLatestValuesByStateByIdAndByQuestionnaireId_should_return_empty_values_ ); //When - SurveyUnitDto surveyUnitDto = surveyUnitServiceStatic.findLatestValuesByStateByIdAndByQuestionnaireId( + SurveyUnitDto surveyUnitDto = surveyUnitServiceStatic.findLatestValuesByStateByIdAndByCollectionInstrumentId( DEFAULT_INTERROGATION_ID, - DEFAULT_QUESTIONNAIRE_ID + DEFAULT_COLLECTION_INSTRUMENT_ID ); List variableDtos = surveyUnitDto.getCollectedVariables().stream().filter( variableDto -> variableDto.getVariableName().equals("TESTVARID") @@ -568,9 +561,9 @@ void findLatestValuesByStateByIdAndByQuestionnaireId_should_return_null_values() ); //When - SurveyUnitDto surveyUnitDto = surveyUnitServiceStatic.findLatestValuesByStateByIdAndByQuestionnaireId( + SurveyUnitDto surveyUnitDto = surveyUnitServiceStatic.findLatestValuesByStateByIdAndByCollectionInstrumentId( DEFAULT_INTERROGATION_ID, - DEFAULT_QUESTIONNAIRE_ID + DEFAULT_COLLECTION_INSTRUMENT_ID ); List variableDtos = surveyUnitDto.getCollectedVariables().stream().filter( variableDto -> variableDto.getVariableName().equals("TABLEAUTIC11") @@ -612,7 +605,7 @@ private void addAdditionnalSurveyUnitModelToMongoStub(){ .campaignId("TEST-TABLEAUX") .mode(Mode.WEB) .interrogationId(DEFAULT_INTERROGATION_ID) - .collectionInstrumentId(DEFAULT_QUESTIONNAIRE_ID) + .collectionInstrumentId(DEFAULT_COLLECTION_INSTRUMENT_ID) .state(DataState.COLLECTED) .fileDate(LocalDateTime.of(2023,2,2,0,0,0)) .recordDate(LocalDateTime.of(2024,2,2,0,0,0)) @@ -697,7 +690,7 @@ private void addAdditionnalSurveyUnitModelToMongoStub(DataState state, .campaignId("TEST-TABLEAUX") .mode(Mode.WEB) .interrogationId(DEFAULT_INTERROGATION_ID) - .collectionInstrumentId(DEFAULT_QUESTIONNAIRE_ID) + .collectionInstrumentId(DEFAULT_COLLECTION_INSTRUMENT_ID) .state(state) .fileDate(fileDate) .recordDate(recordDate) @@ -736,7 +729,7 @@ private void addAdditionnalSurveyUnitModelToMongoStub(DataState state, .campaignId("TEST-TABLEAUX") .mode(Mode.WEB) .interrogationId(DEFAULT_INTERROGATION_ID) - .collectionInstrumentId(DEFAULT_QUESTIONNAIRE_ID) + .collectionInstrumentId(DEFAULT_COLLECTION_INSTRUMENT_ID) .state(state) .fileDate(fileDate) .recordDate(recordDate) diff --git a/src/test/java/fr/insee/genesis/stubs/SurveyUnitPersistencePortStub.java b/src/test/java/fr/insee/genesis/stubs/SurveyUnitPersistencePortStub.java index c7ca0ad0..c9ee0472 100644 --- a/src/test/java/fr/insee/genesis/stubs/SurveyUnitPersistencePortStub.java +++ b/src/test/java/fr/insee/genesis/stubs/SurveyUnitPersistencePortStub.java @@ -21,10 +21,10 @@ public void saveAll(List suList) { } @Override - public List findByIds(String interrogationId, String questionnaireId) { + public List findByIds(String interrogationId, String collectionInstrumentId) { List surveyUnitModelList = new ArrayList<>(); for(SurveyUnitModel SurveyUnitModel : mongoStub){ - if(SurveyUnitModel.getInterrogationId().equals(interrogationId) && SurveyUnitModel.getCollectionInstrumentId().equals(questionnaireId)) + if(SurveyUnitModel.getInterrogationId().equals(interrogationId) && SurveyUnitModel.getCollectionInstrumentId().equals(collectionInstrumentId)) surveyUnitModelList.add(SurveyUnitModel); } @@ -89,6 +89,18 @@ public List findInterrogationIdsByQuestionnaireId(String questi return surveyUnitModelList; } + @Override + public List findInterrogationIdsByCollectionInstrumentId(String collectionInstrumentId) { + List surveyUnitModelList = new ArrayList<>(); + for(SurveyUnitModel SurveyUnitModel : mongoStub){ + if(SurveyUnitModel.getCollectionInstrumentId().equals(collectionInstrumentId)) + surveyUnitModelList.add( + new SurveyUnitModel(SurveyUnitModel.getInterrogationId(), SurveyUnitModel.getMode()) + ); + } + return surveyUnitModelList; + } + @Override public List findModesByQuestionnaireIdV2(String questionnaireId) { return findInterrogationIdsByQuestionnaireId(questionnaireId); @@ -140,17 +152,11 @@ public List findPageableInterrogationIdsByQuestionnaireId(Strin return List.of(); } - - - //======= OPTIMISATIONS PERFS (END) ========= - - - @Override - public Long deleteByQuestionnaireId(String questionnaireId) { - return null; + public Long deleteByCollectionInstrumentId(String collectionInstrumentId) { + return (long) mongoStub.stream().filter(su -> !su.getCollectionInstrumentId().equals(collectionInstrumentId)).toList().size(); } @Override From 95b7a830de2b16c9f796e5917286ed5126044225 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Henninger?= Date: Fri, 21 Nov 2025 16:36:21 +0100 Subject: [PATCH 10/42] feat: WIP modelefiliere contextual variables --- .../ContextualExternalVariableModel.java | 2 +- .../ContextualPreviousVariableModel.java | 2 +- .../ContextualExternalVariableApiPort.java | 4 +- .../ContextualPreviousVariableApiPort.java | 4 +- .../ports/api/ContextualVariableApiPort.java | 4 +- ...extualExternalVariablePersistancePort.java | 10 ++--- ...extualPreviousVariablePersistancePort.java | 10 ++--- .../ContextualVariableJsonService.java | 28 ++++++------ ...ContextualExternalVariableJsonService.java | 26 +++++------ ...ContextualPreviousVariableJsonService.java | 28 ++++++------ ...ontextualExternalVariableMongoAdapter.java | 37 ++++++++++----- ...ontextualPreviousVariableMongoAdapter.java | 45 ++++++++++++------- .../ContextualExternalVariableDocument.java | 10 +++++ .../ContextualPreviousVariableDocument.java | 9 ++++ ...textualExternalVariableDocumentMapper.java | 12 +++++ ...textualPreviousVariableDocumentMapper.java | 12 +++++ ...tualExternalVariableMongoDBRepository.java | 4 ++ ...tualPreviousVariableMongoDBRepository.java | 3 ++ .../ContextualVariableControllerTest.java | 20 ++++----- ...alExternalVariablePersistancePortStub.java | 5 ++- ...alPreviousVariablePersistancePortStub.java | 7 ++- 21 files changed, 180 insertions(+), 102 deletions(-) diff --git a/src/main/java/fr/insee/genesis/domain/model/contextualvariable/ContextualExternalVariableModel.java b/src/main/java/fr/insee/genesis/domain/model/contextualvariable/ContextualExternalVariableModel.java index 8be579b3..fe45ebd6 100644 --- a/src/main/java/fr/insee/genesis/domain/model/contextualvariable/ContextualExternalVariableModel.java +++ b/src/main/java/fr/insee/genesis/domain/model/contextualvariable/ContextualExternalVariableModel.java @@ -9,7 +9,7 @@ @Data public class ContextualExternalVariableModel { String id; - String questionnaireId; + String collectionInstrumentId; String interrogationId; Map variables; } diff --git a/src/main/java/fr/insee/genesis/domain/model/contextualvariable/ContextualPreviousVariableModel.java b/src/main/java/fr/insee/genesis/domain/model/contextualvariable/ContextualPreviousVariableModel.java index 4ebba43c..030ad843 100644 --- a/src/main/java/fr/insee/genesis/domain/model/contextualvariable/ContextualPreviousVariableModel.java +++ b/src/main/java/fr/insee/genesis/domain/model/contextualvariable/ContextualPreviousVariableModel.java @@ -9,7 +9,7 @@ @Data public class ContextualPreviousVariableModel { String id; - String questionnaireId; + String collectionInstrumentId; String interrogationId; Map variables; String sourceState; diff --git a/src/main/java/fr/insee/genesis/domain/ports/api/ContextualExternalVariableApiPort.java b/src/main/java/fr/insee/genesis/domain/ports/api/ContextualExternalVariableApiPort.java index 8c085834..ac889d39 100644 --- a/src/main/java/fr/insee/genesis/domain/ports/api/ContextualExternalVariableApiPort.java +++ b/src/main/java/fr/insee/genesis/domain/ports/api/ContextualExternalVariableApiPort.java @@ -4,7 +4,7 @@ import fr.insee.genesis.exceptions.GenesisException; public interface ContextualExternalVariableApiPort { - boolean readContextualExternalFile(String questionnaireId, String filePath) throws GenesisException; + boolean readContextualExternalFile(String collectionInstrumentId, String filePath) throws GenesisException; - ContextualExternalVariableModel findByQuestionnaireIdAndInterrogationId(String questionnaireId, String interrogationId); + ContextualExternalVariableModel findByCollectionInstrumentIdAndInterrogationId(String collectionInstrumentId, String interrogationId); } diff --git a/src/main/java/fr/insee/genesis/domain/ports/api/ContextualPreviousVariableApiPort.java b/src/main/java/fr/insee/genesis/domain/ports/api/ContextualPreviousVariableApiPort.java index 1580d103..dd6aa4df 100644 --- a/src/main/java/fr/insee/genesis/domain/ports/api/ContextualPreviousVariableApiPort.java +++ b/src/main/java/fr/insee/genesis/domain/ports/api/ContextualPreviousVariableApiPort.java @@ -4,7 +4,7 @@ import fr.insee.genesis.exceptions.GenesisException; public interface ContextualPreviousVariableApiPort { - boolean readContextualPreviousFile(String questionnaireId, String sourceState, String filePath) throws GenesisException; + boolean readContextualPreviousFile(String collectionInstrumentId, String sourceState, String filePath) throws GenesisException; - ContextualPreviousVariableModel findByQuestionnaireIdAndInterrogationId(String questionnaireId, String interrogationId); + ContextualPreviousVariableModel findByCollectionInstrumentIdAndInterrogationId(String collectionInstrumentId, String interrogationId); } diff --git a/src/main/java/fr/insee/genesis/domain/ports/api/ContextualVariableApiPort.java b/src/main/java/fr/insee/genesis/domain/ports/api/ContextualVariableApiPort.java index 25ce22e0..e32f0e4c 100644 --- a/src/main/java/fr/insee/genesis/domain/ports/api/ContextualVariableApiPort.java +++ b/src/main/java/fr/insee/genesis/domain/ports/api/ContextualVariableApiPort.java @@ -5,6 +5,6 @@ import fr.insee.genesis.infrastructure.utils.FileUtils; public interface ContextualVariableApiPort { - ContextualVariableModel getContextualVariable(String questionnaireId, String interrogationId); - int saveContextualVariableFiles(String questionnaireId, FileUtils fileUtils) throws GenesisException; + ContextualVariableModel getContextualVariable(String collectionInstrumentId, String interrogationId); + int saveContextualVariableFiles(String collectionInstrumentId, FileUtils fileUtils) throws GenesisException; } \ No newline at end of file diff --git a/src/main/java/fr/insee/genesis/domain/ports/spi/ContextualExternalVariablePersistancePort.java b/src/main/java/fr/insee/genesis/domain/ports/spi/ContextualExternalVariablePersistancePort.java index 37ed1ef7..7b499021 100644 --- a/src/main/java/fr/insee/genesis/domain/ports/spi/ContextualExternalVariablePersistancePort.java +++ b/src/main/java/fr/insee/genesis/domain/ports/spi/ContextualExternalVariablePersistancePort.java @@ -5,10 +5,10 @@ import java.util.List; public interface ContextualExternalVariablePersistancePort { - void backup(String questionnaireId); - void deleteBackup(String questionnaireId); - void restoreBackup(String questionnaireId); + void backup(String collectionInstrumentId); + void deleteBackup(String collectionInstrumentId); + void restoreBackup(String collectionInstrumentId); void saveAll(List contextualPreviousVariableModelList); - void delete(String questionnaireId); - ContextualExternalVariableModel findByQuestionnaireIdAndInterrogationId(String questionnaireId, String interrogationId); + void delete(String collectionInstrumentId); + ContextualExternalVariableModel findByCollectionInstrumentIdAndInterrogationId(String collectionInstrumentId, String interrogationId); } \ No newline at end of file diff --git a/src/main/java/fr/insee/genesis/domain/ports/spi/ContextualPreviousVariablePersistancePort.java b/src/main/java/fr/insee/genesis/domain/ports/spi/ContextualPreviousVariablePersistancePort.java index d791e91b..57d92931 100644 --- a/src/main/java/fr/insee/genesis/domain/ports/spi/ContextualPreviousVariablePersistancePort.java +++ b/src/main/java/fr/insee/genesis/domain/ports/spi/ContextualPreviousVariablePersistancePort.java @@ -5,10 +5,10 @@ import java.util.List; public interface ContextualPreviousVariablePersistancePort { - void backup(String questionnaireId); - void deleteBackup(String questionnaireId); - void restoreBackup(String questionnaireId); + void backup(String collectionInstrumentId); + void deleteBackup(String collectionInstrumentId); + void restoreBackup(String collectionInstrumentId); void saveAll(List contextualPreviousVariableModelList); - void delete(String questionnaireId); - ContextualPreviousVariableModel findByQuestionnaireIdAndInterrogationId(String questionnaireId, String interrogationId); + void delete(String collectionInstrumentId); + ContextualPreviousVariableModel findByCollectionInstrumentIdAndInterrogationId(String collectionInstrumentId, String interrogationId); } \ No newline at end of file diff --git a/src/main/java/fr/insee/genesis/domain/service/contextualvariable/ContextualVariableJsonService.java b/src/main/java/fr/insee/genesis/domain/service/contextualvariable/ContextualVariableJsonService.java index 780ee420..1f338bba 100644 --- a/src/main/java/fr/insee/genesis/domain/service/contextualvariable/ContextualVariableJsonService.java +++ b/src/main/java/fr/insee/genesis/domain/service/contextualvariable/ContextualVariableJsonService.java @@ -40,7 +40,7 @@ public ContextualVariableJsonService(ContextualPreviousVariableApiPort contextua } @Override - public ContextualVariableModel getContextualVariable(String questionnaireId, String interrogationId) { + public ContextualVariableModel getContextualVariable(String collectionInstrumentId, String interrogationId) { ContextualVariableModel contextualVariableModel = ContextualVariableModel.builder() .interrogationId(interrogationId) .contextualPrevious(new ArrayList<>()) @@ -48,8 +48,8 @@ public ContextualVariableModel getContextualVariable(String questionnaireId, Str .build(); ContextualPreviousVariableModel contextualPreviousVariableModel = - contextualPreviousVariableApiPort.findByQuestionnaireIdAndInterrogationId( - questionnaireId, + contextualPreviousVariableApiPort.findByCollectionInstrumentIdAndInterrogationId( + collectionInstrumentId, interrogationId ); @@ -60,8 +60,8 @@ public ContextualVariableModel getContextualVariable(String questionnaireId, Str } ContextualExternalVariableModel contextualExternalVariableModel = - contextualExternalVariableApiPort.findByQuestionnaireIdAndInterrogationId( - questionnaireId, + contextualExternalVariableApiPort.findByCollectionInstrumentIdAndInterrogationId( + collectionInstrumentId, interrogationId ); @@ -75,11 +75,11 @@ public ContextualVariableModel getContextualVariable(String questionnaireId, Str } @Override - public int saveContextualVariableFiles(String questionnaireId, FileUtils fileUtils) throws GenesisException { + public int saveContextualVariableFiles(String collectionInstrumentId, FileUtils fileUtils) throws GenesisException { int fileCount = 0; for (Mode mode : Mode.values()) { - try (Stream filePaths = Files.list(Path.of(fileUtils.getDataFolder(questionnaireId, + try (Stream filePaths = Files.list(Path.of(fileUtils.getDataFolder(collectionInstrumentId, mode.getFolder() , null)))) { Iterator it = filePaths @@ -87,9 +87,9 @@ public int saveContextualVariableFiles(String questionnaireId, FileUtils fileUti .iterator(); while (it.hasNext()) { Path jsonFilePath = it.next(); - if (processContextualVariableFile(questionnaireId, jsonFilePath)) { + if (processContextualVariableFile(collectionInstrumentId, jsonFilePath)) { //If the file is indeed a contextual variables file and had been processed - moveFile(questionnaireId, mode, fileUtils, jsonFilePath.toString()); + moveFile(collectionInstrumentId, mode, fileUtils, jsonFilePath.toString()); fileCount++; } } @@ -102,9 +102,9 @@ public int saveContextualVariableFiles(String questionnaireId, FileUtils fileUti return fileCount; } - private static void moveFile(String questionnaireId, Mode mode, FileUtils fileUtils, String filePath) throws GenesisException { + private static void moveFile(String collectionInstrumentId, Mode mode, FileUtils fileUtils, String filePath) throws GenesisException { try { - fileUtils.moveFiles(Path.of(filePath), fileUtils.getDoneFolder(questionnaireId, mode.getFolder())); + fileUtils.moveFiles(Path.of(filePath), fileUtils.getDoneFolder(collectionInstrumentId, mode.getFolder())); } catch (IOException e) { throw new GenesisException(500, "Error while moving file to done : %s".formatted(e.toString())); } @@ -147,13 +147,13 @@ private VariableQualityToolDto extractValue(Object variable, String variableName /** * @return true if any contextual variable part found in file, false otherwise */ - private boolean processContextualVariableFile(String questionnaireId, Path jsonFilePath) throws GenesisException { + private boolean processContextualVariableFile(String collectionInstrumentId, Path jsonFilePath) throws GenesisException { return contextualPreviousVariableApiPort.readContextualPreviousFile( - questionnaireId.toUpperCase(), + collectionInstrumentId.toUpperCase(), null, jsonFilePath.toString() ) || contextualExternalVariableApiPort.readContextualExternalFile( - questionnaireId.toUpperCase(), + collectionInstrumentId.toUpperCase(), jsonFilePath.toString() ); } diff --git a/src/main/java/fr/insee/genesis/domain/service/contextualvariable/external/ContextualExternalVariableJsonService.java b/src/main/java/fr/insee/genesis/domain/service/contextualvariable/external/ContextualExternalVariableJsonService.java index cd1f1d3e..f2c835a0 100644 --- a/src/main/java/fr/insee/genesis/domain/service/contextualvariable/external/ContextualExternalVariableJsonService.java +++ b/src/main/java/fr/insee/genesis/domain/service/contextualvariable/external/ContextualExternalVariableJsonService.java @@ -34,10 +34,10 @@ public ContextualExternalVariableJsonService(ContextualExternalVariablePersistan } @Override - public boolean readContextualExternalFile(String questionnaireId, String filePath) throws GenesisException { + public boolean readContextualExternalFile(String collectionInstrumentId, String filePath) throws GenesisException { try(FileInputStream inputStream = new FileInputStream(filePath)){ JsonFactory jsonFactory = new JsonFactory(); - moveCollectionToBackup(questionnaireId); + moveCollectionToBackup(collectionInstrumentId); try(JsonParser jsonParser = jsonFactory.createParser(inputStream)){ List toSave = new ArrayList<>(); goToContextualExternalToken(jsonParser); @@ -51,7 +51,7 @@ public boolean readContextualExternalFile(String questionnaireId, String filePat while (jsonParser.currentToken() != JsonToken.END_ARRAY) { ContextualExternalVariableModel contextualExternalVariableModel = readNextContextualExternal( jsonParser, - questionnaireId + collectionInstrumentId ); checkModel(contextualExternalVariableModel, jsonParser, savedInterrogationIds); @@ -67,21 +67,21 @@ public boolean readContextualExternalFile(String questionnaireId, String filePat contextualExternalVariablePersistancePort.saveAll(toSave); savedCount += toSave.size(); log.info("Reached end of contextual external file, saved %d interrogations".formatted(savedCount)); - contextualExternalVariablePersistancePort.deleteBackup(questionnaireId); + contextualExternalVariablePersistancePort.deleteBackup(collectionInstrumentId); return true; } }catch (JsonParseException jpe){ - contextualExternalVariablePersistancePort.restoreBackup(questionnaireId); + contextualExternalVariablePersistancePort.restoreBackup(collectionInstrumentId); throw new GenesisException(400, "JSON Parsing exception : %s".formatted(jpe.toString())); }catch (IOException ioe){ - contextualExternalVariablePersistancePort.restoreBackup(questionnaireId); + contextualExternalVariablePersistancePort.restoreBackup(collectionInstrumentId); throw new GenesisException(500, ioe.toString()); } } @Override - public ContextualExternalVariableModel findByQuestionnaireIdAndInterrogationId(String questionnaireId, String interrogationId) { - return contextualExternalVariablePersistancePort.findByQuestionnaireIdAndInterrogationId(questionnaireId, interrogationId); + public ContextualExternalVariableModel findByCollectionInstrumentIdAndInterrogationId(String collectionInstrumentId, String interrogationId) { + return contextualExternalVariablePersistancePort.findByCollectionInstrumentIdAndInterrogationId(collectionInstrumentId, interrogationId); } private static void goToContextualExternalToken(JsonParser jsonParser) throws IOException{ @@ -99,9 +99,9 @@ private static void goToContextualExternalToken(JsonParser jsonParser) throws IO } } - private void moveCollectionToBackup(String questionnaireId) { - contextualExternalVariablePersistancePort.backup(questionnaireId); - contextualExternalVariablePersistancePort.delete(questionnaireId); + private void moveCollectionToBackup(String collectionInstrumentId) { + contextualExternalVariablePersistancePort.backup(collectionInstrumentId); + contextualExternalVariablePersistancePort.delete(collectionInstrumentId); } private long saveBlock(List toSave, long savedCount) { @@ -125,13 +125,13 @@ private static void checkModel(ContextualExternalVariableModel contextualExterna } private ContextualExternalVariableModel readNextContextualExternal(JsonParser jsonParser, - String questionnaireId + String collectionInstrumentId ) throws IOException { if(jsonParser.currentToken() != JsonToken.START_OBJECT){ throw new JsonParseException("Expected { on line %d, got token %s".formatted(jsonParser.currentLocation().getLineNr(), jsonParser.currentToken())); } ContextualExternalVariableModel contextualExternalVariableModel = ContextualExternalVariableModel.builder() - .questionnaireId(questionnaireId) + .collectionInstrumentId(collectionInstrumentId) .variables(new HashMap<>()) .build(); jsonParser.nextToken(); diff --git a/src/main/java/fr/insee/genesis/domain/service/contextualvariable/previous/ContextualPreviousVariableJsonService.java b/src/main/java/fr/insee/genesis/domain/service/contextualvariable/previous/ContextualPreviousVariableJsonService.java index 1af94d4c..bc6af354 100644 --- a/src/main/java/fr/insee/genesis/domain/service/contextualvariable/previous/ContextualPreviousVariableJsonService.java +++ b/src/main/java/fr/insee/genesis/domain/service/contextualvariable/previous/ContextualPreviousVariableJsonService.java @@ -34,12 +34,12 @@ public ContextualPreviousVariableJsonService(ContextualPreviousVariablePersistan } @Override - public boolean readContextualPreviousFile(String questionnaireId, + public boolean readContextualPreviousFile(String collectionInstrumentId, String sourceState, String filePath) throws GenesisException { try(FileInputStream inputStream = new FileInputStream(filePath)){ checkSourceStateLength(sourceState); - moveCollectionToBackup(questionnaireId); + moveCollectionToBackup(collectionInstrumentId); JsonFactory jsonFactory = new JsonFactory(); try (JsonParser jsonParser = jsonFactory.createParser(inputStream)) { @@ -55,7 +55,7 @@ public boolean readContextualPreviousFile(String questionnaireId, while (jsonParser.currentToken() != JsonToken.END_ARRAY) { ContextualPreviousVariableModel contextualPreviousVariableModel = readNextContextualPrevious( jsonParser, - questionnaireId, + collectionInstrumentId, sourceState ); @@ -71,22 +71,22 @@ public boolean readContextualPreviousFile(String questionnaireId, } savedCount = saveBlock(toSave, savedCount); log.info("Reached end of contextual previous file, saved %d interrogations".formatted(savedCount)); - contextualPreviousVariablePersistancePort.deleteBackup(questionnaireId); + contextualPreviousVariablePersistancePort.deleteBackup(collectionInstrumentId); return true; } }catch (JsonParseException jpe){ - contextualPreviousVariablePersistancePort.restoreBackup(questionnaireId); + contextualPreviousVariablePersistancePort.restoreBackup(collectionInstrumentId); throw new GenesisException(400, "JSON Parsing exception : %s".formatted(jpe.toString())); }catch (IOException ioe){ - contextualPreviousVariablePersistancePort.restoreBackup(questionnaireId); + contextualPreviousVariablePersistancePort.restoreBackup(collectionInstrumentId); throw new GenesisException(500, ioe.toString()); } } @Override - public ContextualPreviousVariableModel findByQuestionnaireIdAndInterrogationId(String questionnaireId, String interrogationId) { - return contextualPreviousVariablePersistancePort.findByQuestionnaireIdAndInterrogationId( - questionnaireId, + public ContextualPreviousVariableModel findByCollectionInstrumentIdAndInterrogationId(String collectionInstrumentId, String interrogationId) { + return contextualPreviousVariablePersistancePort.findByCollectionInstrumentIdAndInterrogationId( + collectionInstrumentId, interrogationId ); } @@ -98,9 +98,9 @@ private long saveBlock(List toSave, long savedC return savedCount; } - private void moveCollectionToBackup(String questionnaireId) { - contextualPreviousVariablePersistancePort.backup(questionnaireId); - contextualPreviousVariablePersistancePort.delete(questionnaireId); + private void moveCollectionToBackup(String collectionInstrumentId) { + contextualPreviousVariablePersistancePort.backup(collectionInstrumentId); + contextualPreviousVariablePersistancePort.delete(collectionInstrumentId); } private static void checkSourceStateLength(String sourceState) throws GenesisException { @@ -125,14 +125,14 @@ private static void goToEditedPreviousToken(JsonParser jsonParser) throws IOExce } private ContextualPreviousVariableModel readNextContextualPrevious(JsonParser jsonParser, - String questionnaireId, + String collectionInstrumentId, String sourceState ) throws IOException { if(jsonParser.currentToken() != JsonToken.START_OBJECT){ throw new JsonParseException("Expected { on line %d, got token %s".formatted(jsonParser.currentLocation().getLineNr(), jsonParser.currentToken())); } ContextualPreviousVariableModel contextualPreviousVariableModel = ContextualPreviousVariableModel.builder() - .questionnaireId(questionnaireId) + .collectionInstrumentId(collectionInstrumentId) .sourceState(sourceState) .variables(new HashMap<>()) .build(); diff --git a/src/main/java/fr/insee/genesis/infrastructure/adapter/ContextualExternalVariableMongoAdapter.java b/src/main/java/fr/insee/genesis/infrastructure/adapter/ContextualExternalVariableMongoAdapter.java index 80ca813e..155526dd 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/adapter/ContextualExternalVariableMongoAdapter.java +++ b/src/main/java/fr/insee/genesis/infrastructure/adapter/ContextualExternalVariableMongoAdapter.java @@ -29,12 +29,17 @@ public ContextualExternalVariableMongoAdapter(ContextualExternalVariableMongoDBR } @Override - public void backup(String questionnaireId) { - deleteBackup(questionnaireId); - MatchOperation match = Aggregation.match(Criteria.where("questionnaireId").is(questionnaireId)); + public void backup(String collectionInstrumentId) { + deleteBackup(collectionInstrumentId); + MatchOperation match = Aggregation.match( + new Criteria().orOperator( + Criteria.where("questionnaireId").is(collectionInstrumentId), + Criteria.where("collectionInstrumentId").is(collectionInstrumentId) + ) + ); MergeOperation merge = Aggregation .merge() - .intoCollection(getFormattedCollection(questionnaireId)) + .intoCollection(getFormattedCollection(collectionInstrumentId)) .whenMatched(MergeOperation.WhenDocumentsMatch.replaceDocument()) .whenDocumentsDontMatch(MergeOperation.WhenDocumentsDontMatch.insertNewDocument()) .build(); @@ -56,8 +61,8 @@ public void deleteBackup(String questionnaireId) { } @Override - public void restoreBackup(String questionnaireId) { - delete(questionnaireId); + public void restoreBackup(String collectionInstrumentId) { + delete(collectionInstrumentId); MergeOperation merge = Aggregation .merge() .intoCollection("editedExternalResponses") @@ -67,7 +72,7 @@ public void restoreBackup(String questionnaireId) { Aggregation aggregation = Aggregation.newAggregation(merge); - mongoTemplate.aggregate(aggregation, getFormattedCollection(questionnaireId), + mongoTemplate.aggregate(aggregation, getFormattedCollection(collectionInstrumentId), ContextualExternalVariableDocument.class); } @@ -79,19 +84,27 @@ public void saveAll(List contextualExternalVari } @Override - public void delete(String questionnaireId) { - repository.deleteByQuestionnaireId(questionnaireId); + public void delete(String collectionInstrumentId) { + repository.deleteByCollectionInstrumentId(collectionInstrumentId); + // For older documents + repository.deleteByQuestionnaireId(collectionInstrumentId); } @Override - public ContextualExternalVariableModel findByQuestionnaireIdAndInterrogationId(String questionnaireId, String interrogationId) { + public ContextualExternalVariableModel findByCollectionInstrumentIdAndInterrogationId(String collectionInstrumentId, String interrogationId) { List contextualExternalVariableDocumentList = - repository.findByQuestionnaireIdAndInterrogationId(questionnaireId, interrogationId); + repository.findByQuestionnaireIdAndInterrogationId(collectionInstrumentId, interrogationId); + // For older documents + List docIdentifiedByCollectionInstrumentId = + repository.findByCollectionInstrumentIdAndInterrogationId(collectionInstrumentId, interrogationId); + if (!docIdentifiedByCollectionInstrumentId.isEmpty()){ + contextualExternalVariableDocumentList.addAll(docIdentifiedByCollectionInstrumentId); + } if(contextualExternalVariableDocumentList.isEmpty()){ return null; } if(contextualExternalVariableDocumentList.size() > 1){ - log.warn("More than 1 contextual external response document for questionnaire {}, interrogation {}", questionnaireId, interrogationId); + log.warn("More than 1 contextual external response document for collection instrument {}, interrogation {}", collectionInstrumentId, interrogationId); } return ContextualExternalVariableDocumentMapper.INSTANCE.documentToModel(contextualExternalVariableDocumentList.getFirst()); } diff --git a/src/main/java/fr/insee/genesis/infrastructure/adapter/ContextualPreviousVariableMongoAdapter.java b/src/main/java/fr/insee/genesis/infrastructure/adapter/ContextualPreviousVariableMongoAdapter.java index d8cf34f2..f5ffa985 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/adapter/ContextualPreviousVariableMongoAdapter.java +++ b/src/main/java/fr/insee/genesis/infrastructure/adapter/ContextualPreviousVariableMongoAdapter.java @@ -29,12 +29,17 @@ public ContextualPreviousVariableMongoAdapter(ContextualPreviousVariableMongoDBR } @Override - public void backup(String questionnaireId) { - deleteBackup(questionnaireId); - MatchOperation match = Aggregation.match(Criteria.where("questionnaireId").is(questionnaireId)); + public void backup(String collectionInstrumentId) { + deleteBackup(collectionInstrumentId); + MatchOperation match = Aggregation.match( + new Criteria().orOperator( + Criteria.where("questionnaireId").is(collectionInstrumentId), + Criteria.where("collectionInstrumentId").is(collectionInstrumentId) + ) + ); MergeOperation merge = Aggregation .merge() - .intoCollection(getFormattedCollection(questionnaireId)) + .intoCollection(getFormattedCollection(collectionInstrumentId)) .whenMatched(MergeOperation.WhenDocumentsMatch.replaceDocument()) .whenDocumentsDontMatch(MergeOperation.WhenDocumentsDontMatch.insertNewDocument()) .build(); @@ -44,20 +49,20 @@ public void backup(String questionnaireId) { mongoTemplate.aggregate(aggregation, "editedPreviousResponses", ContextualPreviousVariableDocument.class); } - private static String getFormattedCollection(String questionnaireId) { - return "editedPreviousResponses_%s_backup".formatted(questionnaireId); + private static String getFormattedCollection(String collectionInstrumentId) { + return "editedPreviousResponses_%s_backup".formatted(collectionInstrumentId); } @Override - public void deleteBackup(String questionnaireId) { - if (mongoTemplate.collectionExists(getFormattedCollection(questionnaireId))){ - mongoTemplate.dropCollection(getFormattedCollection(questionnaireId)); + public void deleteBackup(String collectionInstrumentId) { + if (mongoTemplate.collectionExists(getFormattedCollection(collectionInstrumentId))){ + mongoTemplate.dropCollection(getFormattedCollection(collectionInstrumentId)); } } @Override - public void restoreBackup(String questionnaireId) { - delete(questionnaireId); + public void restoreBackup(String collectionInstrumentId) { + delete(collectionInstrumentId); MergeOperation merge = Aggregation .merge() .intoCollection("editedPreviousResponses") @@ -67,7 +72,7 @@ public void restoreBackup(String questionnaireId) { Aggregation aggregation = Aggregation.newAggregation(merge); - mongoTemplate.aggregate(aggregation, getFormattedCollection(questionnaireId), + mongoTemplate.aggregate(aggregation, getFormattedCollection(collectionInstrumentId), ContextualPreviousVariableDocument.class); } @@ -79,19 +84,25 @@ public void saveAll(List contextualPreviousVari } @Override - public void delete(String questionnaireId) { - repository.deleteByQuestionnaireId(questionnaireId); + public void delete(String collectionInstrumentId) { + repository.deleteByCollectionInstrumentId(collectionInstrumentId); + repository.deleteByQuestionnaireId(collectionInstrumentId); } @Override - public ContextualPreviousVariableModel findByQuestionnaireIdAndInterrogationId(String questionnaireId, String interrogationId) { + public ContextualPreviousVariableModel findByCollectionInstrumentIdAndInterrogationId(String collectionInstrumentId, String interrogationId) { List contextualPreviousVariableDocumentList = - repository.findByQuestionnaireIdAndInterrogationId(questionnaireId, interrogationId); + repository.findByQuestionnaireIdAndInterrogationId(collectionInstrumentId, interrogationId); + List docIdentifiedByCollectionInstrumentId = + repository.findByCollectionInstrumentIdAndInterrogationId(collectionInstrumentId, interrogationId); + if (!docIdentifiedByCollectionInstrumentId.isEmpty()){ + contextualPreviousVariableDocumentList.addAll(docIdentifiedByCollectionInstrumentId); + } if(contextualPreviousVariableDocumentList.isEmpty()){ return null; } if(contextualPreviousVariableDocumentList.size() > 1){ - log.warn("More than 1 contextual previous response document for questionnaire {}, interrogation {}", questionnaireId, interrogationId); + log.warn("More than 1 contextual previous response document for collection instrument {}, interrogation {}", collectionInstrumentId, interrogationId); } return ContextualPreviousVariableDocumentMapper.INSTANCE.documentToModel(contextualPreviousVariableDocumentList.getFirst()); } diff --git a/src/main/java/fr/insee/genesis/infrastructure/document/contextualexternal/ContextualExternalVariableDocument.java b/src/main/java/fr/insee/genesis/infrastructure/document/contextualexternal/ContextualExternalVariableDocument.java index 1a15caa5..288e6bf7 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/document/contextualexternal/ContextualExternalVariableDocument.java +++ b/src/main/java/fr/insee/genesis/infrastructure/document/contextualexternal/ContextualExternalVariableDocument.java @@ -1,5 +1,6 @@ package fr.insee.genesis.infrastructure.document.contextualexternal; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import fr.insee.genesis.Constants; import lombok.Data; import org.springframework.data.annotation.Id; @@ -12,11 +13,20 @@ @Data @Document(collection = Constants.MONGODB_CONTEXTUAL_EXTERNAL_COLLECTION_NAME) @CompoundIndex(name = "questionnaireId_1_interrogationId_1", def = "{'questionnaireId': 1, 'interrogationId': 1}") +@CompoundIndex(name = "collectionInstrumentId_1_interrogationId_1", def = "{'collectionInstrumentId': 1, 'interrogationId': 1}") +@JsonIgnoreProperties(ignoreUnknown = true) public class ContextualExternalVariableDocument { @Id private String id; + + /** + * @deprecated it will be replaced by collectionInstrumentId + */ + @Deprecated(forRemoval = true) @Indexed String questionnaireId; + @Indexed + String collectionInstrumentId; String interrogationId; Map variables; } diff --git a/src/main/java/fr/insee/genesis/infrastructure/document/contextualprevious/ContextualPreviousVariableDocument.java b/src/main/java/fr/insee/genesis/infrastructure/document/contextualprevious/ContextualPreviousVariableDocument.java index f3ef57a9..f258e76f 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/document/contextualprevious/ContextualPreviousVariableDocument.java +++ b/src/main/java/fr/insee/genesis/infrastructure/document/contextualprevious/ContextualPreviousVariableDocument.java @@ -1,5 +1,6 @@ package fr.insee.genesis.infrastructure.document.contextualprevious; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import fr.insee.genesis.Constants; import lombok.Data; import org.springframework.data.annotation.Id; @@ -12,11 +13,19 @@ @Data @Document(collection = Constants.MONGODB_CONTEXTUAL_PREVIOUS_COLLECTION_NAME) @CompoundIndex(name = "questionnaireId_1_interrogationId_1", def = "{'questionnaireId': 1, 'interrogationId': 1}") +@CompoundIndex(name = "collectionInstrumentId_1_interrogationId_1", def = "{'collectionInstrumentId': 1, 'interrogationId': 1}") +@JsonIgnoreProperties(ignoreUnknown = true) public class ContextualPreviousVariableDocument { @Id private String id; + /** + * @deprecated it will be replaced by collectionInstrumentId + */ + @Deprecated(forRemoval = true) @Indexed String questionnaireId; + @Indexed + String collectionInstrumentId; String interrogationId; Map variables; String sourceState; diff --git a/src/main/java/fr/insee/genesis/infrastructure/mappers/ContextualExternalVariableDocumentMapper.java b/src/main/java/fr/insee/genesis/infrastructure/mappers/ContextualExternalVariableDocumentMapper.java index 5366caff..991b64fc 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/mappers/ContextualExternalVariableDocumentMapper.java +++ b/src/main/java/fr/insee/genesis/infrastructure/mappers/ContextualExternalVariableDocumentMapper.java @@ -2,7 +2,9 @@ import fr.insee.genesis.domain.model.contextualvariable.ContextualExternalVariableModel; import fr.insee.genesis.infrastructure.document.contextualexternal.ContextualExternalVariableDocument; +import org.mapstruct.AfterMapping; import org.mapstruct.Mapper; +import org.mapstruct.MappingTarget; import org.mapstruct.factory.Mappers; import java.util.List; @@ -18,4 +20,14 @@ public interface ContextualExternalVariableDocumentMapper { List listDocumentToListModel(List rawDataDocumentList); List listModelToListDocument(List rawDataModelList); + + @AfterMapping + default void fillModelAfterRead(ContextualExternalVariableDocument doc, + @MappingTarget ContextualExternalVariableModel model) { + + if (model.getCollectionInstrumentId() == null) { + model.setCollectionInstrumentId(doc.getQuestionnaireId()); + } + + } } diff --git a/src/main/java/fr/insee/genesis/infrastructure/mappers/ContextualPreviousVariableDocumentMapper.java b/src/main/java/fr/insee/genesis/infrastructure/mappers/ContextualPreviousVariableDocumentMapper.java index d4919fc6..ba72e779 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/mappers/ContextualPreviousVariableDocumentMapper.java +++ b/src/main/java/fr/insee/genesis/infrastructure/mappers/ContextualPreviousVariableDocumentMapper.java @@ -2,7 +2,9 @@ import fr.insee.genesis.domain.model.contextualvariable.ContextualPreviousVariableModel; import fr.insee.genesis.infrastructure.document.contextualprevious.ContextualPreviousVariableDocument; +import org.mapstruct.AfterMapping; import org.mapstruct.Mapper; +import org.mapstruct.MappingTarget; import org.mapstruct.factory.Mappers; import java.util.List; @@ -18,4 +20,14 @@ public interface ContextualPreviousVariableDocumentMapper { List listDocumentToListModel(List rawDataDocumentList); List listModelToListDocument(List rawDataModelList); + + @AfterMapping + default void fillModelAfterRead(ContextualPreviousVariableDocument doc, + @MappingTarget ContextualPreviousVariableModel model) { + + if (model.getCollectionInstrumentId() == null) { + model.setCollectionInstrumentId(doc.getQuestionnaireId()); + } + + } } diff --git a/src/main/java/fr/insee/genesis/infrastructure/repository/ContextualExternalVariableMongoDBRepository.java b/src/main/java/fr/insee/genesis/infrastructure/repository/ContextualExternalVariableMongoDBRepository.java index e9bc22ec..9b264e19 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/repository/ContextualExternalVariableMongoDBRepository.java +++ b/src/main/java/fr/insee/genesis/infrastructure/repository/ContextualExternalVariableMongoDBRepository.java @@ -10,7 +10,11 @@ @Repository public interface ContextualExternalVariableMongoDBRepository extends MongoRepository { void deleteByQuestionnaireId(String questionnaireId); + void deleteByCollectionInstrumentId(String collectionInstrumentId); @Query(value = "{ 'questionnaireId' : ?0, 'interrogationId' : ?1 }") List findByQuestionnaireIdAndInterrogationId(String questionnaireId, String interrogationId); + + @Query(value = "{ 'collectionInstrumentId' : ?0, 'interrogationId' : ?1 }") + List findByCollectionInstrumentIdAndInterrogationId(String collectionInstrumentId, String interrogationId); } diff --git a/src/main/java/fr/insee/genesis/infrastructure/repository/ContextualPreviousVariableMongoDBRepository.java b/src/main/java/fr/insee/genesis/infrastructure/repository/ContextualPreviousVariableMongoDBRepository.java index f5604747..9adcecdf 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/repository/ContextualPreviousVariableMongoDBRepository.java +++ b/src/main/java/fr/insee/genesis/infrastructure/repository/ContextualPreviousVariableMongoDBRepository.java @@ -10,7 +10,10 @@ @Repository public interface ContextualPreviousVariableMongoDBRepository extends MongoRepository { void deleteByQuestionnaireId(String questionnaireId); + void deleteByCollectionInstrumentId(String collectionInstrumentId); @Query(value = "{ 'questionnaireId' : ?0, 'interrogationId' : ?1 }") List findByQuestionnaireIdAndInterrogationId(String questionnaireId, String interrogationId); + @Query(value = "{ 'collectionInstrumentId' : ?0, 'interrogationId' : ?1 }") + List findByCollectionInstrumentIdAndInterrogationId(String collectionInstrumentId, String interrogationId); } diff --git a/src/test/java/fr/insee/genesis/controller/rest/responses/ContextualVariableControllerTest.java b/src/test/java/fr/insee/genesis/controller/rest/responses/ContextualVariableControllerTest.java index b1c8490b..5a3e2c5d 100644 --- a/src/test/java/fr/insee/genesis/controller/rest/responses/ContextualVariableControllerTest.java +++ b/src/test/java/fr/insee/genesis/controller/rest/responses/ContextualVariableControllerTest.java @@ -211,7 +211,7 @@ void saveContextualVariables_previous_test( ContextualPreviousVariableDocument firstDocument = filter.getFirst(); Assertions.assertNull(firstDocument.getSourceState()); - Assertions.assertEquals(QUESTIONNAIRE_ID_PREVIOUS,firstDocument.getQuestionnaireId()); + Assertions.assertEquals(QUESTIONNAIRE_ID_PREVIOUS,firstDocument.getCollectionInstrumentId()); Assertions.assertEquals(expectedVarCount,firstDocument.getVariables().size()); @@ -304,7 +304,7 @@ void saveContextualVariables_external_test( String interrogationId, Assertions.assertEquals(1,filter.size()); ContextualExternalVariableDocument firstDoc = filter.getFirst(); - Assertions.assertEquals(QUESTIONNAIRE_ID_EXTERNAL, firstDoc.getQuestionnaireId()); + Assertions.assertEquals(QUESTIONNAIRE_ID_EXTERNAL, firstDoc.getCollectionInstrumentId()); Assertions.assertEquals(expectedVarCount, firstDoc.getVariables().size()); assertionsForDoc.accept(firstDoc); @@ -422,7 +422,7 @@ private void testOKCase(String sourceState) throws IOException { Assertions.assertNotNull(filter.getFirst().getSourceState()); Assertions.assertEquals(sourceState, filter.getFirst().getSourceState()); } - Assertions.assertEquals(QUESTIONNAIRE_ID_PREVIOUS,filter.getFirst().getQuestionnaireId()); + Assertions.assertEquals(QUESTIONNAIRE_ID_PREVIOUS,filter.getFirst().getCollectionInstrumentId()); Assertions.assertEquals(15,filter.getFirst().getVariables().size()); assertVariable(filter.getFirst(), "TEXTECOURT", ""); @@ -455,7 +455,7 @@ private void testOKCase(String sourceState) throws IOException { Assertions.assertNotNull(filter.getFirst().getSourceState()); Assertions.assertEquals(sourceState, filter.getFirst().getSourceState()); } - Assertions.assertEquals(QUESTIONNAIRE_ID_PREVIOUS,filter.getFirst().getQuestionnaireId()); + Assertions.assertEquals(QUESTIONNAIRE_ID_PREVIOUS,filter.getFirst().getCollectionInstrumentId()); Assertions.assertEquals(14,filter.getFirst().getVariables().size()); assertVariable(filter.getFirst(), "TEXTECOURT", "test previous"); @@ -511,17 +511,17 @@ void readPreviousJson_override_interrogation_id( String interrogationId, .stream().filter(doc -> doc.getInterrogationId().equals(interrogationId)).toList(); Assertions.assertEquals(1, filter.size()); ContextualPreviousVariableDocument firstDoc = filter.getFirst(); - Assertions.assertEquals(QUESTIONNAIRE_ID_PREVIOUS, firstDoc.getQuestionnaireId()); + Assertions.assertEquals(QUESTIONNAIRE_ID_PREVIOUS, firstDoc.getCollectionInstrumentId()); Assertions.assertAll(interrogationId + " metadata", - () -> Assertions.assertEquals(QUESTIONNAIRE_ID_PREVIOUS, firstDoc.getQuestionnaireId()), + () -> Assertions.assertEquals(QUESTIONNAIRE_ID_PREVIOUS, firstDoc.getCollectionInstrumentId()), () -> Assertions.assertEquals(expectedVarCount, firstDoc.getVariables().size()) ); assertionsForDoc.accept(firstDoc); Assertions.assertTrue( previousStub.getMongoStub().get(Constants.MONGODB_CONTEXTUAL_PREVIOUS_COLLECTION_NAME).stream() - .noneMatch(d -> "AUTO108".equals(d.getInterrogationId())), + .noneMatch(d -> "AUTO108".equals(d.getCollectionInstrumentId())), "AUTO108 ne devrait plus être présent après override" ); } @@ -680,7 +680,7 @@ void readExternalJson_test( String interrogationId, Assertions.assertEquals(1,filter.size()); ContextualExternalVariableDocument firstDoc = filter.getFirst(); - Assertions.assertEquals(QUESTIONNAIRE_ID_EXTERNAL, firstDoc.getQuestionnaireId()); + Assertions.assertEquals(QUESTIONNAIRE_ID_EXTERNAL, firstDoc.getCollectionInstrumentId()); Assertions.assertEquals(expectedVarCount, firstDoc.getVariables().size()); assertionsForDoc.accept(firstDoc); } @@ -767,7 +767,7 @@ void readExternalJson_override_interrogation_id( String interrogationId, externalStub.getMongoStub().get(Constants.MONGODB_CONTEXTUAL_EXTERNAL_COLLECTION_NAME) .stream().filter(doc -> doc.getInterrogationId().equals(interrogationId)).toList(); Assertions.assertEquals(1,filter.size()); - Assertions.assertEquals(QUESTIONNAIRE_ID_EXTERNAL,filter.getFirst().getQuestionnaireId()); + Assertions.assertEquals(QUESTIONNAIRE_ID_EXTERNAL,filter.getFirst().getCollectionInstrumentId()); Assertions.assertEquals(expectedVarCount,filter.getFirst().getVariables().size()); assertionsForDoc.accept(filter.getFirst()); @@ -775,7 +775,7 @@ void readExternalJson_override_interrogation_id( String interrogationId, // L'ancien AUTO208 ne doit plus être présent Assertions.assertTrue( externalStub.getMongoStub().get(Constants.MONGODB_CONTEXTUAL_EXTERNAL_COLLECTION_NAME) - .stream().noneMatch(d -> "AUTO208".equals(d.getInterrogationId())), + .stream().noneMatch(d -> "AUTO208".equals(d.getCollectionInstrumentId())), "AUTO208 ne devrait plus être présent après override" ); } diff --git a/src/test/java/fr/insee/genesis/stubs/ContextualExternalVariablePersistancePortStub.java b/src/test/java/fr/insee/genesis/stubs/ContextualExternalVariablePersistancePortStub.java index 6f5cdd2a..7e31e527 100644 --- a/src/test/java/fr/insee/genesis/stubs/ContextualExternalVariablePersistancePortStub.java +++ b/src/test/java/fr/insee/genesis/stubs/ContextualExternalVariablePersistancePortStub.java @@ -52,12 +52,13 @@ public void saveAll(List contextualExternalVari @Override public void delete(String questionnaireId) { mongoStub.get(Constants.MONGODB_CONTEXTUAL_EXTERNAL_COLLECTION_NAME).removeIf( - contextualExternalVariableDocument -> contextualExternalVariableDocument.getQuestionnaireId().equals(questionnaireId) + extDoc -> (extDoc.getQuestionnaireId() != null && extDoc.getQuestionnaireId().equals(questionnaireId)) || + (extDoc.getCollectionInstrumentId() != null && extDoc.getCollectionInstrumentId().equals(questionnaireId)) ); } @Override - public ContextualExternalVariableModel findByQuestionnaireIdAndInterrogationId(String questionnaireId, String interrogationId) { + public ContextualExternalVariableModel findByCollectionInstrumentIdAndInterrogationId(String questionnaireId, String interrogationId) { List contextualExternalVariableDocumentList = mongoStub.get(Constants.MONGODB_CONTEXTUAL_EXTERNAL_COLLECTION_NAME).stream().filter( contextualExternalVariableDocument -> contextualExternalVariableDocument.getQuestionnaireId().equals(questionnaireId) diff --git a/src/test/java/fr/insee/genesis/stubs/ContextualPreviousVariablePersistancePortStub.java b/src/test/java/fr/insee/genesis/stubs/ContextualPreviousVariablePersistancePortStub.java index 255c6a9b..3fa59bbd 100644 --- a/src/test/java/fr/insee/genesis/stubs/ContextualPreviousVariablePersistancePortStub.java +++ b/src/test/java/fr/insee/genesis/stubs/ContextualPreviousVariablePersistancePortStub.java @@ -52,12 +52,15 @@ public void saveAll(List contextualPreviousVari @Override public void delete(String questionnaireId) { mongoStub.get(Constants.MONGODB_CONTEXTUAL_PREVIOUS_COLLECTION_NAME).removeIf( - contextualPreviousVariableDocument -> contextualPreviousVariableDocument.getQuestionnaireId().equals(questionnaireId) + previousDoc -> + (previousDoc.getQuestionnaireId()!=null && previousDoc.getQuestionnaireId().equals(questionnaireId)) || + (previousDoc.getCollectionInstrumentId() != null && previousDoc.getCollectionInstrumentId().equals(questionnaireId)) + ); } @Override - public ContextualPreviousVariableModel findByQuestionnaireIdAndInterrogationId(String questionnaireId, String interrogationId) { + public ContextualPreviousVariableModel findByCollectionInstrumentIdAndInterrogationId(String questionnaireId, String interrogationId) { List contextualPreviousVariableDocumentList = mongoStub.get(Constants.MONGODB_CONTEXTUAL_PREVIOUS_COLLECTION_NAME).stream().filter( contextualPreviousVariableDocument -> From ef1204fdf2587c442d5da500730c2a3175c55abb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Henninger?= Date: Fri, 21 Nov 2025 17:07:09 +0100 Subject: [PATCH 11/42] feat: WIP modelefiliere add last extraction date --- .../genesis/controller/rest/JsonExtractionController.java | 8 ++++---- .../model/extraction/json/LastJsonExtractionModel.java | 2 +- .../domain/ports/api/LastJsonExtractionApiPort.java | 2 +- .../ports/spi/LastJsonExtractionPersistencePort.java | 2 +- .../service/extraction/LastJsonExtractionService.java | 6 +++--- .../adapter/LastJsonExtractionMongoAdapter.java | 6 +++--- .../extraction/json/LastJsonExtractionDocument.java | 2 +- 7 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/main/java/fr/insee/genesis/controller/rest/JsonExtractionController.java b/src/main/java/fr/insee/genesis/controller/rest/JsonExtractionController.java index 595ec153..79d60d16 100644 --- a/src/main/java/fr/insee/genesis/controller/rest/JsonExtractionController.java +++ b/src/main/java/fr/insee/genesis/controller/rest/JsonExtractionController.java @@ -30,11 +30,11 @@ public class JsonExtractionController { @PutMapping(path = "/json") @PreAuthorize("hasAnyRole('USER_KRAFTWERK','SCHEDULER')") public ResponseEntity saveLastJsonExtractionDate( - @RequestParam("questionnaireId") String questionnaireId, + @RequestParam("collectionInstrumentId") String collectionInstrumentId, @RequestParam(value = "mode", required = false) Mode mode){ LocalDateTime extractDate = LocalDateTime.now(); LastJsonExtractionModel extract = LastJsonExtractionModel.builder() - .questionnaireModelId(questionnaireId) + .collectionInstrumentId(collectionInstrumentId) .mode(mode) .lastExtractionDate(extractDate) .build(); @@ -46,10 +46,10 @@ public ResponseEntity saveLastJsonExtractionDate( @GetMapping(path = "/json") @PreAuthorize("hasAnyRole('USER_KRAFTWERK','SCHEDULER')") public ResponseEntity getLastJsonExtractionDate( - @RequestParam("questionnaireId") String questionnaireId, + @RequestParam("collectionInstrumentId") String collectionInstrumentId, @RequestParam(value = "mode", required = false) Mode mode){ try{ - LastJsonExtractionModel lastJsonExtraction = lastJsonExtractionApiPort.getLastExtractionDate(questionnaireId,mode); + LastJsonExtractionModel lastJsonExtraction = lastJsonExtractionApiPort.getLastExtractionDate(collectionInstrumentId,mode); return ResponseEntity.ok(new LastExtractionResponseDto(lastJsonExtraction.getLastExtractionDate())); } catch (GenesisException e){ return ResponseEntity.notFound().build(); diff --git a/src/main/java/fr/insee/genesis/domain/model/extraction/json/LastJsonExtractionModel.java b/src/main/java/fr/insee/genesis/domain/model/extraction/json/LastJsonExtractionModel.java index 861d9e6e..478f3e8b 100644 --- a/src/main/java/fr/insee/genesis/domain/model/extraction/json/LastJsonExtractionModel.java +++ b/src/main/java/fr/insee/genesis/domain/model/extraction/json/LastJsonExtractionModel.java @@ -16,7 +16,7 @@ public class LastJsonExtractionModel { @Id private String id; //Used to remove warning - String questionnaireModelId; + String collectionInstrumentId; Mode mode; LocalDateTime lastExtractionDate; } diff --git a/src/main/java/fr/insee/genesis/domain/ports/api/LastJsonExtractionApiPort.java b/src/main/java/fr/insee/genesis/domain/ports/api/LastJsonExtractionApiPort.java index 196f3b10..331e2b3d 100644 --- a/src/main/java/fr/insee/genesis/domain/ports/api/LastJsonExtractionApiPort.java +++ b/src/main/java/fr/insee/genesis/domain/ports/api/LastJsonExtractionApiPort.java @@ -7,5 +7,5 @@ public interface LastJsonExtractionApiPort { void recordDate(LastJsonExtractionModel extraction); - LastJsonExtractionModel getLastExtractionDate(String questionnaireModelId, Mode mode) throws GenesisException; + LastJsonExtractionModel getLastExtractionDate(String collectionInstrumentId, Mode mode) throws GenesisException; } diff --git a/src/main/java/fr/insee/genesis/domain/ports/spi/LastJsonExtractionPersistencePort.java b/src/main/java/fr/insee/genesis/domain/ports/spi/LastJsonExtractionPersistencePort.java index 39265b50..53a58e0f 100644 --- a/src/main/java/fr/insee/genesis/domain/ports/spi/LastJsonExtractionPersistencePort.java +++ b/src/main/java/fr/insee/genesis/domain/ports/spi/LastJsonExtractionPersistencePort.java @@ -6,5 +6,5 @@ public interface LastJsonExtractionPersistencePort { void save(LastJsonExtractionModel extraction); - LastJsonExtractionModel getLastExecutionDate(String questionnaireModelId, Mode mode) throws GenesisException; + LastJsonExtractionModel getLastExecutionDate(String collectionInstrumentId, Mode mode) throws GenesisException; } diff --git a/src/main/java/fr/insee/genesis/domain/service/extraction/LastJsonExtractionService.java b/src/main/java/fr/insee/genesis/domain/service/extraction/LastJsonExtractionService.java index dca95323..7a4d9fb0 100644 --- a/src/main/java/fr/insee/genesis/domain/service/extraction/LastJsonExtractionService.java +++ b/src/main/java/fr/insee/genesis/domain/service/extraction/LastJsonExtractionService.java @@ -25,15 +25,15 @@ public LastJsonExtractionService(LastJsonExtractionPersistencePort extractionPer @Override public void recordDate(LastJsonExtractionModel extraction) { // Create a unique ID based on the questionnaire and the mode. - extraction.setId(String.format("%s_%s",extraction.getQuestionnaireModelId(),extraction.getMode())); + extraction.setId(String.format("%s_%s",extraction.getCollectionInstrumentId(),extraction.getMode())); // save() does an insert if the id doesn't exist, otherwise an update extractionPersistencePort.save(extraction); } @Override - public LastJsonExtractionModel getLastExtractionDate(String questionnaireModelId, Mode mode) throws GenesisException { - return extractionPersistencePort.getLastExecutionDate(questionnaireModelId,mode); + public LastJsonExtractionModel getLastExtractionDate(String collectionInstrumentId, Mode mode) throws GenesisException { + return extractionPersistencePort.getLastExecutionDate(collectionInstrumentId,mode); } diff --git a/src/main/java/fr/insee/genesis/infrastructure/adapter/LastJsonExtractionMongoAdapter.java b/src/main/java/fr/insee/genesis/infrastructure/adapter/LastJsonExtractionMongoAdapter.java index 89635f6b..0cbc0923 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/adapter/LastJsonExtractionMongoAdapter.java +++ b/src/main/java/fr/insee/genesis/infrastructure/adapter/LastJsonExtractionMongoAdapter.java @@ -32,13 +32,13 @@ public void save(LastJsonExtractionModel extraction) { } @Override - public LastJsonExtractionModel getLastExecutionDate(String questionnaireModelId, Mode mode) throws GenesisException { - String id = String.format("%s_%s",questionnaireModelId, mode); + public LastJsonExtractionModel getLastExecutionDate(String collectionInstrumentId, Mode mode) throws GenesisException { + String id = String.format("%s_%s",collectionInstrumentId, mode); Optional extraction = extractionRepository.findById(id); if (extraction.isPresent()) { return LastJsonExtractionDocumentMapper.INSTANCE.documentToModel(extraction.get()); } else { - String message = String.format("No extraction date found for questionnaire %s and mode %s",questionnaireModelId,mode==null?null:mode.getModeName()); + String message = String.format("No extraction date found for collection instrument %s and mode %s",collectionInstrumentId,mode==null?null:mode.getModeName()); throw new GenesisException(404,message); } } diff --git a/src/main/java/fr/insee/genesis/infrastructure/document/extraction/json/LastJsonExtractionDocument.java b/src/main/java/fr/insee/genesis/infrastructure/document/extraction/json/LastJsonExtractionDocument.java index 0e942f27..137af63c 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/document/extraction/json/LastJsonExtractionDocument.java +++ b/src/main/java/fr/insee/genesis/infrastructure/document/extraction/json/LastJsonExtractionDocument.java @@ -14,7 +14,7 @@ public class LastJsonExtractionDocument { @Id private String id; - private String questionnaireModelId; + private String collectionInstrumentId; private Mode mode; private LocalDateTime lastExtractionDate; From 11eccdcd9d990d7e53f715b810f3cf2dfa0d7c8c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Henninger?= Date: Mon, 24 Nov 2025 14:49:16 +0100 Subject: [PATCH 12/42] feat: WIP modelefiliere add lunaticmodel --- .../model/lunaticmodel/LunaticModelModel.java | 2 +- .../domain/ports/api/LunaticModelApiPort.java | 5 ++--- .../spi/LunaticModelPersistancePort.java | 2 +- .../lunaticmodel/LunaticModelService.java | 10 +++++----- ...ontextualExternalVariableMongoAdapter.java | 17 +++++++--------- ...ontextualPreviousVariableMongoAdapter.java | 17 +++++++--------- .../adapter/LunaticModelMongoAdapter.java | 20 ++++++++++++++----- .../adapter/SurveyUnitMongoAdapter.java | 10 ++++------ .../lunaticmodel/LunaticModelDocument.java | 8 ++++++++ .../mappers/LunaticModelMapper.java | 16 +++++++++++++-- .../LunaticModelMongoDBRepository.java | 3 +++ ...LastJsonExtractionPersistencePortStub.java | 4 ++-- .../stubs/LunaticModelPersistanceStub.java | 7 ++++--- 13 files changed, 73 insertions(+), 48 deletions(-) diff --git a/src/main/java/fr/insee/genesis/domain/model/lunaticmodel/LunaticModelModel.java b/src/main/java/fr/insee/genesis/domain/model/lunaticmodel/LunaticModelModel.java index 22d3f3d2..395bd342 100644 --- a/src/main/java/fr/insee/genesis/domain/model/lunaticmodel/LunaticModelModel.java +++ b/src/main/java/fr/insee/genesis/domain/model/lunaticmodel/LunaticModelModel.java @@ -7,7 +7,7 @@ @Builder public record LunaticModelModel ( - String questionnaireId, + String collectionInstrumentId, Map lunaticModel, LocalDateTime recordDate ){} diff --git a/src/main/java/fr/insee/genesis/domain/ports/api/LunaticModelApiPort.java b/src/main/java/fr/insee/genesis/domain/ports/api/LunaticModelApiPort.java index 6150ac77..bc85632e 100644 --- a/src/main/java/fr/insee/genesis/domain/ports/api/LunaticModelApiPort.java +++ b/src/main/java/fr/insee/genesis/domain/ports/api/LunaticModelApiPort.java @@ -6,7 +6,6 @@ import java.util.Map; public interface LunaticModelApiPort { - void save(String questionnaireId, Map lunaticModel); - - LunaticModelModel get(String questionnaireId) throws GenesisException; + void save(String collectionInstrumentId, Map lunaticModel); + LunaticModelModel get(String collectionInstrumentId) throws GenesisException; } diff --git a/src/main/java/fr/insee/genesis/domain/ports/spi/LunaticModelPersistancePort.java b/src/main/java/fr/insee/genesis/domain/ports/spi/LunaticModelPersistancePort.java index 81bcfd81..632991f2 100644 --- a/src/main/java/fr/insee/genesis/domain/ports/spi/LunaticModelPersistancePort.java +++ b/src/main/java/fr/insee/genesis/domain/ports/spi/LunaticModelPersistancePort.java @@ -8,5 +8,5 @@ public interface LunaticModelPersistancePort { void save(LunaticModelModel lunaticModelModel); - List find(String questionnaireId); + List find(String collectionInstrumentId); } diff --git a/src/main/java/fr/insee/genesis/domain/service/lunaticmodel/LunaticModelService.java b/src/main/java/fr/insee/genesis/domain/service/lunaticmodel/LunaticModelService.java index 5325e1dc..f4f46983 100644 --- a/src/main/java/fr/insee/genesis/domain/service/lunaticmodel/LunaticModelService.java +++ b/src/main/java/fr/insee/genesis/domain/service/lunaticmodel/LunaticModelService.java @@ -23,10 +23,10 @@ public LunaticModelService(LunaticModelPersistancePort lunaticModelPersistancePo } @Override - public void save(String questionnaireId, Map lunaticModel) { + public void save(String collectionInstrumentId, Map lunaticModel) { lunaticModelPersistancePort.save( LunaticModelModel.builder() - .questionnaireId(questionnaireId) + .collectionInstrumentId(collectionInstrumentId) .lunaticModel(lunaticModel) .recordDate(LocalDateTime.now()) .build() @@ -34,10 +34,10 @@ public void save(String questionnaireId, Map lunaticModel) { } @Override - public LunaticModelModel get(String questionnaireId) throws GenesisException { - if(lunaticModelPersistancePort.find(questionnaireId).isEmpty()){ + public LunaticModelModel get(String collectionInstrumentId) throws GenesisException { + if(lunaticModelPersistancePort.find(collectionInstrumentId).isEmpty()){ throw new GenesisException(404,"Questionnaire not found"); } - return LunaticModelMapper.INSTANCE.documentToModel(lunaticModelPersistancePort.find(questionnaireId).getFirst()); + return LunaticModelMapper.INSTANCE.documentToModel(lunaticModelPersistancePort.find(collectionInstrumentId).getFirst()); } } diff --git a/src/main/java/fr/insee/genesis/infrastructure/adapter/ContextualExternalVariableMongoAdapter.java b/src/main/java/fr/insee/genesis/infrastructure/adapter/ContextualExternalVariableMongoAdapter.java index 155526dd..6211c1b0 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/adapter/ContextualExternalVariableMongoAdapter.java +++ b/src/main/java/fr/insee/genesis/infrastructure/adapter/ContextualExternalVariableMongoAdapter.java @@ -14,6 +14,7 @@ import org.springframework.data.mongodb.core.query.Criteria; import org.springframework.stereotype.Service; +import java.util.ArrayList; import java.util.List; @Slf4j @@ -92,20 +93,16 @@ public void delete(String collectionInstrumentId) { @Override public ContextualExternalVariableModel findByCollectionInstrumentIdAndInterrogationId(String collectionInstrumentId, String interrogationId) { - List contextualExternalVariableDocumentList = - repository.findByQuestionnaireIdAndInterrogationId(collectionInstrumentId, interrogationId); + List results = new ArrayList<>(); + results.addAll(repository.findByQuestionnaireIdAndInterrogationId(collectionInstrumentId, interrogationId)); // For older documents - List docIdentifiedByCollectionInstrumentId = - repository.findByCollectionInstrumentIdAndInterrogationId(collectionInstrumentId, interrogationId); - if (!docIdentifiedByCollectionInstrumentId.isEmpty()){ - contextualExternalVariableDocumentList.addAll(docIdentifiedByCollectionInstrumentId); - } - if(contextualExternalVariableDocumentList.isEmpty()){ + results.addAll(repository.findByCollectionInstrumentIdAndInterrogationId(collectionInstrumentId, interrogationId)); + if(results.isEmpty()){ return null; } - if(contextualExternalVariableDocumentList.size() > 1){ + if(results.size() > 1){ log.warn("More than 1 contextual external response document for collection instrument {}, interrogation {}", collectionInstrumentId, interrogationId); } - return ContextualExternalVariableDocumentMapper.INSTANCE.documentToModel(contextualExternalVariableDocumentList.getFirst()); + return ContextualExternalVariableDocumentMapper.INSTANCE.documentToModel(results.getFirst()); } } diff --git a/src/main/java/fr/insee/genesis/infrastructure/adapter/ContextualPreviousVariableMongoAdapter.java b/src/main/java/fr/insee/genesis/infrastructure/adapter/ContextualPreviousVariableMongoAdapter.java index f5ffa985..8a46393e 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/adapter/ContextualPreviousVariableMongoAdapter.java +++ b/src/main/java/fr/insee/genesis/infrastructure/adapter/ContextualPreviousVariableMongoAdapter.java @@ -14,6 +14,7 @@ import org.springframework.data.mongodb.core.query.Criteria; import org.springframework.stereotype.Service; +import java.util.ArrayList; import java.util.List; @Slf4j @@ -91,19 +92,15 @@ public void delete(String collectionInstrumentId) { @Override public ContextualPreviousVariableModel findByCollectionInstrumentIdAndInterrogationId(String collectionInstrumentId, String interrogationId) { - List contextualPreviousVariableDocumentList = - repository.findByQuestionnaireIdAndInterrogationId(collectionInstrumentId, interrogationId); - List docIdentifiedByCollectionInstrumentId = - repository.findByCollectionInstrumentIdAndInterrogationId(collectionInstrumentId, interrogationId); - if (!docIdentifiedByCollectionInstrumentId.isEmpty()){ - contextualPreviousVariableDocumentList.addAll(docIdentifiedByCollectionInstrumentId); - } - if(contextualPreviousVariableDocumentList.isEmpty()){ + List results = new ArrayList<>(); + results.addAll(repository.findByQuestionnaireIdAndInterrogationId(collectionInstrumentId, interrogationId)); + results.addAll(repository.findByCollectionInstrumentIdAndInterrogationId(collectionInstrumentId, interrogationId)); + if(results.isEmpty()){ return null; } - if(contextualPreviousVariableDocumentList.size() > 1){ + if(results.size() > 1){ log.warn("More than 1 contextual previous response document for collection instrument {}, interrogation {}", collectionInstrumentId, interrogationId); } - return ContextualPreviousVariableDocumentMapper.INSTANCE.documentToModel(contextualPreviousVariableDocumentList.getFirst()); + return ContextualPreviousVariableDocumentMapper.INSTANCE.documentToModel(results.getFirst()); } } diff --git a/src/main/java/fr/insee/genesis/infrastructure/adapter/LunaticModelMongoAdapter.java b/src/main/java/fr/insee/genesis/infrastructure/adapter/LunaticModelMongoAdapter.java index 44f3cc79..5903ea6f 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/adapter/LunaticModelMongoAdapter.java +++ b/src/main/java/fr/insee/genesis/infrastructure/adapter/LunaticModelMongoAdapter.java @@ -7,13 +7,13 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.data.mongodb.core.MongoTemplate; +import org.springframework.data.mongodb.core.query.Criteria; import org.springframework.data.mongodb.core.query.Update; import org.springframework.stereotype.Service; +import java.util.ArrayList; import java.util.List; -import static org.springframework.data.mongodb.core.query.Criteria.where; - @Slf4j @Service @Qualifier("lunaticModelMongoAdapter") @@ -28,14 +28,24 @@ public LunaticModelMongoAdapter(LunaticModelMongoDBRepository repository, MongoT @Override public void save(LunaticModelModel lunaticModelModel) { + Criteria criteria = new Criteria().orOperator( + Criteria.where("questionnaireId").is(lunaticModelModel.collectionInstrumentId()), + Criteria.where("collectionInstrumentId").is(lunaticModelModel.collectionInstrumentId()) + ); mongoTemplate.update(LunaticModelDocument.class) - .matching(where("questionnaireId").is(lunaticModelModel.questionnaireId())) + .matching(criteria) .apply(new Update().set("lunaticModel", lunaticModelModel.lunaticModel())) .upsert(); } @Override - public List find(String questionnaireId) { - return repository.findByQuestionnaireId(questionnaireId); + public List find(String collectionInstrumentId) { + List results = new ArrayList<>(); + results.addAll(repository.findByCollectionInstrumentId(collectionInstrumentId)); + results.addAll(repository.findByQuestionnaireId(collectionInstrumentId)); + return results.stream() + .distinct() + .toList(); } + } diff --git a/src/main/java/fr/insee/genesis/infrastructure/adapter/SurveyUnitMongoAdapter.java b/src/main/java/fr/insee/genesis/infrastructure/adapter/SurveyUnitMongoAdapter.java index dae26c3e..c9278070 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/adapter/SurveyUnitMongoAdapter.java +++ b/src/main/java/fr/insee/genesis/infrastructure/adapter/SurveyUnitMongoAdapter.java @@ -47,13 +47,11 @@ public void saveAll(List surveyUnitModels) { @Override public List findByIds(String interrogationId, String collectionInstrumentId) { - List surveyUnits = mongoRepository.findByInterrogationIdAndCollectionInstrumentId(interrogationId, collectionInstrumentId); + List results = new ArrayList<>(); + results.addAll(mongoRepository.findByInterrogationIdAndCollectionInstrumentId(interrogationId, collectionInstrumentId)); // To ensure compatibility with older documents (with questionnaireId instead of collectionInstrumentId) - List surveyUnitsLegacy = mongoRepository.findByInterrogationIdAndQuestionnaireId(interrogationId, collectionInstrumentId); - if(!surveyUnitsLegacy.isEmpty()){ - surveyUnits.addAll(surveyUnitsLegacy); - } - return surveyUnits.isEmpty() ? Collections.emptyList() : SurveyUnitDocumentMapper.INSTANCE.listDocumentToListModel(surveyUnits); + results.addAll(mongoRepository.findByInterrogationIdAndQuestionnaireId(interrogationId, collectionInstrumentId)); + return results.isEmpty() ? Collections.emptyList() : SurveyUnitDocumentMapper.INSTANCE.listDocumentToListModel(results); } diff --git a/src/main/java/fr/insee/genesis/infrastructure/document/lunaticmodel/LunaticModelDocument.java b/src/main/java/fr/insee/genesis/infrastructure/document/lunaticmodel/LunaticModelDocument.java index d04fbebd..6a8d1d74 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/document/lunaticmodel/LunaticModelDocument.java +++ b/src/main/java/fr/insee/genesis/infrastructure/document/lunaticmodel/LunaticModelDocument.java @@ -6,10 +6,18 @@ import java.time.LocalDateTime; import java.util.Map; +/* It is not possible to respect these rules in a record (compilation error). +I choose to keep the record here (this choice can be challenged)*/ +@SuppressWarnings({ + "java:S1123", // deprecated annotation usage + "java:S6355" // Add 'since' and/or 'forRemoval' arguments to the @Deprecated annotation +}) @Builder @Document(collection = "lunaticmodels") public record LunaticModelDocument ( + @Deprecated String questionnaireId, + String collectionInstrumentId, Map lunaticModel, LocalDateTime recordDate ){} diff --git a/src/main/java/fr/insee/genesis/infrastructure/mappers/LunaticModelMapper.java b/src/main/java/fr/insee/genesis/infrastructure/mappers/LunaticModelMapper.java index d5702cbd..07e8beec 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/mappers/LunaticModelMapper.java +++ b/src/main/java/fr/insee/genesis/infrastructure/mappers/LunaticModelMapper.java @@ -1,12 +1,13 @@ package fr.insee.genesis.infrastructure.mappers; + import fr.insee.genesis.domain.model.lunaticmodel.LunaticModelModel; import fr.insee.genesis.infrastructure.document.lunaticmodel.LunaticModelDocument; -import org.mapstruct.Mapper; +import org.mapstruct.AfterMapping; +import org.mapstruct.MappingTarget; import org.mapstruct.factory.Mappers; import java.util.List; -@Mapper public interface LunaticModelMapper { LunaticModelMapper INSTANCE = Mappers.getMapper(LunaticModelMapper.class); @@ -17,4 +18,15 @@ public interface LunaticModelMapper { List listDocumentToListModel(List documentList); List listModelToListDocument(List modelList); + + @SuppressWarnings("removal") + @AfterMapping + default void fillModelAfterRead( + LunaticModelDocument doc, + @MappingTarget LunaticModelModel.LunaticModelModelBuilder builder + ) { + if (doc.collectionInstrumentId() == null && doc.questionnaireId()!=null) { + builder.collectionInstrumentId(doc.questionnaireId()); + } + } } diff --git a/src/main/java/fr/insee/genesis/infrastructure/repository/LunaticModelMongoDBRepository.java b/src/main/java/fr/insee/genesis/infrastructure/repository/LunaticModelMongoDBRepository.java index faa4faa8..01ab6d86 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/repository/LunaticModelMongoDBRepository.java +++ b/src/main/java/fr/insee/genesis/infrastructure/repository/LunaticModelMongoDBRepository.java @@ -12,4 +12,7 @@ public interface LunaticModelMongoDBRepository extends CrudRepository findByQuestionnaireId(String questionnaireId); + + @Query("{'collectionInstrumentId' : ?0}") + List findByCollectionInstrumentId(String collectionInstrumentId); } diff --git a/src/test/java/fr/insee/genesis/stubs/LastJsonExtractionPersistencePortStub.java b/src/test/java/fr/insee/genesis/stubs/LastJsonExtractionPersistencePortStub.java index 15a94c46..defb61a0 100644 --- a/src/test/java/fr/insee/genesis/stubs/LastJsonExtractionPersistencePortStub.java +++ b/src/test/java/fr/insee/genesis/stubs/LastJsonExtractionPersistencePortStub.java @@ -21,9 +21,9 @@ public void save(LastJsonExtractionModel extraction) { } @Override - public LastJsonExtractionModel getLastExecutionDate(String questionnaireModelId, Mode mode) throws GenesisException { + public LastJsonExtractionModel getLastExecutionDate(String collectionInstrumentId, Mode mode) throws GenesisException { LastJsonExtractionDocument extraction = mongoStub.stream() - .filter(doc -> doc.getId().equals(String.format("%s_%s",questionnaireModelId,mode))) + .filter(doc -> doc.getId().equals(String.format("%s_%s",collectionInstrumentId,mode))) .findFirst() .orElseThrow(); return LastJsonExtractionDocumentMapper.INSTANCE.documentToModel(extraction); diff --git a/src/test/java/fr/insee/genesis/stubs/LunaticModelPersistanceStub.java b/src/test/java/fr/insee/genesis/stubs/LunaticModelPersistanceStub.java index 5beaebcb..8b0ca13d 100644 --- a/src/test/java/fr/insee/genesis/stubs/LunaticModelPersistanceStub.java +++ b/src/test/java/fr/insee/genesis/stubs/LunaticModelPersistanceStub.java @@ -16,14 +16,15 @@ public class LunaticModelPersistanceStub implements LunaticModelPersistancePort @Override public void save(LunaticModelModel lunaticModelModel) { //Replace - mongoStub.removeIf(document -> document.questionnaireId().equals(lunaticModelModel.questionnaireId())); + mongoStub.removeIf(document -> document.questionnaireId().equals(lunaticModelModel.collectionInstrumentId())); + mongoStub.removeIf(document -> document.collectionInstrumentId().equals(lunaticModelModel.collectionInstrumentId())); mongoStub.add(LunaticModelMapper.INSTANCE.modelToDocument(lunaticModelModel)); } @Override - public List find(String questionnaireId) { + public List find(String collectionInstrumentId) { return mongoStub.stream().filter( - document -> document.questionnaireId().equals(questionnaireId) + document -> (document.questionnaireId().equals(collectionInstrumentId) || document.collectionInstrumentId().equals(collectionInstrumentId)) ).toList(); } } From fdb3e3e7effe2bc8037d234564a57eee155ad329 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Henninger?= Date: Mon, 24 Nov 2025 15:13:52 +0100 Subject: [PATCH 13/42] chore: delete unused code --- .../model/variabletype/VariableTypeModel.java | 13 ------ .../domain/ports/api/VariableTypeApiPort.java | 8 ---- .../spi/VariableTypePersistancePort.java | 12 ----- .../variabletype/VariableTypeService.java | 39 ---------------- .../adapter/VariableTypeMongoAdapter.java | 44 ------------------- .../variabletype/VariableTypeDocument.java | 16 ------- .../mappers/VariableTypeDocumentMapper.java | 21 --------- .../VariableTypeMongoDBRepository.java | 14 ------ .../config/CucumberSpringConfiguration.java | 3 -- .../controller/rest/ControllerAccessTest.java | 3 -- .../stubs/VariableTypePersistanceStub.java | 30 ------------- 11 files changed, 203 deletions(-) delete mode 100644 src/main/java/fr/insee/genesis/domain/model/variabletype/VariableTypeModel.java delete mode 100644 src/main/java/fr/insee/genesis/domain/ports/api/VariableTypeApiPort.java delete mode 100644 src/main/java/fr/insee/genesis/domain/ports/spi/VariableTypePersistancePort.java delete mode 100644 src/main/java/fr/insee/genesis/domain/service/variabletype/VariableTypeService.java delete mode 100644 src/main/java/fr/insee/genesis/infrastructure/adapter/VariableTypeMongoAdapter.java delete mode 100644 src/main/java/fr/insee/genesis/infrastructure/document/variabletype/VariableTypeDocument.java delete mode 100644 src/main/java/fr/insee/genesis/infrastructure/mappers/VariableTypeDocumentMapper.java delete mode 100644 src/main/java/fr/insee/genesis/infrastructure/repository/VariableTypeMongoDBRepository.java delete mode 100644 src/test/java/fr/insee/genesis/stubs/VariableTypePersistanceStub.java diff --git a/src/main/java/fr/insee/genesis/domain/model/variabletype/VariableTypeModel.java b/src/main/java/fr/insee/genesis/domain/model/variabletype/VariableTypeModel.java deleted file mode 100644 index c0e79a4d..00000000 --- a/src/main/java/fr/insee/genesis/domain/model/variabletype/VariableTypeModel.java +++ /dev/null @@ -1,13 +0,0 @@ -package fr.insee.genesis.domain.model.variabletype; - -import fr.insee.bpm.metadata.model.VariablesMap; -import fr.insee.genesis.domain.model.surveyunit.Mode; -import lombok.Builder; - -@Builder -public record VariableTypeModel ( - String campaignId, - String questionnaireId, - Mode mode, - VariablesMap variablesMap -){} diff --git a/src/main/java/fr/insee/genesis/domain/ports/api/VariableTypeApiPort.java b/src/main/java/fr/insee/genesis/domain/ports/api/VariableTypeApiPort.java deleted file mode 100644 index 17220bfe..00000000 --- a/src/main/java/fr/insee/genesis/domain/ports/api/VariableTypeApiPort.java +++ /dev/null @@ -1,8 +0,0 @@ -package fr.insee.genesis.domain.ports.api; - -import fr.insee.bpm.metadata.model.VariablesMap; -import fr.insee.genesis.domain.model.surveyunit.Mode; - -public interface VariableTypeApiPort { - void saveMetadatas(String campaignId, String questionnaireId, Mode mode, VariablesMap variablesMap); -} diff --git a/src/main/java/fr/insee/genesis/domain/ports/spi/VariableTypePersistancePort.java b/src/main/java/fr/insee/genesis/domain/ports/spi/VariableTypePersistancePort.java deleted file mode 100644 index 3c2e3639..00000000 --- a/src/main/java/fr/insee/genesis/domain/ports/spi/VariableTypePersistancePort.java +++ /dev/null @@ -1,12 +0,0 @@ -package fr.insee.genesis.domain.ports.spi; - -import fr.insee.genesis.domain.model.surveyunit.Mode; -import fr.insee.genesis.domain.model.variabletype.VariableTypeModel; -import fr.insee.genesis.infrastructure.document.variabletype.VariableTypeDocument; -import org.springframework.cache.annotation.Cacheable; - -public interface VariableTypePersistancePort { - void save(VariableTypeModel variableTypeModel); - @Cacheable("variableTypes") - VariableTypeDocument find(String campaignId, String questionnaireId, Mode mode); -} diff --git a/src/main/java/fr/insee/genesis/domain/service/variabletype/VariableTypeService.java b/src/main/java/fr/insee/genesis/domain/service/variabletype/VariableTypeService.java deleted file mode 100644 index 52ed64f9..00000000 --- a/src/main/java/fr/insee/genesis/domain/service/variabletype/VariableTypeService.java +++ /dev/null @@ -1,39 +0,0 @@ -package fr.insee.genesis.domain.service.variabletype; - -import fr.insee.bpm.metadata.model.Variable; -import fr.insee.bpm.metadata.model.VariablesMap; -import fr.insee.genesis.domain.model.surveyunit.Mode; -import fr.insee.genesis.domain.model.variabletype.VariableTypeModel; -import fr.insee.genesis.domain.ports.api.VariableTypeApiPort; -import fr.insee.genesis.domain.ports.spi.VariableTypePersistancePort; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.stereotype.Service; - -@Service -public class VariableTypeService implements VariableTypeApiPort { - @Qualifier("variableTypeMongoAdapter") - private final VariableTypePersistancePort variableTypePersistancePort; - - @Autowired - public VariableTypeService(VariableTypePersistancePort variableTypePersistancePort) { - this.variableTypePersistancePort = variableTypePersistancePort; - } - - @Override - public void saveMetadatas(String campaignId, String questionnaireId, Mode mode, VariablesMap variablesMap) { - VariableTypeModel variableTypeModel = VariableTypeModel.builder() - .campaignId(campaignId) - .questionnaireId(questionnaireId) - .mode(mode) - .variablesMap(new VariablesMap()) - .build(); - - for(String variableName : variablesMap.getVariables().keySet()){ - Variable bpmVariable = variablesMap.getVariable(variableName); - variableTypeModel.variablesMap().putVariable(bpmVariable); - } - - variableTypePersistancePort.save(variableTypeModel); - } -} diff --git a/src/main/java/fr/insee/genesis/infrastructure/adapter/VariableTypeMongoAdapter.java b/src/main/java/fr/insee/genesis/infrastructure/adapter/VariableTypeMongoAdapter.java deleted file mode 100644 index 9b12c07c..00000000 --- a/src/main/java/fr/insee/genesis/infrastructure/adapter/VariableTypeMongoAdapter.java +++ /dev/null @@ -1,44 +0,0 @@ -package fr.insee.genesis.infrastructure.adapter; - -import fr.insee.genesis.domain.model.surveyunit.Mode; -import fr.insee.genesis.domain.model.variabletype.VariableTypeModel; -import fr.insee.genesis.domain.ports.spi.VariableTypePersistancePort; -import fr.insee.genesis.infrastructure.document.variabletype.VariableTypeDocument; -import fr.insee.genesis.infrastructure.mappers.VariableTypeDocumentMapper; -import fr.insee.genesis.infrastructure.repository.VariableTypeMongoDBRepository; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.cache.annotation.Cacheable; -import org.springframework.data.mongodb.core.FindAndReplaceOptions; -import org.springframework.data.mongodb.core.MongoTemplate; -import org.springframework.data.mongodb.core.query.Criteria; -import org.springframework.data.mongodb.core.query.Query; -import org.springframework.stereotype.Service; - -@Service -@Qualifier("variableTypeMongoAdapter") -public class VariableTypeMongoAdapter implements VariableTypePersistancePort { - private final VariableTypeMongoDBRepository variableTypeMongoDBRepository; - private final MongoTemplate mongoTemplate; - - @Autowired - public VariableTypeMongoAdapter(VariableTypeMongoDBRepository variableTypeMongoDBRepository, MongoTemplate mongoTemplate) { - this.variableTypeMongoDBRepository = variableTypeMongoDBRepository; - this.mongoTemplate = mongoTemplate; - } - - @Override - public void save(VariableTypeModel variableTypeModel) { - mongoTemplate.update(VariableTypeDocument.class) - .matching(Query.query(Criteria.where("campaignId").is(variableTypeModel.campaignId()))) - .replaceWith(VariableTypeDocumentMapper.INSTANCE.modelToDocument(variableTypeModel)) - .withOptions(FindAndReplaceOptions.options().upsert()) - .findAndReplace(); - } - - @Override - @Cacheable("variableTypes") - public VariableTypeDocument find(String campaignId, String questionnaireId, Mode mode) { - return variableTypeMongoDBRepository.findFirstByCampaignIdQuestionnaireIdMode(campaignId, questionnaireId, mode); - } -} diff --git a/src/main/java/fr/insee/genesis/infrastructure/document/variabletype/VariableTypeDocument.java b/src/main/java/fr/insee/genesis/infrastructure/document/variabletype/VariableTypeDocument.java deleted file mode 100644 index 5fb3cc6a..00000000 --- a/src/main/java/fr/insee/genesis/infrastructure/document/variabletype/VariableTypeDocument.java +++ /dev/null @@ -1,16 +0,0 @@ -package fr.insee.genesis.infrastructure.document.variabletype; - -import fr.insee.bpm.metadata.model.VariablesMap; -import fr.insee.genesis.Constants; -import fr.insee.genesis.domain.model.surveyunit.Mode; -import lombok.Data; -import org.springframework.data.mongodb.core.mapping.Document; - -@Data -@Document(collection = Constants.MONGODB_VARIABLETYPE_COLLECTION_NAME) -public class VariableTypeDocument { - private String campaignId; - private String questionnaireId; - private Mode mode; - private VariablesMap variablesMap; -} diff --git a/src/main/java/fr/insee/genesis/infrastructure/mappers/VariableTypeDocumentMapper.java b/src/main/java/fr/insee/genesis/infrastructure/mappers/VariableTypeDocumentMapper.java deleted file mode 100644 index ed8cdecb..00000000 --- a/src/main/java/fr/insee/genesis/infrastructure/mappers/VariableTypeDocumentMapper.java +++ /dev/null @@ -1,21 +0,0 @@ -package fr.insee.genesis.infrastructure.mappers; - -import fr.insee.genesis.domain.model.variabletype.VariableTypeModel; -import fr.insee.genesis.infrastructure.document.variabletype.VariableTypeDocument; -import org.mapstruct.Mapper; -import org.mapstruct.factory.Mappers; - -import java.util.List; - -@Mapper(componentModel = "spring") -public interface VariableTypeDocumentMapper { - VariableTypeDocumentMapper INSTANCE = Mappers.getMapper(VariableTypeDocumentMapper.class); - - VariableTypeModel documentToModel(VariableTypeDocument variableTypeDocument); - - VariableTypeDocument modelToDocument(VariableTypeModel variableTypeModel); - - List listDocumentToListModel(List variableTypeDocuments); - - List listModelToListDocument(List variableTypeModels); -} diff --git a/src/main/java/fr/insee/genesis/infrastructure/repository/VariableTypeMongoDBRepository.java b/src/main/java/fr/insee/genesis/infrastructure/repository/VariableTypeMongoDBRepository.java deleted file mode 100644 index ecedd6d1..00000000 --- a/src/main/java/fr/insee/genesis/infrastructure/repository/VariableTypeMongoDBRepository.java +++ /dev/null @@ -1,14 +0,0 @@ -package fr.insee.genesis.infrastructure.repository; - -import fr.insee.genesis.domain.model.surveyunit.Mode; -import fr.insee.genesis.infrastructure.document.variabletype.VariableTypeDocument; -import org.springframework.data.mongodb.repository.MongoRepository; -import org.springframework.data.mongodb.repository.Query; -import org.springframework.stereotype.Repository; - -@Repository -public interface VariableTypeMongoDBRepository extends MongoRepository { - - @Query(value = "{ 'campaignId' : ?0 , 'questionnaireId' : ?1 , 'mode' : ?2 }") - VariableTypeDocument findFirstByCampaignIdQuestionnaireIdMode(String campaignId, String questionnaireId, Mode mode); -} diff --git a/src/test/java/cucumber/functional_tests/config/CucumberSpringConfiguration.java b/src/test/java/cucumber/functional_tests/config/CucumberSpringConfiguration.java index 81c8ef3d..45c92bc2 100644 --- a/src/test/java/cucumber/functional_tests/config/CucumberSpringConfiguration.java +++ b/src/test/java/cucumber/functional_tests/config/CucumberSpringConfiguration.java @@ -10,7 +10,6 @@ import fr.insee.genesis.infrastructure.repository.QuestionnaireMetadataMongoDBRepository; import fr.insee.genesis.infrastructure.repository.RundeckExecutionDBRepository; import fr.insee.genesis.infrastructure.repository.SurveyUnitMongoDBRepository; -import fr.insee.genesis.infrastructure.repository.VariableTypeMongoDBRepository; import io.cucumber.spring.CucumberContextConfiguration; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.boot.autoconfigure.data.mongo.MongoDataAutoConfiguration; @@ -34,8 +33,6 @@ public class CucumberSpringConfiguration { @MockitoBean private RundeckExecutionDBRepository rundeckExecutionDBRepository; @MockitoBean - private VariableTypeMongoDBRepository variableTypeMongoDBRepository; - @MockitoBean private LunaticModelMongoDBRepository lunaticModelMongoDBRepository; @MockitoBean private LastJsonExtractionMongoDBRepository lastJsonExtractionMongoDBRepository; diff --git a/src/test/java/fr/insee/genesis/controller/rest/ControllerAccessTest.java b/src/test/java/fr/insee/genesis/controller/rest/ControllerAccessTest.java index ef8b422f..d8647c9d 100644 --- a/src/test/java/fr/insee/genesis/controller/rest/ControllerAccessTest.java +++ b/src/test/java/fr/insee/genesis/controller/rest/ControllerAccessTest.java @@ -15,7 +15,6 @@ import fr.insee.genesis.infrastructure.repository.RawResponseRepository; import fr.insee.genesis.infrastructure.repository.RundeckExecutionDBRepository; import fr.insee.genesis.infrastructure.repository.SurveyUnitMongoDBRepository; -import fr.insee.genesis.infrastructure.repository.VariableTypeMongoDBRepository; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; @@ -94,8 +93,6 @@ class ControllerAccessTest { @MockitoBean private DataProcessingContextMongoDBRepository dataProcessingContextMongoDBRepository; @MockitoBean - private VariableTypeMongoDBRepository variableTypeMongoDBRepository; - @MockitoBean private LunaticModelMongoDBRepository lunaticModelMongoDBRepository; @MockitoBean private ContextualPreviousVariableMongoDBRepository contextualPreviousVariableMongoDBRepository; diff --git a/src/test/java/fr/insee/genesis/stubs/VariableTypePersistanceStub.java b/src/test/java/fr/insee/genesis/stubs/VariableTypePersistanceStub.java deleted file mode 100644 index e15220b4..00000000 --- a/src/test/java/fr/insee/genesis/stubs/VariableTypePersistanceStub.java +++ /dev/null @@ -1,30 +0,0 @@ -package fr.insee.genesis.stubs; - -import fr.insee.genesis.domain.model.surveyunit.Mode; -import fr.insee.genesis.domain.model.variabletype.VariableTypeModel; -import fr.insee.genesis.domain.ports.spi.VariableTypePersistancePort; -import fr.insee.genesis.infrastructure.document.variabletype.VariableTypeDocument; -import fr.insee.genesis.infrastructure.mappers.VariableTypeDocumentMapper; -import lombok.Getter; - -import java.util.ArrayList; -import java.util.List; - -@Getter -public class VariableTypePersistanceStub implements VariableTypePersistancePort { - List mongoStub = new ArrayList<>(); - - @Override - public void save(VariableTypeModel variableTypeModel) { - mongoStub.add(VariableTypeDocumentMapper.INSTANCE.modelToDocument(variableTypeModel)); - } - - @Override - public VariableTypeDocument find(String campaignId, String questionnaireId, Mode mode) { - return mongoStub.stream().filter(variableTypeDocument -> - variableTypeDocument.getCampaignId().equals(campaignId) - && variableTypeDocument.getQuestionnaireId().equals(questionnaireId) - && variableTypeDocument.getMode().equals(mode) - ).toList().getFirst(); - } -} From d4151c356562c6602a9984118253d95602715cac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Henninger?= Date: Mon, 24 Nov 2025 17:06:36 +0100 Subject: [PATCH 14/42] feat: WIP modelefiliere add questionnaireMetadata and correct typo "persistance" --- .../metadata/QuestionnaireMetadataModel.java | 4 +- .../api/QuestionnaireMetadataApiPort.java | 8 +-- ...QuestionnaireMetadataPersistencePort.java} | 6 +-- .../QuestionnaireMetadataService.java | 32 ++++++------ .../QuestionnaireMetadataMongoAdapter.java | 30 ++++++------ .../QuestionnaireMetadataDocument.java | 8 +++ .../QuestionnaireMetadataDocumentMapper.java | 13 +++++ ...uestionnaireMetadataMongoDBRepository.java | 6 +++ .../LunaticModelDefinitions.java | 6 +-- .../functional_tests/MainDefinitions.java | 16 +++--- .../functional_tests/RawDataDefinitions.java | 8 +-- .../rest/HealthCheckControllerTest.java | 4 +- .../QuestionnaireMetadataControllerTest.java | 49 ++++++++++--------- .../controller/rest/UtilsControllerTest.java | 4 +- .../responses/CampaignControllerTest.java | 4 +- .../InterrogationControllerTest.java | 4 +- .../rest/responses/ModeControllerTest.java | 4 +- .../QuestionnaireControllerTest.java | 4 +- .../responses/RawResponseControllerTest.java | 10 ++-- .../responses/ResponseControllerTest.java | 10 ++-- .../LunaticJsonRawDataServiceTest.java | 4 +- .../surveyunit/SurveyUnitServiceTest.java | 4 +- ...tionnaireMetadataPersistencePortStub.java} | 17 ++++--- 23 files changed, 146 insertions(+), 109 deletions(-) rename src/main/java/fr/insee/genesis/domain/ports/spi/{QuestionnaireMetadataPersistancePort.java => QuestionnaireMetadataPersistencePort.java} (57%) rename src/test/java/fr/insee/genesis/stubs/{QuestionnaireMetadataPersistancePortStub.java => QuestionnaireMetadataPersistencePortStub.java} (50%) diff --git a/src/main/java/fr/insee/genesis/domain/model/metadata/QuestionnaireMetadataModel.java b/src/main/java/fr/insee/genesis/domain/model/metadata/QuestionnaireMetadataModel.java index b924b2b2..d5291b21 100644 --- a/src/main/java/fr/insee/genesis/domain/model/metadata/QuestionnaireMetadataModel.java +++ b/src/main/java/fr/insee/genesis/domain/model/metadata/QuestionnaireMetadataModel.java @@ -2,9 +2,11 @@ import fr.insee.bpm.metadata.model.MetadataModel; import fr.insee.genesis.domain.model.surveyunit.Mode; +import lombok.Builder; +@Builder public record QuestionnaireMetadataModel ( - String questionnaireId, + String collectionInstrumentId, Mode mode, MetadataModel metadataModel ){} diff --git a/src/main/java/fr/insee/genesis/domain/ports/api/QuestionnaireMetadataApiPort.java b/src/main/java/fr/insee/genesis/domain/ports/api/QuestionnaireMetadataApiPort.java index 5ccca0e8..33593726 100644 --- a/src/main/java/fr/insee/genesis/domain/ports/api/QuestionnaireMetadataApiPort.java +++ b/src/main/java/fr/insee/genesis/domain/ports/api/QuestionnaireMetadataApiPort.java @@ -9,10 +9,10 @@ import java.util.List; public interface QuestionnaireMetadataApiPort { - MetadataModel find(String questionnaireId, Mode mode) throws GenesisException; - MetadataModel loadAndSaveIfNotExists(String campaignName, String questionnaireId, Mode mode, FileUtils fileUtils, + MetadataModel find(String collectionInstrumentId, Mode mode) throws GenesisException; + MetadataModel loadAndSaveIfNotExists(String campaignName, String collectionInstrumentId, Mode mode, FileUtils fileUtils, List errors) throws GenesisException; - void remove(String questionnaireId, Mode mode); + void remove(String collectionInstrumentId, Mode mode); - void save(String questionnaireId, Mode mode, MetadataModel metadataModel); + void save(String collectionInstrumentId, Mode mode, MetadataModel metadataModel); } diff --git a/src/main/java/fr/insee/genesis/domain/ports/spi/QuestionnaireMetadataPersistancePort.java b/src/main/java/fr/insee/genesis/domain/ports/spi/QuestionnaireMetadataPersistencePort.java similarity index 57% rename from src/main/java/fr/insee/genesis/domain/ports/spi/QuestionnaireMetadataPersistancePort.java rename to src/main/java/fr/insee/genesis/domain/ports/spi/QuestionnaireMetadataPersistencePort.java index b33977d2..39588695 100644 --- a/src/main/java/fr/insee/genesis/domain/ports/spi/QuestionnaireMetadataPersistancePort.java +++ b/src/main/java/fr/insee/genesis/domain/ports/spi/QuestionnaireMetadataPersistencePort.java @@ -5,9 +5,9 @@ import java.util.List; -public interface QuestionnaireMetadataPersistancePort { - List find(String questionnaireId, Mode mode); +public interface QuestionnaireMetadataPersistencePort { + List find(String collectionInstrumentId, Mode mode); void save(QuestionnaireMetadataModel questionnaireMetadataModel); - void remove(String questionnaireId, Mode mode); + void remove(String collectionInstrumentId, Mode mode); } diff --git a/src/main/java/fr/insee/genesis/domain/service/metadata/QuestionnaireMetadataService.java b/src/main/java/fr/insee/genesis/domain/service/metadata/QuestionnaireMetadataService.java index cc172e0a..6f1f50d5 100644 --- a/src/main/java/fr/insee/genesis/domain/service/metadata/QuestionnaireMetadataService.java +++ b/src/main/java/fr/insee/genesis/domain/service/metadata/QuestionnaireMetadataService.java @@ -10,7 +10,7 @@ import fr.insee.genesis.domain.model.metadata.QuestionnaireMetadataModel; import fr.insee.genesis.domain.model.surveyunit.Mode; import fr.insee.genesis.domain.ports.api.QuestionnaireMetadataApiPort; -import fr.insee.genesis.domain.ports.spi.QuestionnaireMetadataPersistancePort; +import fr.insee.genesis.domain.ports.spi.QuestionnaireMetadataPersistencePort; import fr.insee.genesis.exceptions.GenesisError; import fr.insee.genesis.exceptions.GenesisException; import fr.insee.genesis.infrastructure.utils.FileUtils; @@ -30,36 +30,36 @@ public class QuestionnaireMetadataService implements QuestionnaireMetadataApiPor private static final String DDI_FILE_PATTERN = "ddi[\\w,\\s-]+\\.xml"; private static final String LUNATIC_FILE_PATTERN = "lunatic[\\w,\\s-]+\\.json"; - QuestionnaireMetadataPersistancePort questionnaireMetadataPersistancePort; + QuestionnaireMetadataPersistencePort questionnaireMetadataPersistencePort; @Override - public MetadataModel find(String questionnaireId, Mode mode) throws GenesisException { + public MetadataModel find(String collectionInstrumentId, Mode mode) throws GenesisException { List questionnaireMetadataModels = - questionnaireMetadataPersistancePort.find(questionnaireId, mode); + questionnaireMetadataPersistencePort.find(collectionInstrumentId, mode); if(questionnaireMetadataModels.isEmpty()){ - throw new GenesisException(404, "Questionnaire metadata not found"); + throw new GenesisException(404, "Collection instrument metadata not found"); } return questionnaireMetadataModels.getFirst().metadataModel(); } @Override - public MetadataModel loadAndSaveIfNotExists(String campaignName, String questionnaireId, Mode mode, FileUtils fileUtils, + public MetadataModel loadAndSaveIfNotExists(String campaignName, String collectionInstrumentId, Mode mode, FileUtils fileUtils, List errors) throws GenesisException { List questionnaireMetadataModels = - questionnaireMetadataPersistancePort.find(questionnaireId.toUpperCase(), mode); + questionnaireMetadataPersistencePort.find(collectionInstrumentId.toUpperCase(), mode); if(questionnaireMetadataModels.isEmpty() || questionnaireMetadataModels.getFirst().metadataModel() == null){ MetadataModel metadataModel = readMetadatas(campaignName, mode.getModeName(), fileUtils, errors); - saveMetadata(questionnaireId.toUpperCase(), mode, metadataModel); + saveMetadata(collectionInstrumentId.toUpperCase(), mode, metadataModel); return metadataModel; } return questionnaireMetadataModels.getFirst().metadataModel(); } - private void saveMetadata(String questionnaireId, Mode mode, MetadataModel metadataModel) { - questionnaireMetadataPersistancePort.save( + private void saveMetadata(String collectionInstrumentId, Mode mode, MetadataModel metadataModel) { + questionnaireMetadataPersistencePort.save( new QuestionnaireMetadataModel( - questionnaireId, + collectionInstrumentId, mode, metadataModel ) @@ -140,14 +140,14 @@ private MetadataModel parseMetadata(String metadataFilePath, boolean withDDI) { } @Override - public void remove(String questionnaireId, Mode mode) { - questionnaireMetadataPersistancePort.remove(questionnaireId, mode); + public void remove(String collectionInstrumentId, Mode mode) { + questionnaireMetadataPersistencePort.remove(collectionInstrumentId, mode); } @Override - public void save(String questionnaireId, Mode mode, MetadataModel metadataModel) { - questionnaireMetadataPersistancePort.save(new QuestionnaireMetadataModel( - questionnaireId, + public void save(String collectionInstrumentId, Mode mode, MetadataModel metadataModel) { + questionnaireMetadataPersistencePort.save(new QuestionnaireMetadataModel( + collectionInstrumentId, mode, metadataModel )); diff --git a/src/main/java/fr/insee/genesis/infrastructure/adapter/QuestionnaireMetadataMongoAdapter.java b/src/main/java/fr/insee/genesis/infrastructure/adapter/QuestionnaireMetadataMongoAdapter.java index 3317c481..4eed576c 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/adapter/QuestionnaireMetadataMongoAdapter.java +++ b/src/main/java/fr/insee/genesis/infrastructure/adapter/QuestionnaireMetadataMongoAdapter.java @@ -2,7 +2,8 @@ import fr.insee.genesis.domain.model.metadata.QuestionnaireMetadataModel; import fr.insee.genesis.domain.model.surveyunit.Mode; -import fr.insee.genesis.domain.ports.spi.QuestionnaireMetadataPersistancePort; +import fr.insee.genesis.domain.ports.spi.QuestionnaireMetadataPersistencePort; +import fr.insee.genesis.infrastructure.document.metadata.QuestionnaireMetadataDocument; import fr.insee.genesis.infrastructure.mappers.QuestionnaireMetadataDocumentMapper; import fr.insee.genesis.infrastructure.repository.QuestionnaireMetadataMongoDBRepository; import lombok.AllArgsConstructor; @@ -11,36 +12,37 @@ import org.springframework.cache.annotation.Cacheable; import org.springframework.stereotype.Service; +import java.util.ArrayList; import java.util.List; @Service @AllArgsConstructor @Qualifier("questionnaireMetadataMongoAdapter") -public class QuestionnaireMetadataMongoAdapter implements QuestionnaireMetadataPersistancePort { +public class QuestionnaireMetadataMongoAdapter implements QuestionnaireMetadataPersistencePort { private final QuestionnaireMetadataMongoDBRepository questionnaireMetadataMongoDBRepository; @Override - @Cacheable(value = "metadatas", key = "#questionnaireId + '-' + #mode") - public List find(String questionnaireId, Mode mode) { - return QuestionnaireMetadataDocumentMapper.INSTANCE.listDocumentToListModel( - questionnaireMetadataMongoDBRepository.findByQuestionnaireIdAndMode( - questionnaireId, mode - ) - ); + @Cacheable(value = "metadatas", key = "#collectionInstrumentId + '-' + #mode") + public List find(String collectionInstrumentId, Mode mode) { + List results = new ArrayList<>(); + results.addAll(questionnaireMetadataMongoDBRepository.findByQuestionnaireIdAndMode(collectionInstrumentId, mode)); + results.addAll(questionnaireMetadataMongoDBRepository.findByCollectionInstrumentIdAndMode(collectionInstrumentId, mode)); + return QuestionnaireMetadataDocumentMapper.INSTANCE.listDocumentToListModel(results); } @Override - @CacheEvict(value = "metadatas", key = "#questionnaireMetadataModel.questionnaireId() + '-' + #questionnaireMetadataModel.mode()") + @CacheEvict(value = "metadatas", key = "#questionnaireMetadataModel.collectionInstrumentId() + '-' + #questionnaireMetadataModel.mode()") public void save(QuestionnaireMetadataModel questionnaireMetadataModel) { - remove(questionnaireMetadataModel.questionnaireId(), questionnaireMetadataModel.mode()); + remove(questionnaireMetadataModel.collectionInstrumentId(), questionnaireMetadataModel.mode()); questionnaireMetadataMongoDBRepository.save( QuestionnaireMetadataDocumentMapper.INSTANCE.modelToDocument(questionnaireMetadataModel) ); } @Override - @CacheEvict(value = "metadatas", key = "#questionnaireId + '-' + #mode") - public void remove(String questionnaireId, Mode mode) { - questionnaireMetadataMongoDBRepository.deleteByQuestionnaireIdAndMode(questionnaireId, mode); + @CacheEvict(value = "metadatas", key = "#collectionInstrumentId + '-' + #mode") + public void remove(String collectionInstrumentId, Mode mode) { + questionnaireMetadataMongoDBRepository.deleteByQuestionnaireIdAndMode(collectionInstrumentId, mode); + questionnaireMetadataMongoDBRepository.deleteByCollectionInstrumentIdIdAndMode(collectionInstrumentId, mode); } } diff --git a/src/main/java/fr/insee/genesis/infrastructure/document/metadata/QuestionnaireMetadataDocument.java b/src/main/java/fr/insee/genesis/infrastructure/document/metadata/QuestionnaireMetadataDocument.java index 7c9d0528..ec22b51b 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/document/metadata/QuestionnaireMetadataDocument.java +++ b/src/main/java/fr/insee/genesis/infrastructure/document/metadata/QuestionnaireMetadataDocument.java @@ -5,10 +5,18 @@ import org.springframework.data.mongodb.core.index.CompoundIndex; import org.springframework.data.mongodb.core.mapping.Document; +/* It is not possible to respect these rules in a record (compilation error). +I choose to keep the record here (this choice can be challenged)*/ +@SuppressWarnings({ + "java:S1123", // deprecated annotation usage + "java:S6355" // Add 'since' and/or 'forRemoval' arguments to the @Deprecated annotation +}) @CompoundIndex(name = "questionnaireId_1_mode_1", def = "{'questionnaireId': 1, 'mode': 1}") @Document(collection = "questionnaireMetadatas") public record QuestionnaireMetadataDocument( + @Deprecated String questionnaireId, + String collectionInstrumentId, Mode mode, MetadataModel metadataModel ){} diff --git a/src/main/java/fr/insee/genesis/infrastructure/mappers/QuestionnaireMetadataDocumentMapper.java b/src/main/java/fr/insee/genesis/infrastructure/mappers/QuestionnaireMetadataDocumentMapper.java index 8b951f3e..94ed2060 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/mappers/QuestionnaireMetadataDocumentMapper.java +++ b/src/main/java/fr/insee/genesis/infrastructure/mappers/QuestionnaireMetadataDocumentMapper.java @@ -2,7 +2,9 @@ import fr.insee.genesis.domain.model.metadata.QuestionnaireMetadataModel; import fr.insee.genesis.infrastructure.document.metadata.QuestionnaireMetadataDocument; +import org.mapstruct.AfterMapping; import org.mapstruct.Mapper; +import org.mapstruct.MappingTarget; import org.mapstruct.factory.Mappers; import java.util.List; @@ -18,4 +20,15 @@ public interface QuestionnaireMetadataDocumentMapper { List listDocumentToListModel(List documentList); List listModelToListDocument(List modelList); + + @SuppressWarnings("removal") + @AfterMapping + default void fillModelAfterRead( + QuestionnaireMetadataDocument doc, + @MappingTarget QuestionnaireMetadataModel.QuestionnaireMetadataModelBuilder builder + ) { + if (doc.collectionInstrumentId() == null && doc.questionnaireId()!=null) { + builder.collectionInstrumentId(doc.questionnaireId()); + } + } } diff --git a/src/main/java/fr/insee/genesis/infrastructure/repository/QuestionnaireMetadataMongoDBRepository.java b/src/main/java/fr/insee/genesis/infrastructure/repository/QuestionnaireMetadataMongoDBRepository.java index 2d3484d1..3f09e88f 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/repository/QuestionnaireMetadataMongoDBRepository.java +++ b/src/main/java/fr/insee/genesis/infrastructure/repository/QuestionnaireMetadataMongoDBRepository.java @@ -11,6 +11,12 @@ public interface QuestionnaireMetadataMongoDBRepository extends CrudRepository findByQuestionnaireIdAndMode(String questionnaireId, Mode mode); + @Query("{'collectionInstrumentId' : ?0, 'mode' : ?1}") + List findByCollectionInstrumentIdAndMode(String collectionInstrumentId, Mode mode); + @Query(value = "{'questionnaireId' : ?0, 'mode' : ?1}", delete = true) void deleteByQuestionnaireIdAndMode(String questionnaireId, Mode mode); + + @Query(value = "{'collectionInstrumentId' : ?0, 'mode' : ?1}", delete = true) + void deleteByCollectionInstrumentIdIdAndMode(String collectionInstrumentId, Mode mode); } diff --git a/src/test/java/cucumber/functional_tests/LunaticModelDefinitions.java b/src/test/java/cucumber/functional_tests/LunaticModelDefinitions.java index b785d234..dca3aac2 100644 --- a/src/test/java/cucumber/functional_tests/LunaticModelDefinitions.java +++ b/src/test/java/cucumber/functional_tests/LunaticModelDefinitions.java @@ -17,7 +17,7 @@ import fr.insee.genesis.stubs.ConfigStub; import fr.insee.genesis.stubs.DataProcessingContextPersistancePortStub; import fr.insee.genesis.stubs.LunaticModelPersistanceStub; -import fr.insee.genesis.stubs.QuestionnaireMetadataPersistancePortStub; +import fr.insee.genesis.stubs.QuestionnaireMetadataPersistencePortStub; import fr.insee.genesis.stubs.SurveyUnitPersistencePortStub; import io.cucumber.java.Before; import io.cucumber.java.en.Given; @@ -49,11 +49,11 @@ public class LunaticModelDefinitions { LunaticModelController lunaticModelController = new LunaticModelController(new LunaticModelService(lunaticModelPersistanceStub)); SurveyUnitPersistencePortStub surveyUnitPersistencePortStub = new SurveyUnitPersistencePortStub(); - static QuestionnaireMetadataPersistancePortStub questionnaireMetadataPersistancePortStub = new QuestionnaireMetadataPersistancePortStub(); + static QuestionnaireMetadataPersistencePortStub questionnaireMetadataPersistencePortStub = new QuestionnaireMetadataPersistencePortStub(); QuestionnaireController questionnaireController = new QuestionnaireController( new SurveyUnitService( surveyUnitPersistencePortStub, - new QuestionnaireMetadataService(questionnaireMetadataPersistancePortStub), + new QuestionnaireMetadataService(questionnaireMetadataPersistencePortStub), new FileUtils(new ConfigStub()) ), new DataProcessingContextService( diff --git a/src/test/java/cucumber/functional_tests/MainDefinitions.java b/src/test/java/cucumber/functional_tests/MainDefinitions.java index b717ad1d..2a2d524e 100644 --- a/src/test/java/cucumber/functional_tests/MainDefinitions.java +++ b/src/test/java/cucumber/functional_tests/MainDefinitions.java @@ -31,7 +31,7 @@ import fr.insee.genesis.infrastructure.utils.FileUtils; import fr.insee.genesis.stubs.ConfigStub; import fr.insee.genesis.stubs.DataProcessingContextPersistancePortStub; -import fr.insee.genesis.stubs.QuestionnaireMetadataPersistancePortStub; +import fr.insee.genesis.stubs.QuestionnaireMetadataPersistencePortStub; import fr.insee.genesis.stubs.SurveyUnitPersistencePortStub; import io.cucumber.java.After; import io.cucumber.java.Before; @@ -65,10 +65,10 @@ public class MainDefinitions { SurveyUnitQualityService surveyUnitQualityService = new SurveyUnitQualityService(); SurveyUnitPersistencePortStub surveyUnitPersistence = new SurveyUnitPersistencePortStub(); - static QuestionnaireMetadataPersistancePortStub questionnaireMetadataPersistancePortStub = - new QuestionnaireMetadataPersistancePortStub(); + static QuestionnaireMetadataPersistencePortStub questionnaireMetadataPersistencePortStub = + new QuestionnaireMetadataPersistencePortStub(); static QuestionnaireMetadataService questionnaireMetadataService = - new QuestionnaireMetadataService(questionnaireMetadataPersistancePortStub); + new QuestionnaireMetadataService(questionnaireMetadataPersistencePortStub); DataProcessingContextPersistancePortStub dataProcessingContextPersistancePortStub = new DataProcessingContextPersistancePortStub(); DataProcessingContextApiPort dataProcessingContextApiPort = new DataProcessingContextService( @@ -81,7 +81,7 @@ public class MainDefinitions { ResponseEntity surveyUnitLatestStatesResponse; ResponseController responseController = new ResponseController( - new SurveyUnitService(surveyUnitPersistence, new QuestionnaireMetadataService(questionnaireMetadataPersistancePortStub), new FileUtils(config)), + new SurveyUnitService(surveyUnitPersistence, new QuestionnaireMetadataService(questionnaireMetadataPersistencePortStub), new FileUtils(config)), surveyUnitQualityService, new FileUtils(config), new ControllerUtils(new FileUtils(config)), @@ -163,18 +163,18 @@ public void get_models(String fileName, String ddiName) throws IOException, Pars LunaticXmlCampaign campaign; campaign = parser.parseDataFile(filePath); List questionnaireMetadataModels = - questionnaireMetadataPersistancePortStub.find(directory, Mode.WEB); + questionnaireMetadataPersistencePortStub.find(directory, Mode.WEB); if(questionnaireMetadataModels.isEmpty()){ MetadataModel metadataModel = DDIReader.getMetadataFromDDI( ddiFilePath.toFile().toURI().toURL().toString(), new FileInputStream(ddiFilePath.toFile()) ); - questionnaireMetadataPersistancePortStub.save(new QuestionnaireMetadataModel( + questionnaireMetadataPersistencePortStub.save(new QuestionnaireMetadataModel( directory, Mode.WEB, metadataModel )); - questionnaireMetadataModels = questionnaireMetadataPersistancePortStub.find(directory, Mode.WEB); + questionnaireMetadataModels = questionnaireMetadataPersistencePortStub.find(directory, Mode.WEB); } List surveyUnitModels1 = new ArrayList<>(); for (LunaticXmlSurveyUnit su : campaign.getSurveyUnits()) { diff --git a/src/test/java/cucumber/functional_tests/RawDataDefinitions.java b/src/test/java/cucumber/functional_tests/RawDataDefinitions.java index e4dbbfa9..6257ff92 100644 --- a/src/test/java/cucumber/functional_tests/RawDataDefinitions.java +++ b/src/test/java/cucumber/functional_tests/RawDataDefinitions.java @@ -26,7 +26,7 @@ import fr.insee.genesis.stubs.ConfigStub; import fr.insee.genesis.stubs.DataProcessingContextPersistancePortStub; import fr.insee.genesis.stubs.LunaticJsonRawDataPersistanceStub; -import fr.insee.genesis.stubs.QuestionnaireMetadataPersistancePortStub; +import fr.insee.genesis.stubs.QuestionnaireMetadataPersistencePortStub; import fr.insee.genesis.stubs.SurveyUnitPersistencePortStub; import fr.insee.genesis.stubs.SurveyUnitQualityToolPerretAdapterStub; import fr.insee.modelefiliere.RawResponseDto; @@ -70,12 +70,12 @@ public class RawDataDefinitions { private TestRestTemplate rest; LunaticJsonRawDataPersistanceStub lunaticJsonRawDataPersistanceStub = new LunaticJsonRawDataPersistanceStub(); - static QuestionnaireMetadataPersistancePortStub questionnaireMetadataPersistancePortStub = new QuestionnaireMetadataPersistancePortStub(); + static QuestionnaireMetadataPersistencePortStub questionnaireMetadataPersistencePortStub = new QuestionnaireMetadataPersistencePortStub(); SurveyUnitPersistencePortStub surveyUnitPersistencePortStub = new SurveyUnitPersistencePortStub(); DataProcessingContextPersistancePortStub contextStub = new DataProcessingContextPersistancePortStub(); Config config = new ConfigStub(); FileUtils fileUtils = new FileUtils(config); - SurveyUnitService surveyUnitService = new SurveyUnitService(surveyUnitPersistencePortStub, new QuestionnaireMetadataService(questionnaireMetadataPersistancePortStub), fileUtils); + SurveyUnitService surveyUnitService = new SurveyUnitService(surveyUnitPersistencePortStub, new QuestionnaireMetadataService(questionnaireMetadataPersistencePortStub), fileUtils); ControllerUtils controllerUtils = new ControllerUtils(fileUtils); SurveyUnitQualityService surveyUnitQualityService = new SurveyUnitQualityService(); DataProcessingContextPersistancePortStub dataProcessingContextPersistancePortStub = @@ -85,7 +85,7 @@ public class RawDataDefinitions { new LunaticJsonRawDataService( lunaticJsonRawDataPersistanceStub, controllerUtils, - new QuestionnaireMetadataService(questionnaireMetadataPersistancePortStub), + new QuestionnaireMetadataService(questionnaireMetadataPersistencePortStub), surveyUnitService, surveyUnitQualityService, fileUtils, diff --git a/src/test/java/fr/insee/genesis/controller/rest/HealthCheckControllerTest.java b/src/test/java/fr/insee/genesis/controller/rest/HealthCheckControllerTest.java index 31d1aa4d..b5ebd30a 100644 --- a/src/test/java/fr/insee/genesis/controller/rest/HealthCheckControllerTest.java +++ b/src/test/java/fr/insee/genesis/controller/rest/HealthCheckControllerTest.java @@ -11,7 +11,7 @@ import fr.insee.genesis.infrastructure.utils.FileUtils; import fr.insee.genesis.stubs.ConfigStub; import fr.insee.genesis.stubs.DataProcessingContextPersistancePortStub; -import fr.insee.genesis.stubs.QuestionnaireMetadataPersistancePortStub; +import fr.insee.genesis.stubs.QuestionnaireMetadataPersistencePortStub; import fr.insee.genesis.stubs.SurveyUnitPersistencePortStub; import lombok.extern.slf4j.Slf4j; import org.assertj.core.api.Assertions; @@ -34,7 +34,7 @@ static void init() { surveyUnitPersistencePortStub = new SurveyUnitPersistencePortStub(); SurveyUnitApiPort surveyUnitApiPort = new SurveyUnitService( surveyUnitPersistencePortStub, - new QuestionnaireMetadataService(new QuestionnaireMetadataPersistancePortStub()), + new QuestionnaireMetadataService(new QuestionnaireMetadataPersistencePortStub()), new FileUtils(new ConfigStub()) ); List externalVariableList = new ArrayList<>(); diff --git a/src/test/java/fr/insee/genesis/controller/rest/QuestionnaireMetadataControllerTest.java b/src/test/java/fr/insee/genesis/controller/rest/QuestionnaireMetadataControllerTest.java index a671feff..0186515a 100644 --- a/src/test/java/fr/insee/genesis/controller/rest/QuestionnaireMetadataControllerTest.java +++ b/src/test/java/fr/insee/genesis/controller/rest/QuestionnaireMetadataControllerTest.java @@ -6,7 +6,7 @@ import fr.insee.genesis.domain.model.surveyunit.Mode; import fr.insee.genesis.domain.service.metadata.QuestionnaireMetadataService; import fr.insee.genesis.infrastructure.document.metadata.QuestionnaireMetadataDocument; -import fr.insee.genesis.stubs.QuestionnaireMetadataPersistancePortStub; +import fr.insee.genesis.stubs.QuestionnaireMetadataPersistencePortStub; import org.assertj.core.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -14,17 +14,17 @@ class QuestionnaireMetadataControllerTest { public static final String VARIABLE_NAME = "TESTVAR"; - static QuestionnaireMetadataPersistancePortStub questionnaireMetadataPersistancePortStub - = new QuestionnaireMetadataPersistancePortStub(); + static QuestionnaireMetadataPersistencePortStub questionnaireMetadataPersistencePortStub + = new QuestionnaireMetadataPersistencePortStub(); static QuestionnaireMetadataController questionnaireMetadataController = new QuestionnaireMetadataController( - new QuestionnaireMetadataService(questionnaireMetadataPersistancePortStub) + new QuestionnaireMetadataService(questionnaireMetadataPersistencePortStub) ); @BeforeEach void clean(){ - questionnaireMetadataPersistancePortStub.getMongoStub().clear(); + questionnaireMetadataPersistencePortStub.getMongoStub().clear(); } @Test @@ -33,14 +33,15 @@ void getMetadataTest(){ String questionnaireId = "TESTQUEST"; Mode mode = Mode.WEB; String variableName = VARIABLE_NAME; - questionnaireMetadataPersistancePortStub.getMongoStub().add( + questionnaireMetadataPersistencePortStub.getMongoStub().add( new QuestionnaireMetadataDocument( questionnaireId, + null, mode, getMetadataModel(variableName) ) ); - Assertions.assertThat(questionnaireMetadataPersistancePortStub.getMongoStub()).hasSize(1); + Assertions.assertThat(questionnaireMetadataPersistencePortStub.getMongoStub()).hasSize(1); //WHEN ResponseEntity response = questionnaireMetadataController.getMetadata(questionnaireId, mode); @@ -82,16 +83,16 @@ void saveMetadataTest(){ //THEN Assertions.assertThat(response.getStatusCode().is2xxSuccessful()).isTrue(); - Assertions.assertThat(questionnaireMetadataPersistancePortStub.getMongoStub()).hasSize(1); - Assertions.assertThat(questionnaireMetadataPersistancePortStub.getMongoStub().getFirst().questionnaireId()) + Assertions.assertThat(questionnaireMetadataPersistencePortStub.getMongoStub()).hasSize(1); + Assertions.assertThat(questionnaireMetadataPersistencePortStub.getMongoStub().getFirst().collectionInstrumentId()) .isEqualTo(questionnaireId); - Assertions.assertThat(questionnaireMetadataPersistancePortStub.getMongoStub().getFirst().mode()) + Assertions.assertThat(questionnaireMetadataPersistencePortStub.getMongoStub().getFirst().mode()) .isEqualTo(mode); - Assertions.assertThat(questionnaireMetadataPersistancePortStub.getMongoStub().getFirst().metadataModel().getVariables() + Assertions.assertThat(questionnaireMetadataPersistencePortStub.getMongoStub().getFirst().metadataModel().getVariables() .hasVariable(variableName)).isTrue(); - Assertions.assertThat(questionnaireMetadataPersistancePortStub.getMongoStub().getFirst().metadataModel().getVariables() + Assertions.assertThat(questionnaireMetadataPersistencePortStub.getMongoStub().getFirst().metadataModel().getVariables() .getVariable(variableName).getGroup()).isEqualTo(metadataModel.getRootGroup()); - Assertions.assertThat(questionnaireMetadataPersistancePortStub.getMongoStub().getFirst().metadataModel().getVariables() + Assertions.assertThat(questionnaireMetadataPersistencePortStub.getMongoStub().getFirst().metadataModel().getVariables() .getVariable(variableName).getType()).isEqualTo(VariableType.STRING); } @@ -102,9 +103,10 @@ void saveMetadataTest_overwrite(){ Mode mode = Mode.WEB; String variableName = VARIABLE_NAME; MetadataModel metadataModel = getMetadataModel(variableName); - questionnaireMetadataPersistancePortStub.getMongoStub().add( + questionnaireMetadataPersistencePortStub.getMongoStub().add( new QuestionnaireMetadataDocument( questionnaireId, + null, mode, metadataModel ) @@ -124,16 +126,16 @@ void saveMetadataTest_overwrite(){ //THEN Assertions.assertThat(response.getStatusCode().is2xxSuccessful()).isTrue(); - Assertions.assertThat(questionnaireMetadataPersistancePortStub.getMongoStub()).hasSize(1); - Assertions.assertThat(questionnaireMetadataPersistancePortStub.getMongoStub().getFirst().questionnaireId()) + Assertions.assertThat(questionnaireMetadataPersistencePortStub.getMongoStub()).hasSize(1); + Assertions.assertThat(questionnaireMetadataPersistencePortStub.getMongoStub().getFirst().collectionInstrumentId()) .isEqualTo(questionnaireId); - Assertions.assertThat(questionnaireMetadataPersistancePortStub.getMongoStub().getFirst().mode()) + Assertions.assertThat(questionnaireMetadataPersistencePortStub.getMongoStub().getFirst().mode()) .isEqualTo(mode); - Assertions.assertThat(questionnaireMetadataPersistancePortStub.getMongoStub().getFirst().metadataModel().getVariables() + Assertions.assertThat(questionnaireMetadataPersistencePortStub.getMongoStub().getFirst().metadataModel().getVariables() .hasVariable(variableName)).isTrue(); - Assertions.assertThat(questionnaireMetadataPersistancePortStub.getMongoStub().getFirst().metadataModel().getVariables() + Assertions.assertThat(questionnaireMetadataPersistencePortStub.getMongoStub().getFirst().metadataModel().getVariables() .getVariable(variableName).getGroup()).isEqualTo(metadataModel.getRootGroup()); - Assertions.assertThat(questionnaireMetadataPersistancePortStub.getMongoStub().getFirst().metadataModel().getVariables() + Assertions.assertThat(questionnaireMetadataPersistencePortStub.getMongoStub().getFirst().metadataModel().getVariables() .getVariable(variableName).getType()).isEqualTo(VariableType.INTEGER); } @@ -142,20 +144,21 @@ void deleteMetadataTest(){ //GIVEN String questionnaireId = "TESTQUEST"; Mode mode = Mode.WEB; - questionnaireMetadataPersistancePortStub.getMongoStub().add( + questionnaireMetadataPersistencePortStub.getMongoStub().add( new QuestionnaireMetadataDocument( questionnaireId, + null, mode, new MetadataModel() ) ); - Assertions.assertThat(questionnaireMetadataPersistancePortStub.getMongoStub()).hasSize(1); + Assertions.assertThat(questionnaireMetadataPersistencePortStub.getMongoStub()).hasSize(1); //WHEN questionnaireMetadataController.deleteMetadata(questionnaireId, mode); //THEN - Assertions.assertThat(questionnaireMetadataPersistancePortStub.getMongoStub()).isEmpty(); + Assertions.assertThat(questionnaireMetadataPersistencePortStub.getMongoStub()).isEmpty(); } private static MetadataModel getMetadataModel(String variableName) { diff --git a/src/test/java/fr/insee/genesis/controller/rest/UtilsControllerTest.java b/src/test/java/fr/insee/genesis/controller/rest/UtilsControllerTest.java index 5f389189..da751d0f 100644 --- a/src/test/java/fr/insee/genesis/controller/rest/UtilsControllerTest.java +++ b/src/test/java/fr/insee/genesis/controller/rest/UtilsControllerTest.java @@ -22,7 +22,7 @@ import fr.insee.genesis.stubs.ConfigStub; import fr.insee.genesis.stubs.DataProcessingContextPersistancePortStub; import fr.insee.genesis.stubs.LunaticJsonRawDataPersistanceStub; -import fr.insee.genesis.stubs.QuestionnaireMetadataPersistancePortStub; +import fr.insee.genesis.stubs.QuestionnaireMetadataPersistencePortStub; import fr.insee.genesis.stubs.SurveyUnitPersistencePortStub; import fr.insee.genesis.stubs.SurveyUnitQualityToolPerretAdapterStub; import org.assertj.core.api.Assertions; @@ -54,7 +54,7 @@ class UtilsControllerTest { static List interrogationIdList; static FileUtils fileUtils = new FileUtils(new ConfigStub()); static ControllerUtils controllerUtils = new ControllerUtils(fileUtils); - static QuestionnaireMetadataService metadataService = new QuestionnaireMetadataService(new QuestionnaireMetadataPersistancePortStub()); + static QuestionnaireMetadataService metadataService = new QuestionnaireMetadataService(new QuestionnaireMetadataPersistencePortStub()); static SurveyUnitService surveyUnitService = new SurveyUnitService(new SurveyUnitPersistencePortStub(), metadataService, fileUtils); static SurveyUnitQualityService surveyUnitQualityService = new SurveyUnitQualityService(); //Constants diff --git a/src/test/java/fr/insee/genesis/controller/rest/responses/CampaignControllerTest.java b/src/test/java/fr/insee/genesis/controller/rest/responses/CampaignControllerTest.java index 981553ee..45b07a62 100644 --- a/src/test/java/fr/insee/genesis/controller/rest/responses/CampaignControllerTest.java +++ b/src/test/java/fr/insee/genesis/controller/rest/responses/CampaignControllerTest.java @@ -6,7 +6,7 @@ import fr.insee.genesis.domain.service.surveyunit.SurveyUnitService; import fr.insee.genesis.infrastructure.utils.FileUtils; import fr.insee.genesis.stubs.ConfigStub; -import fr.insee.genesis.stubs.QuestionnaireMetadataPersistancePortStub; +import fr.insee.genesis.stubs.QuestionnaireMetadataPersistencePortStub; import fr.insee.genesis.stubs.SurveyUnitPersistencePortStub; import org.assertj.core.api.Assertions; import org.junit.jupiter.api.BeforeAll; @@ -32,7 +32,7 @@ static void init() { surveyUnitPersistencePortStub = new SurveyUnitPersistencePortStub(); SurveyUnitApiPort surveyUnitApiPort = new SurveyUnitService( surveyUnitPersistencePortStub, - new QuestionnaireMetadataService(new QuestionnaireMetadataPersistancePortStub()), + new QuestionnaireMetadataService(new QuestionnaireMetadataPersistencePortStub()), new FileUtils(new ConfigStub()) ); diff --git a/src/test/java/fr/insee/genesis/controller/rest/responses/InterrogationControllerTest.java b/src/test/java/fr/insee/genesis/controller/rest/responses/InterrogationControllerTest.java index 323ac14e..739dbc13 100644 --- a/src/test/java/fr/insee/genesis/controller/rest/responses/InterrogationControllerTest.java +++ b/src/test/java/fr/insee/genesis/controller/rest/responses/InterrogationControllerTest.java @@ -6,7 +6,7 @@ import fr.insee.genesis.domain.service.surveyunit.SurveyUnitService; import fr.insee.genesis.infrastructure.utils.FileUtils; import fr.insee.genesis.stubs.ConfigStub; -import fr.insee.genesis.stubs.QuestionnaireMetadataPersistancePortStub; +import fr.insee.genesis.stubs.QuestionnaireMetadataPersistencePortStub; import fr.insee.genesis.stubs.SurveyUnitPersistencePortStub; import org.assertj.core.api.Assertions; import org.junit.jupiter.api.BeforeAll; @@ -30,7 +30,7 @@ static void init() { surveyUnitPersistencePortStub = new SurveyUnitPersistencePortStub(); SurveyUnitApiPort surveyUnitApiPort = new SurveyUnitService( surveyUnitPersistencePortStub, - new QuestionnaireMetadataService(new QuestionnaireMetadataPersistancePortStub()), + new QuestionnaireMetadataService(new QuestionnaireMetadataPersistencePortStub()), new FileUtils(new ConfigStub()) ); diff --git a/src/test/java/fr/insee/genesis/controller/rest/responses/ModeControllerTest.java b/src/test/java/fr/insee/genesis/controller/rest/responses/ModeControllerTest.java index 45643112..339a27ab 100644 --- a/src/test/java/fr/insee/genesis/controller/rest/responses/ModeControllerTest.java +++ b/src/test/java/fr/insee/genesis/controller/rest/responses/ModeControllerTest.java @@ -6,7 +6,7 @@ import fr.insee.genesis.domain.service.surveyunit.SurveyUnitService; import fr.insee.genesis.infrastructure.utils.FileUtils; import fr.insee.genesis.stubs.ConfigStub; -import fr.insee.genesis.stubs.QuestionnaireMetadataPersistancePortStub; +import fr.insee.genesis.stubs.QuestionnaireMetadataPersistencePortStub; import fr.insee.genesis.stubs.SurveyUnitPersistencePortStub; import org.assertj.core.api.Assertions; import org.junit.jupiter.api.BeforeAll; @@ -30,7 +30,7 @@ static void init() { surveyUnitPersistencePortStub = new SurveyUnitPersistencePortStub(); SurveyUnitApiPort surveyUnitApiPort = new SurveyUnitService( surveyUnitPersistencePortStub, - new QuestionnaireMetadataService(new QuestionnaireMetadataPersistancePortStub()), + new QuestionnaireMetadataService(new QuestionnaireMetadataPersistencePortStub()), new FileUtils(new ConfigStub()) ); diff --git a/src/test/java/fr/insee/genesis/controller/rest/responses/QuestionnaireControllerTest.java b/src/test/java/fr/insee/genesis/controller/rest/responses/QuestionnaireControllerTest.java index a93d8888..3906a388 100644 --- a/src/test/java/fr/insee/genesis/controller/rest/responses/QuestionnaireControllerTest.java +++ b/src/test/java/fr/insee/genesis/controller/rest/responses/QuestionnaireControllerTest.java @@ -9,7 +9,7 @@ import fr.insee.genesis.infrastructure.utils.FileUtils; import fr.insee.genesis.stubs.ConfigStub; import fr.insee.genesis.stubs.DataProcessingContextPersistancePortStub; -import fr.insee.genesis.stubs.QuestionnaireMetadataPersistancePortStub; +import fr.insee.genesis.stubs.QuestionnaireMetadataPersistencePortStub; import fr.insee.genesis.stubs.SurveyUnitPersistencePortStub; import org.assertj.core.api.Assertions; import org.junit.jupiter.api.BeforeAll; @@ -36,7 +36,7 @@ static void init() { dataProcessingContextPersistancePortStub = new DataProcessingContextPersistancePortStub(); SurveyUnitApiPort surveyUnitApiPort = new SurveyUnitService( surveyUnitPersistencePortStub, - new QuestionnaireMetadataService(new QuestionnaireMetadataPersistancePortStub()), + new QuestionnaireMetadataService(new QuestionnaireMetadataPersistencePortStub()), new FileUtils(new ConfigStub()) ); diff --git a/src/test/java/fr/insee/genesis/controller/rest/responses/RawResponseControllerTest.java b/src/test/java/fr/insee/genesis/controller/rest/responses/RawResponseControllerTest.java index 1b0545ca..48faab62 100644 --- a/src/test/java/fr/insee/genesis/controller/rest/responses/RawResponseControllerTest.java +++ b/src/test/java/fr/insee/genesis/controller/rest/responses/RawResponseControllerTest.java @@ -28,7 +28,7 @@ import fr.insee.genesis.stubs.ConfigStub; import fr.insee.genesis.stubs.DataProcessingContextPersistancePortStub; import fr.insee.genesis.stubs.LunaticJsonRawDataPersistanceStub; -import fr.insee.genesis.stubs.QuestionnaireMetadataPersistancePortStub; +import fr.insee.genesis.stubs.QuestionnaireMetadataPersistencePortStub; import fr.insee.genesis.stubs.SurveyUnitPersistencePortStub; import fr.insee.genesis.stubs.SurveyUnitQualityToolPerretAdapterStub; import fr.insee.modelefiliere.RawResponseDto; @@ -51,14 +51,14 @@ class RawResponseControllerTest { private final SurveyUnitQualityToolPerretAdapterStub surveyUnitQualityToolPerretAdapterStub = new SurveyUnitQualityToolPerretAdapterStub(); private final DataProcessingContextPersistancePortStub dataProcessingContextPersistancePortStub = new DataProcessingContextPersistancePortStub(); - private static final QuestionnaireMetadataPersistancePortStub questionnaireMetadataPersistancePortStub = - new QuestionnaireMetadataPersistancePortStub(); + private static final QuestionnaireMetadataPersistencePortStub questionnaireMetadataPersistencePortStub = + new QuestionnaireMetadataPersistencePortStub(); private final LunaticJsonRawDataApiPort lunaticJsonRawDataApiPort = new LunaticJsonRawDataService(lunaticJsonRawDataPersistanceStub, new ControllerUtils(fileUtils), - new QuestionnaireMetadataService(questionnaireMetadataPersistancePortStub), - new SurveyUnitService(surveyUnitPersistencePortStub, new QuestionnaireMetadataService(questionnaireMetadataPersistancePortStub), fileUtils), + new QuestionnaireMetadataService(questionnaireMetadataPersistencePortStub), + new SurveyUnitService(surveyUnitPersistencePortStub, new QuestionnaireMetadataService(questionnaireMetadataPersistencePortStub), fileUtils), new SurveyUnitQualityService(), fileUtils, new DataProcessingContextService(dataProcessingContextPersistancePortStub, surveyUnitPersistencePortStub), diff --git a/src/test/java/fr/insee/genesis/controller/rest/responses/ResponseControllerTest.java b/src/test/java/fr/insee/genesis/controller/rest/responses/ResponseControllerTest.java index 55842089..3ce2927f 100644 --- a/src/test/java/fr/insee/genesis/controller/rest/responses/ResponseControllerTest.java +++ b/src/test/java/fr/insee/genesis/controller/rest/responses/ResponseControllerTest.java @@ -26,7 +26,7 @@ import fr.insee.genesis.infrastructure.utils.FileUtils; import fr.insee.genesis.stubs.ConfigStub; import fr.insee.genesis.stubs.DataProcessingContextPersistancePortStub; -import fr.insee.genesis.stubs.QuestionnaireMetadataPersistancePortStub; +import fr.insee.genesis.stubs.QuestionnaireMetadataPersistencePortStub; import fr.insee.genesis.stubs.SurveyUnitPersistencePortStub; import org.assertj.core.api.Assertions; import org.junit.jupiter.api.BeforeAll; @@ -47,7 +47,7 @@ class ResponseControllerTest { static ResponseController responseControllerStatic; static SurveyUnitPersistencePortStub surveyUnitPersistencePortStub; static DataProcessingContextPersistancePortStub dataProcessingContextPersistancePortStub; - static QuestionnaireMetadataPersistancePortStub questionnaireMetadataPersistancePortStub; + static QuestionnaireMetadataPersistencePortStub questionnaireMetadataPersistencePortStub; static List interrogationIdList; //Constants @@ -62,13 +62,13 @@ static void init() { surveyUnitPersistencePortStub = new SurveyUnitPersistencePortStub(); dataProcessingContextPersistancePortStub = new DataProcessingContextPersistancePortStub(); - questionnaireMetadataPersistancePortStub = new QuestionnaireMetadataPersistancePortStub(); + questionnaireMetadataPersistencePortStub = new QuestionnaireMetadataPersistencePortStub(); Config config = new ConfigStub(); FileUtils fileUtils = new FileUtils(config); SurveyUnitApiPort surveyUnitApiPort = new SurveyUnitService( surveyUnitPersistencePortStub, - new QuestionnaireMetadataService(questionnaireMetadataPersistancePortStub), + new QuestionnaireMetadataService(questionnaireMetadataPersistencePortStub), fileUtils ); @@ -78,7 +78,7 @@ static void init() { , fileUtils , new ControllerUtils(fileUtils) , new AuthUtils(config) - , new QuestionnaireMetadataService(questionnaireMetadataPersistancePortStub) + , new QuestionnaireMetadataService(questionnaireMetadataPersistencePortStub) , new DataProcessingContextService(dataProcessingContextPersistancePortStub, surveyUnitPersistencePortStub) ); diff --git a/src/test/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataServiceTest.java b/src/test/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataServiceTest.java index c37ab461..7f03c962 100644 --- a/src/test/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataServiceTest.java +++ b/src/test/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataServiceTest.java @@ -20,7 +20,7 @@ import fr.insee.genesis.stubs.ConfigStub; import fr.insee.genesis.stubs.DataProcessingContextPersistancePortStub; import fr.insee.genesis.stubs.LunaticJsonRawDataPersistanceStub; -import fr.insee.genesis.stubs.QuestionnaireMetadataPersistancePortStub; +import fr.insee.genesis.stubs.QuestionnaireMetadataPersistencePortStub; import fr.insee.genesis.stubs.SurveyUnitPersistencePortStub; import fr.insee.genesis.stubs.SurveyUnitQualityToolPerretAdapterStub; import org.assertj.core.api.Assertions; @@ -41,7 +41,7 @@ class LunaticJsonRawDataServiceTest { ControllerUtils controllerUtils = new ControllerUtils(fileUtils); static QuestionnaireMetadataService metadataService = - new QuestionnaireMetadataService(new QuestionnaireMetadataPersistancePortStub()); + new QuestionnaireMetadataService(new QuestionnaireMetadataPersistencePortStub()); SurveyUnitPersistencePortStub surveyUnitPersistencePortStub = new SurveyUnitPersistencePortStub(); DataProcessingContextPersistancePortStub dataProcessingContextPersistancePortStub = new DataProcessingContextPersistancePortStub(); diff --git a/src/test/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitServiceTest.java b/src/test/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitServiceTest.java index fe38f681..c7c8b0ab 100644 --- a/src/test/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitServiceTest.java +++ b/src/test/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitServiceTest.java @@ -9,7 +9,7 @@ import fr.insee.genesis.domain.service.metadata.QuestionnaireMetadataService; import fr.insee.genesis.infrastructure.utils.FileUtils; import fr.insee.genesis.stubs.ConfigStub; -import fr.insee.genesis.stubs.QuestionnaireMetadataPersistancePortStub; +import fr.insee.genesis.stubs.QuestionnaireMetadataPersistencePortStub; import fr.insee.genesis.stubs.SurveyUnitPersistencePortStub; import lombok.SneakyThrows; import org.assertj.core.api.Assertions; @@ -38,7 +38,7 @@ static void init(){ surveyUnitPersistencePortStub = new SurveyUnitPersistencePortStub(); surveyUnitServiceStatic = new SurveyUnitService(surveyUnitPersistencePortStub, - new QuestionnaireMetadataService(new QuestionnaireMetadataPersistancePortStub()), + new QuestionnaireMetadataService(new QuestionnaireMetadataPersistencePortStub()), new FileUtils(new ConfigStub())); } diff --git a/src/test/java/fr/insee/genesis/stubs/QuestionnaireMetadataPersistancePortStub.java b/src/test/java/fr/insee/genesis/stubs/QuestionnaireMetadataPersistencePortStub.java similarity index 50% rename from src/test/java/fr/insee/genesis/stubs/QuestionnaireMetadataPersistancePortStub.java rename to src/test/java/fr/insee/genesis/stubs/QuestionnaireMetadataPersistencePortStub.java index d9e4e0e3..accf752f 100644 --- a/src/test/java/fr/insee/genesis/stubs/QuestionnaireMetadataPersistancePortStub.java +++ b/src/test/java/fr/insee/genesis/stubs/QuestionnaireMetadataPersistencePortStub.java @@ -2,7 +2,7 @@ import fr.insee.genesis.domain.model.metadata.QuestionnaireMetadataModel; import fr.insee.genesis.domain.model.surveyunit.Mode; -import fr.insee.genesis.domain.ports.spi.QuestionnaireMetadataPersistancePort; +import fr.insee.genesis.domain.ports.spi.QuestionnaireMetadataPersistencePort; import fr.insee.genesis.infrastructure.document.metadata.QuestionnaireMetadataDocument; import fr.insee.genesis.infrastructure.mappers.QuestionnaireMetadataDocumentMapper; import lombok.Getter; @@ -11,27 +11,30 @@ import java.util.List; @Getter -public class QuestionnaireMetadataPersistancePortStub implements QuestionnaireMetadataPersistancePort { +public class QuestionnaireMetadataPersistencePortStub implements QuestionnaireMetadataPersistencePort { List mongoStub = new ArrayList<>(); @Override - public List find(String questionnaireId, Mode mode) { + public List find(String collectionInstrumentId, Mode mode) { return QuestionnaireMetadataDocumentMapper.INSTANCE.listDocumentToListModel(mongoStub.stream().filter( - questionnaireMetadataDocument -> questionnaireMetadataDocument.questionnaireId().equals(questionnaireId) + questionnaireMetadataDocument -> (questionnaireMetadataDocument.questionnaireId()!= null && questionnaireMetadataDocument.questionnaireId().equals(collectionInstrumentId) || questionnaireMetadataDocument.collectionInstrumentId()!= null && questionnaireMetadataDocument.collectionInstrumentId().equals(collectionInstrumentId)) && questionnaireMetadataDocument.mode().equals(mode)) .toList()); } @Override public void save(QuestionnaireMetadataModel questionnaireMetadataModel) { - remove(questionnaireMetadataModel.questionnaireId(), questionnaireMetadataModel.mode()); + remove(questionnaireMetadataModel.collectionInstrumentId(), questionnaireMetadataModel.mode()); mongoStub.add(QuestionnaireMetadataDocumentMapper.INSTANCE.modelToDocument(questionnaireMetadataModel)); } @Override - public void remove(String questionnaireId, Mode mode) { - mongoStub.removeIf(questionnaireMetadataDocument -> questionnaireMetadataDocument.questionnaireId().equals(questionnaireId) + public void remove(String collectionInstrumentId, Mode mode) { + mongoStub.removeIf(questionnaireMetadataDocument -> questionnaireMetadataDocument.collectionInstrumentId()!= null && questionnaireMetadataDocument.collectionInstrumentId().equals(collectionInstrumentId) && questionnaireMetadataDocument.mode().equals(mode) ); + mongoStub.removeIf(questionnaireMetadataDocument -> questionnaireMetadataDocument.questionnaireId()!= null && questionnaireMetadataDocument.questionnaireId().equals(collectionInstrumentId) + && questionnaireMetadataDocument.mode().equals(mode) + ); } } From 3d3dd5b2b042578b287480c78e577560de73a800 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Henninger?= Date: Tue, 25 Nov 2025 17:28:53 +0100 Subject: [PATCH 15/42] fix: questionnaire with new collecttionInstrumentId were not listed --- .../domain/model/surveyunit/SurveyUnitModel.java | 4 ++++ .../genesis/domain/ports/api/RawResponseApiPort.java | 4 ++-- .../adapter/SurveyUnitMongoAdapter.java | 12 +++++++----- 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/src/main/java/fr/insee/genesis/domain/model/surveyunit/SurveyUnitModel.java b/src/main/java/fr/insee/genesis/domain/model/surveyunit/SurveyUnitModel.java index 307d592d..94f0e1d2 100644 --- a/src/main/java/fr/insee/genesis/domain/model/surveyunit/SurveyUnitModel.java +++ b/src/main/java/fr/insee/genesis/domain/model/surveyunit/SurveyUnitModel.java @@ -24,6 +24,10 @@ public class SurveyUnitModel { // New name of questionnaireId private String collectionInstrumentId; // To be removed + /** + * @deprecated We will not receive this identifier anymore + */ + @Deprecated(forRemoval = true) private String campaignId; private String interrogationId; // New name of idUE diff --git a/src/main/java/fr/insee/genesis/domain/ports/api/RawResponseApiPort.java b/src/main/java/fr/insee/genesis/domain/ports/api/RawResponseApiPort.java index 1d318538..e5d589c5 100644 --- a/src/main/java/fr/insee/genesis/domain/ports/api/RawResponseApiPort.java +++ b/src/main/java/fr/insee/genesis/domain/ports/api/RawResponseApiPort.java @@ -12,8 +12,8 @@ public interface RawResponseApiPort { - List getRawResponses(String questionnaireModelId, Mode mode, List interrogationIdList); - DataProcessResult processRawResponses(String questionnaireId, List interrogationIdList, List errors) throws GenesisException; + List getRawResponses(String collectionInstrumentId, Mode mode, List interrogationIdList); + DataProcessResult processRawResponses(String collectionInstrumentId, List interrogationIdList, List errors) throws GenesisException; List convertRawResponse(List rawResponses, VariablesMap variablesMap); void updateProcessDates(List surveyUnitModels); } diff --git a/src/main/java/fr/insee/genesis/infrastructure/adapter/SurveyUnitMongoAdapter.java b/src/main/java/fr/insee/genesis/infrastructure/adapter/SurveyUnitMongoAdapter.java index c9278070..37772dfa 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/adapter/SurveyUnitMongoAdapter.java +++ b/src/main/java/fr/insee/genesis/infrastructure/adapter/SurveyUnitMongoAdapter.java @@ -3,6 +3,7 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import com.mongodb.client.MongoCollection; import fr.insee.genesis.Constants; import fr.insee.genesis.domain.model.surveyunit.SurveyUnitModel; import fr.insee.genesis.domain.ports.spi.SurveyUnitPersistencePort; @@ -11,6 +12,7 @@ import fr.insee.genesis.infrastructure.repository.SurveyUnitMongoDBRepository; import jakarta.validation.constraints.NotNull; import lombok.extern.slf4j.Slf4j; +import org.bson.Document; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.data.mongodb.core.MongoTemplate; @@ -207,11 +209,11 @@ public long countByCampaignId(String campaignId){ @Override public Set findDistinctQuestionnaireIds() { Set questionnaireIds = new HashSet<>(); - for(String questionnaireId : mongoTemplate.getCollection(Constants.MONGODB_RESPONSE_COLLECTION_NAME).distinct( - QUESTIONNAIRE_ID, - String.class)){ - questionnaireIds.add(questionnaireId); - } + // Id selection is executed by mongoDB + MongoCollection collection = mongoTemplate.getCollection(Constants.MONGODB_RESPONSE_COLLECTION_NAME); + collection.distinct("questionnaireId", String.class).into(questionnaireIds); + collection.distinct("collectionInstrumentId", String.class).into(questionnaireIds); + questionnaireIds.remove(null); return questionnaireIds; } From 70aba98f4f7359b343fda2ab04091fec0f190fb8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Henninger?= Date: Tue, 25 Nov 2025 17:41:07 +0100 Subject: [PATCH 16/42] fix: raw responses was not updated with processDate --- src/main/java/fr/insee/genesis/Constants.java | 3 +-- .../infrastructure/adapter/LunaticJsonRawDataMongoAdapter.java | 2 +- .../infrastructure/adapter/RawResponseMongoAdapter.java | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/main/java/fr/insee/genesis/Constants.java b/src/main/java/fr/insee/genesis/Constants.java index 25fabfbd..92e6ab66 100644 --- a/src/main/java/fr/insee/genesis/Constants.java +++ b/src/main/java/fr/insee/genesis/Constants.java @@ -23,8 +23,7 @@ public class Constants { public static final String MONGODB_CONTEXTUAL_EXTERNAL_COLLECTION_NAME = "editedExternal"; public static final String LOOP_NAME_PREFIX = "BOUCLE"; public static final String MONGODB_RESPONSE_COLLECTION_NAME = "responses"; - public static final String MONGODB_RESPONSE_RAW_COLLECTION_NAME = "lunaticjsondata"; - public static final String MONGODB_VARIABLETYPE_COLLECTION_NAME = "variabletypes"; + public static final String MONGODB_RAW_RESPONSES_COLLECTION_NAME = "rawResponses"; public static final String VOLUMETRY_FOLDER_NAME = "genesis_volumetries"; public static final String VOLUMETRY_FILE_SUFFIX = "_VOLUMETRY"; public static final String VOLUMETRY_RAW_FILE_SUFFIX = "_RAW_VOLUMETRY"; diff --git a/src/main/java/fr/insee/genesis/infrastructure/adapter/LunaticJsonRawDataMongoAdapter.java b/src/main/java/fr/insee/genesis/infrastructure/adapter/LunaticJsonRawDataMongoAdapter.java index bd05024d..e13a20bc 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/adapter/LunaticJsonRawDataMongoAdapter.java +++ b/src/main/java/fr/insee/genesis/infrastructure/adapter/LunaticJsonRawDataMongoAdapter.java @@ -66,7 +66,7 @@ public void updateProcessDates(String campaignId, Set interrogationIds) @Override public Set findDistinctQuestionnaireIds() { Set questionnaireIds = new HashSet<>(); - for(String questionnaireId : mongoTemplate.getCollection(Constants.MONGODB_RESPONSE_RAW_COLLECTION_NAME).distinct( + for(String questionnaireId : mongoTemplate.getCollection(Constants.MONGODB_RAW_RESPONSES_COLLECTION_NAME).distinct( "questionnaireId", String.class)){ questionnaireIds.add(questionnaireId); diff --git a/src/main/java/fr/insee/genesis/infrastructure/adapter/RawResponseMongoAdapter.java b/src/main/java/fr/insee/genesis/infrastructure/adapter/RawResponseMongoAdapter.java index ad7b7bee..aad390b5 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/adapter/RawResponseMongoAdapter.java +++ b/src/main/java/fr/insee/genesis/infrastructure/adapter/RawResponseMongoAdapter.java @@ -43,7 +43,7 @@ public void updateProcessDates(String collectionInstrumentId, Set interr mongoTemplate.updateMulti( Query.query(Criteria.where("collectionInstrumentId").is(collectionInstrumentId).and("interrogationId").in(interrogationIds)) , new Update().set("processDate", LocalDateTime.now()) - , Constants.MONGODB_LUNATIC_RAWDATA_COLLECTION_NAME + , Constants.MONGODB_RAW_RESPONSES_COLLECTION_NAME ); } } From 456dba46b6e6f533b792d05e78d288c4498fde1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Henninger?= Date: Tue, 25 Nov 2025 18:16:52 +0100 Subject: [PATCH 17/42] fix; collectionInstrumentId in lunaticmodels was not persisted on insert --- .../infrastructure/adapter/LunaticModelMongoAdapter.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/java/fr/insee/genesis/infrastructure/adapter/LunaticModelMongoAdapter.java b/src/main/java/fr/insee/genesis/infrastructure/adapter/LunaticModelMongoAdapter.java index 5903ea6f..20324413 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/adapter/LunaticModelMongoAdapter.java +++ b/src/main/java/fr/insee/genesis/infrastructure/adapter/LunaticModelMongoAdapter.java @@ -34,7 +34,9 @@ public void save(LunaticModelModel lunaticModelModel) { ); mongoTemplate.update(LunaticModelDocument.class) .matching(criteria) - .apply(new Update().set("lunaticModel", lunaticModelModel.lunaticModel())) + .apply(new Update() + .set("lunaticModel", lunaticModelModel.lunaticModel()) + .setOnInsert("collectionInstrumentId", lunaticModelModel.collectionInstrumentId())) .upsert(); } From 72aa21d858099ea6c2aec91f70c7ff04cade6163 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Henninger?= Date: Mon, 1 Dec 2025 15:41:33 +0100 Subject: [PATCH 18/42] fix: fixes for KW --- .../controller/dto/SurveyUnitSimplified.java | 8 ++++++-- .../controller/rest/responses/ModeController.java | 2 +- .../rest/responses/ResponseController.java | 10 +++++----- .../ports/spi/SurveyUnitPersistencePort.java | 2 -- .../service/surveyunit/SurveyUnitService.java | 4 ++-- .../adapter/SurveyUnitMongoAdapter.java | 12 ++++-------- .../controller/rest/ControllerAccessTest.java | 2 +- .../stubs/SurveyUnitPersistencePortStub.java | 15 +-------------- 8 files changed, 20 insertions(+), 35 deletions(-) diff --git a/src/main/java/fr/insee/genesis/controller/dto/SurveyUnitSimplified.java b/src/main/java/fr/insee/genesis/controller/dto/SurveyUnitSimplified.java index c1d4abec..259621e6 100644 --- a/src/main/java/fr/insee/genesis/controller/dto/SurveyUnitSimplified.java +++ b/src/main/java/fr/insee/genesis/controller/dto/SurveyUnitSimplified.java @@ -11,10 +11,14 @@ @Data public class SurveyUnitSimplified { - private String questionnaireId; + private String collectionInstrumentId; + /** + * @deprecated We will not reveive this piece of information anymore + */ + @Deprecated(forRemoval = true) private String campaignId; private String interrogationId; - private String surveyUnitId; + private String usualSurveyUnitId; private Mode mode; private List variablesUpdate; private List externalVariables; diff --git a/src/main/java/fr/insee/genesis/controller/rest/responses/ModeController.java b/src/main/java/fr/insee/genesis/controller/rest/responses/ModeController.java index efb6400c..ac3b071d 100644 --- a/src/main/java/fr/insee/genesis/controller/rest/responses/ModeController.java +++ b/src/main/java/fr/insee/genesis/controller/rest/responses/ModeController.java @@ -28,7 +28,7 @@ public ModeController(SurveyUnitApiPort surveyUnitService) { @Operation(summary = "List sources/modes used for a given collection instrument (ex questionnaire)") @GetMapping(path = "/by-questionnaire") - public ResponseEntity> getModesByQuestionnaire(@RequestParam("collectionInstrumentId") String collectionInstrumentId) { + public ResponseEntity> getModesByQuestionnaire(@RequestParam("questionnaireId") String collectionInstrumentId) { List modes = surveyUnitService.findModesByCollectionInstrumentId(collectionInstrumentId); return ResponseEntity.ok(modes); } diff --git a/src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java b/src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java index 21a009d0..642dfceb 100644 --- a/src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java +++ b/src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java @@ -283,10 +283,10 @@ public ResponseEntity getLatestByInterrogationOneObject(@R outputExternalVariables.addAll(response.getExternalVariables()); }); return ResponseEntity.ok(SurveyUnitSimplified.builder() - .questionnaireId(responses.getFirst().getCollectionInstrumentId()) + .collectionInstrumentId(responses.getFirst().getCollectionInstrumentId()) .campaignId(responses.getFirst().getCampaignId()) .interrogationId(responses.getFirst().getInterrogationId()) - .surveyUnitId(responses.getFirst().getUsualSurveyUnitId()) + .usualSurveyUnitId(responses.getFirst().getUsualSurveyUnitId()) .variablesUpdate(outputVariables) .externalVariables(outputExternalVariables) .build()); @@ -315,10 +315,10 @@ public ResponseEntity> getLatestForInterrogationListA }); if (!outputVariables.isEmpty() || !outputExternalVariables.isEmpty()) { results.add(SurveyUnitSimplified.builder() - .questionnaireId(responses.getFirst().getCollectionInstrumentId()) + .collectionInstrumentId(responses.getFirst().getCollectionInstrumentId()) .campaignId(responses.getFirst().getCampaignId()) .interrogationId(responses.getFirst().getInterrogationId()) - .surveyUnitId(responses.getFirst().getUsualSurveyUnitId()) + .usualSurveyUnitId(responses.getFirst().getUsualSurveyUnitId()) .mode(mode) .variablesUpdate(outputVariables) .externalVariables(outputExternalVariables) @@ -395,7 +395,7 @@ private SurveyUnitSimplified fusionWithLastUpdated(List respons Mode modeWrapped = Mode.getEnumFromModeName(mode); simplifiedResponse = SurveyUnitSimplified.builder() - .questionnaireId(responsesForSingleInterrId.getFirst().getCollectionInstrumentId()) + .collectionInstrumentId(responsesForSingleInterrId.getFirst().getCollectionInstrumentId()) .campaignId(responsesForSingleInterrId.getFirst().getCampaignId()) .interrogationId(responsesForSingleInterrId.getFirst().getInterrogationId()) .mode(modeWrapped) diff --git a/src/main/java/fr/insee/genesis/domain/ports/spi/SurveyUnitPersistencePort.java b/src/main/java/fr/insee/genesis/domain/ports/spi/SurveyUnitPersistencePort.java index e7237830..21986d55 100644 --- a/src/main/java/fr/insee/genesis/domain/ports/spi/SurveyUnitPersistencePort.java +++ b/src/main/java/fr/insee/genesis/domain/ports/spi/SurveyUnitPersistencePort.java @@ -26,8 +26,6 @@ public interface SurveyUnitPersistencePort { Stream findByQuestionnaireId(String questionnaireId); - List findInterrogationIdsByQuestionnaireId(String questionnaireId); - List findInterrogationIdsByCollectionInstrumentId(String collectionInstrumentId); List findInterrogationIdsByQuestionnaireIdAndDateAfter(String questionnaireId, LocalDateTime since); diff --git a/src/main/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitService.java b/src/main/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitService.java index 67abea3c..89a082a5 100644 --- a/src/main/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitService.java +++ b/src/main/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitService.java @@ -265,7 +265,7 @@ public SurveyUnitDto findLatestValuesByStateByIdAndByCollectionInstrumentId( @Override public List findDistinctInterrogationIdsByQuestionnaireId(String questionnaireId) { - List surveyUnitModels = surveyUnitPersistencePort.findInterrogationIdsByQuestionnaireId(questionnaireId); + List surveyUnitModels = surveyUnitPersistencePort.findInterrogationIdsByCollectionInstrumentId(questionnaireId); List suIds = new ArrayList<>(); surveyUnitModels.forEach(surveyUnitModel -> suIds.add(new InterrogationId(surveyUnitModel.getInterrogationId()))); return suIds.stream().distinct().toList(); @@ -325,7 +325,7 @@ public long countInterrogationIdsByQuestionnaireId(String questionnaireId) { @Override public List findInterrogationIdsAndModesByQuestionnaireId(String questionnaireId) { - List surveyUnitModels = surveyUnitPersistencePort.findInterrogationIdsByQuestionnaireId(questionnaireId); + List surveyUnitModels = surveyUnitPersistencePort.findInterrogationIdsByCollectionInstrumentId(questionnaireId); return surveyUnitModels.stream().distinct().toList(); } diff --git a/src/main/java/fr/insee/genesis/infrastructure/adapter/SurveyUnitMongoAdapter.java b/src/main/java/fr/insee/genesis/infrastructure/adapter/SurveyUnitMongoAdapter.java index 37772dfa..8a95adec 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/adapter/SurveyUnitMongoAdapter.java +++ b/src/main/java/fr/insee/genesis/infrastructure/adapter/SurveyUnitMongoAdapter.java @@ -151,16 +151,12 @@ public Set findDistinctCampaignIds() { return campaignIds; } - @Override - public List findInterrogationIdsByQuestionnaireId(String questionnaireId) { - List surveyUnits = mongoRepository.findInterrogationIdsByQuestionnaireId(questionnaireId); - return surveyUnits.isEmpty() ? Collections.emptyList() : SurveyUnitDocumentMapper.INSTANCE.listDocumentToListModel(surveyUnits); - } - @Override public List findInterrogationIdsByCollectionInstrumentId(String collectionInstrumentId) { - List surveyUnits = mongoRepository.findInterrogationIdsByCollectionInstrumentId(collectionInstrumentId); - return surveyUnits.isEmpty() ? Collections.emptyList() : SurveyUnitDocumentMapper.INSTANCE.listDocumentToListModel(surveyUnits); + List results = new ArrayList<>(); + results.addAll(mongoRepository.findInterrogationIdsByCollectionInstrumentId(collectionInstrumentId)); + results.addAll(mongoRepository.findInterrogationIdsByQuestionnaireId(collectionInstrumentId)); + return results.isEmpty() ? Collections.emptyList() : SurveyUnitDocumentMapper.INSTANCE.listDocumentToListModel(results); } @Override diff --git a/src/test/java/fr/insee/genesis/controller/rest/ControllerAccessTest.java b/src/test/java/fr/insee/genesis/controller/rest/ControllerAccessTest.java index d8647c9d..7591fb83 100644 --- a/src/test/java/fr/insee/genesis/controller/rest/ControllerAccessTest.java +++ b/src/test/java/fr/insee/genesis/controller/rest/ControllerAccessTest.java @@ -111,7 +111,7 @@ private static Stream endpointsReader() { Arguments.of("/questionnaires/with-campaigns"), Arguments.of("/questionnaires/by-campaign?campaignId=CAMPAIGNTEST"), Arguments.of("/questionnaires/"), - Arguments.of("/modes/by-questionnaire?collectionInstrumentId=QUESTTEST"), + Arguments.of("/modes/by-questionnaire?questionnaireId=QUESTTEST"), Arguments.of("/modes/by-campaign?campaignId=CAMPAIGNTEST"), Arguments.of("/interrogations/by-questionnaire?questionnaireId=QUESTTEST"), Arguments.of("/campaigns/with-questionnaires"), diff --git a/src/test/java/fr/insee/genesis/stubs/SurveyUnitPersistencePortStub.java b/src/test/java/fr/insee/genesis/stubs/SurveyUnitPersistencePortStub.java index c9ee0472..0dfeda2c 100644 --- a/src/test/java/fr/insee/genesis/stubs/SurveyUnitPersistencePortStub.java +++ b/src/test/java/fr/insee/genesis/stubs/SurveyUnitPersistencePortStub.java @@ -76,19 +76,6 @@ public Stream findByQuestionnaireId(String questionnaireId) { return surveyUnitModelList.stream(); } - @Override - public List findInterrogationIdsByQuestionnaireId(String questionnaireId) { - List surveyUnitModelList = new ArrayList<>(); - for(SurveyUnitModel SurveyUnitModel : mongoStub){ - if(SurveyUnitModel.getCollectionInstrumentId().equals(questionnaireId)) - surveyUnitModelList.add( - new SurveyUnitModel(SurveyUnitModel.getInterrogationId(), SurveyUnitModel.getMode()) - ); - } - - return surveyUnitModelList; - } - @Override public List findInterrogationIdsByCollectionInstrumentId(String collectionInstrumentId) { List surveyUnitModelList = new ArrayList<>(); @@ -103,7 +90,7 @@ public List findInterrogationIdsByCollectionInstrumentId(String @Override public List findModesByQuestionnaireIdV2(String questionnaireId) { - return findInterrogationIdsByQuestionnaireId(questionnaireId); + return findInterrogationIdsByCollectionInstrumentId(questionnaireId); } @Override From 04ff521dacb0b73cdf0e5bf9decec32d9b7bee49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Henninger?= Date: Mon, 1 Dec 2025 15:53:32 +0100 Subject: [PATCH 19/42] fix: fix new ids --- .../controller/rest/responses/ResponseControllerTest.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/test/java/fr/insee/genesis/controller/rest/responses/ResponseControllerTest.java b/src/test/java/fr/insee/genesis/controller/rest/responses/ResponseControllerTest.java index 3ce2927f..eee52879 100644 --- a/src/test/java/fr/insee/genesis/controller/rest/responses/ResponseControllerTest.java +++ b/src/test/java/fr/insee/genesis/controller/rest/responses/ResponseControllerTest.java @@ -182,8 +182,8 @@ void getLatestByUEOneObjectTest() { Assertions.assertThat(response.getStatusCode().is2xxSuccessful()).isTrue(); Assertions.assertThat(response.getBody()).isNotNull(); Assertions.assertThat(response.getBody().getInterrogationId()).isEqualTo(DEFAULT_INTERROGATION_ID); - Assertions.assertThat(response.getBody().getSurveyUnitId()).isEqualTo(DEFAULT_ID_UE); - Assertions.assertThat(response.getBody().getQuestionnaireId()).isEqualTo(DEFAULT_QUESTIONNAIRE_ID); + Assertions.assertThat(response.getBody().getUsualSurveyUnitId()).isEqualTo(DEFAULT_ID_UE); + Assertions.assertThat(response.getBody().getCollectionInstrumentId()).isEqualTo(DEFAULT_QUESTIONNAIRE_ID); } @Test @@ -193,7 +193,7 @@ void getLatestForUEListTest() { Assertions.assertThat(response.getStatusCode().is2xxSuccessful()).isTrue(); Assertions.assertThat(response.getBody()).isNotNull().isNotEmpty(); Assertions.assertThat(response.getBody().getFirst().getInterrogationId()).isEqualTo(DEFAULT_INTERROGATION_ID); - Assertions.assertThat(response.getBody().getFirst().getSurveyUnitId()).isEqualTo(DEFAULT_ID_UE); + Assertions.assertThat(response.getBody().getFirst().getUsualSurveyUnitId()).isEqualTo(DEFAULT_ID_UE); } // Perret tests From f19f02597a446ba016a7412441ece2d24d139fc4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Henninger?= Date: Tue, 2 Dec 2025 17:59:48 +0100 Subject: [PATCH 20/42] fix: missing@Mapper annotation --- .../genesis/infrastructure/mappers/LunaticModelMapper.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/java/fr/insee/genesis/infrastructure/mappers/LunaticModelMapper.java b/src/main/java/fr/insee/genesis/infrastructure/mappers/LunaticModelMapper.java index 07e8beec..ca65dac3 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/mappers/LunaticModelMapper.java +++ b/src/main/java/fr/insee/genesis/infrastructure/mappers/LunaticModelMapper.java @@ -3,11 +3,13 @@ import fr.insee.genesis.domain.model.lunaticmodel.LunaticModelModel; import fr.insee.genesis.infrastructure.document.lunaticmodel.LunaticModelDocument; import org.mapstruct.AfterMapping; +import org.mapstruct.Mapper; import org.mapstruct.MappingTarget; import org.mapstruct.factory.Mappers; import java.util.List; +@Mapper public interface LunaticModelMapper { LunaticModelMapper INSTANCE = Mappers.getMapper(LunaticModelMapper.class); From 242ab9eff7813bdf73fac57d2d6d76cbf3019120 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Henninger?= Date: Wed, 3 Dec 2025 10:18:27 +0100 Subject: [PATCH 21/42] fix: improve performance --- .../document/metadata/QuestionnaireMetadataDocument.java | 1 + .../infrastructure/document/surveyunit/SurveyUnitDocument.java | 2 ++ 2 files changed, 3 insertions(+) diff --git a/src/main/java/fr/insee/genesis/infrastructure/document/metadata/QuestionnaireMetadataDocument.java b/src/main/java/fr/insee/genesis/infrastructure/document/metadata/QuestionnaireMetadataDocument.java index ec22b51b..060e6765 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/document/metadata/QuestionnaireMetadataDocument.java +++ b/src/main/java/fr/insee/genesis/infrastructure/document/metadata/QuestionnaireMetadataDocument.java @@ -12,6 +12,7 @@ I choose to keep the record here (this choice can be challenged)*/ "java:S6355" // Add 'since' and/or 'forRemoval' arguments to the @Deprecated annotation }) @CompoundIndex(name = "questionnaireId_1_mode_1", def = "{'questionnaireId': 1, 'mode': 1}") +@CompoundIndex(name = "collectionInstrumentId_1_mode_1", def = "{'collectionInstrumentId': 1, 'mode': 1}") @Document(collection = "questionnaireMetadatas") public record QuestionnaireMetadataDocument( @Deprecated diff --git a/src/main/java/fr/insee/genesis/infrastructure/document/surveyunit/SurveyUnitDocument.java b/src/main/java/fr/insee/genesis/infrastructure/document/surveyunit/SurveyUnitDocument.java index 0eaae474..95503c7f 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/document/surveyunit/SurveyUnitDocument.java +++ b/src/main/java/fr/insee/genesis/infrastructure/document/surveyunit/SurveyUnitDocument.java @@ -16,6 +16,8 @@ @CompoundIndex(name = "questionnaireId_1_campaignId_1", def = "{'questionnaireId': 1, 'campaignId': 1}") @CompoundIndex(name = "questionnaireId_1_interrogationId_1", def = "{'questionnaireId': 1, 'interrogationId': 1}") @CompoundIndex(name = "interrogationId_1_questionnaireId_1", def = "{'interrogationId': 1, 'questionnaireId': 1}") +@CompoundIndex(name = "collectionInstrumentId_1_interrogationId_1", def = "{'collectionInstrumentId': 1, 'interrogationId': 1}") +@CompoundIndex(name = "interrogationId_1_collectionInstrumentId_1", def = "{'interrogationId': 1, 'collectionInstrumentId': 1}") @JsonIgnoreProperties(ignoreUnknown = true) public class SurveyUnitDocument { From cafdcaf432ce58c8f58d4803560e9c08e4f27d9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Henninger?= Date: Wed, 3 Dec 2025 11:36:43 +0100 Subject: [PATCH 22/42] chore: modelefiliere bump to 2.0.0 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 304c8a1c..4270eebd 100644 --- a/pom.xml +++ b/pom.xml @@ -101,7 +101,7 @@ fr.insee modelefiliere - add-processed-response-SNAPSHOT + 2.0.0 com.networknt From 8fd8b91956dfd1a06e3db6c9fd9494a161608853 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Henninger?= Date: Thu, 4 Dec 2025 15:03:54 +0100 Subject: [PATCH 23/42] chore: sonar --- .../service/rawdata/LunaticJsonRawDataService.java | 10 ---------- .../infrastructure/adapter/SurveyUnitMongoAdapter.java | 2 +- .../cucumber/functional_tests/RawDataDefinitions.java | 5 +++-- 3 files changed, 4 insertions(+), 13 deletions(-) diff --git a/src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java b/src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java index 1518ced1..98a7d85a 100644 --- a/src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java +++ b/src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java @@ -195,7 +195,6 @@ public List convertRawData(List rawDat RawDataModelType rawDataModelType = getRawDataModelType(rawData); //Get optional fields - String contextualId = getContextualId(rawData); Boolean isCapturedIndirectly = getIsCapturedIndirectly(rawData); LocalDateTime validationDate = getValidationDate(rawData); @@ -265,15 +264,6 @@ private static Boolean getIsCapturedIndirectly(LunaticJsonRawDataModel rawData) } - private static String getContextualId(LunaticJsonRawDataModel rawData) { - try{ - return rawData.data().get("contextualId") == null ? null : rawData.data().get("contextualId").toString(); - }catch(Exception e){ - log.warn("Exception when parsing contextual id : {}}",e.toString()); - return null; - } - } - @Override public List getUnprocessedDataIds() { List dtos = new ArrayList<>(); diff --git a/src/main/java/fr/insee/genesis/infrastructure/adapter/SurveyUnitMongoAdapter.java b/src/main/java/fr/insee/genesis/infrastructure/adapter/SurveyUnitMongoAdapter.java index 8a95adec..79de453a 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/adapter/SurveyUnitMongoAdapter.java +++ b/src/main/java/fr/insee/genesis/infrastructure/adapter/SurveyUnitMongoAdapter.java @@ -207,7 +207,7 @@ public Set findDistinctQuestionnaireIds() { Set questionnaireIds = new HashSet<>(); // Id selection is executed by mongoDB MongoCollection collection = mongoTemplate.getCollection(Constants.MONGODB_RESPONSE_COLLECTION_NAME); - collection.distinct("questionnaireId", String.class).into(questionnaireIds); + collection.distinct(QUESTIONNAIRE_ID, String.class).into(questionnaireIds); collection.distinct("collectionInstrumentId", String.class).into(questionnaireIds); questionnaireIds.remove(null); return questionnaireIds; diff --git a/src/test/java/cucumber/functional_tests/RawDataDefinitions.java b/src/test/java/cucumber/functional_tests/RawDataDefinitions.java index 6257ff92..40aa92fa 100644 --- a/src/test/java/cucumber/functional_tests/RawDataDefinitions.java +++ b/src/test/java/cucumber/functional_tests/RawDataDefinitions.java @@ -100,7 +100,7 @@ public class RawDataDefinitions { RawResponseInputRepository rawResponseInputRepositoryStub = new RawResponseInputRepository(null, null) { @Override public void saveAsRawJson(RawResponseDto dto) { - // Ne rien faire — stub pour les tests + // Do nothing - stub for test } }; @@ -122,9 +122,10 @@ public List convertRawResponse(List rawResponses, @Override public void updateProcessDates(List surveyUnitModels) { - + // Do nothing - stub for test } }; + RawResponseController rawResponseController = new RawResponseController( lunaticJsonRawDataService, rawResponseApiPortStub, rawResponseInputRepositoryStub ); From f03b20d4072514ed80a416eb6000f146eb238e71 Mon Sep 17 00:00:00 2001 From: Alice Lambois Date: Thu, 4 Dec 2025 16:44:08 +0100 Subject: [PATCH 24/42] style: fix vocabulary --- .../controller/rest/responses/RawResponseController.java | 2 +- .../controller/rest/responses/RawResponseControllerTest.java | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseController.java b/src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseController.java index 31432317..1c2a9ae2 100644 --- a/src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseController.java +++ b/src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseController.java @@ -220,7 +220,7 @@ public ResponseEntity processRawResponses( @Operation(summary = "Get campaign id and interrogationId from all unprocessed raw json data") @GetMapping(path = "/responses/raw/lunatic-json/get/unprocessed") @PreAuthorize("hasRole('SCHEDULER')") - public ResponseEntity> getUnproccessedJsonRawData() { + public ResponseEntity> getUnprocessedJsonRawData() { log.info("Try to get unprocessed raw JSON datas..."); return ResponseEntity.ok(lunaticJsonRawDataApiPort.getUnprocessedDataIds()); } diff --git a/src/test/java/fr/insee/genesis/controller/rest/responses/RawResponseControllerTest.java b/src/test/java/fr/insee/genesis/controller/rest/responses/RawResponseControllerTest.java index 48faab62..69799ea4 100644 --- a/src/test/java/fr/insee/genesis/controller/rest/responses/RawResponseControllerTest.java +++ b/src/test/java/fr/insee/genesis/controller/rest/responses/RawResponseControllerTest.java @@ -150,7 +150,7 @@ void getUnprocessedDataTest(){ addJsonRawDataDocumentToStub(campaignId, questionnaireId, interrogationId, null); //WHEN - List dtos = rawResponseController.getUnproccessedJsonRawData().getBody(); + List dtos = rawResponseController.getUnprocessedJsonRawData().getBody(); //THEN Assertions.assertThat(dtos).isNotNull().isNotEmpty().hasSize(1); @@ -169,7 +169,7 @@ void getUnprocessedDataTest_processDate_present(){ addJsonRawDataDocumentToStub(campaignId, questionnaireId, interrogationId, LocalDateTime.now()); //WHEN - List dtos = rawResponseController.getUnproccessedJsonRawData().getBody(); + List dtos = rawResponseController.getUnprocessedJsonRawData().getBody(); //THEN Assertions.assertThat(dtos).isNotNull().isEmpty(); From aaff23506e1d61e0d5c179f31e94f732cfb60968 Mon Sep 17 00:00:00 2001 From: Alice Lambois Date: Thu, 4 Dec 2025 16:44:44 +0100 Subject: [PATCH 25/42] style : fix some issues --- .../controller/rest/responses/ResponseController.java | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java b/src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java index 642dfceb..e8a75af0 100644 --- a/src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java +++ b/src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java @@ -67,7 +67,7 @@ @RequestMapping(path = "/responses" ) @Controller -@Tag(name = "Response services", description = "A **response** is considered the entire set of data associated with an interrogation (survey unit x questionnaireId). \n\n These data may have different state (collected, edited, external, ...) ") +@Tag(name = "Response services", description = "A **response** is considered the entire set of data associated with an interrogation (survey unit x collecton Intrument Id [ex questionnaire]). \n\n These data may have different state (collected, edited, external, ...) ") @Slf4j public class ResponseController implements CommonApiResponse { @@ -108,7 +108,7 @@ public ResponseEntity saveResponsesFromXmlFile(@RequestParam("pathLunati @RequestParam(value = "pathSpecFile") String metadataFilePath, @RequestParam(value = "mode") Mode modeSpecified )throws Exception { - log.info("Try to read Xml file : {}", xmlFile); + log.info("Try to read one Xml file : {}", xmlFile); Path filepath = Paths.get(xmlFile); if (getFileSizeInMB(filepath) <= Constants.MAX_FILE_SIZE_UNTIL_SEQUENTIAL) { @@ -248,8 +248,7 @@ public ResponseEntity findResponsesByInterrogationAndCollectionInstrumen @RequestParam("interrogationId") String interrogationId, @RequestParam("collectionInstrumentId") String collectionInstrumentId) throws GenesisException { //Check context - DataProcessingContextModel dataProcessingContextModel = - contextService.getContext(interrogationId); + DataProcessingContextModel dataProcessingContextModel = contextService.getContext(interrogationId); if(dataProcessingContextModel == null || !dataProcessingContextModel.isWithReview()){ return ResponseEntity.status(403).body(new ApiError("Review is disabled for that partition")); @@ -347,7 +346,7 @@ public ResponseEntity> getLatestForInterrogationListV //!!!WARNING!!! : FOR PERFORMANCES PURPOSES, WE DONT'MAKE REQUESTS ON INDIVIDUAL ELEMENTS ANYMORE, BUT ON A SUBLIST OF THE INPUTLIST final int SUBBLOCK_SIZE = 100; int offset = 0; - List interrogationIdsSubList = null; + List interrogationIdsSubList; for(String mode : modes) { From 55d63e8ae0a6a06263f318034c0d7b74ffb4682f Mon Sep 17 00:00:00 2001 From: Alice Lambois Date: Thu, 4 Dec 2025 16:45:25 +0100 Subject: [PATCH 26/42] style: deprecate some methods --- .../controller/dto/SurveyUnitSimplified.java | 2 +- .../rest/responses/RawResponseController.java | 51 ------------------- .../rest/responses/ResponseController.java | 6 ++- 3 files changed, 5 insertions(+), 54 deletions(-) diff --git a/src/main/java/fr/insee/genesis/controller/dto/SurveyUnitSimplified.java b/src/main/java/fr/insee/genesis/controller/dto/SurveyUnitSimplified.java index 259621e6..57ea1d84 100644 --- a/src/main/java/fr/insee/genesis/controller/dto/SurveyUnitSimplified.java +++ b/src/main/java/fr/insee/genesis/controller/dto/SurveyUnitSimplified.java @@ -15,7 +15,7 @@ public class SurveyUnitSimplified { /** * @deprecated We will not reveive this piece of information anymore */ - @Deprecated(forRemoval = true) + @Deprecated(forRemoval = true, since = "2026-01-01") private String campaignId; private String interrogationId; private String usualSurveyUnitId; diff --git a/src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseController.java b/src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseController.java index 1c2a9ae2..9ba6bc57 100644 --- a/src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseController.java +++ b/src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseController.java @@ -98,57 +98,6 @@ public ResponseEntity saveRawResponsesFromJsonBody( return ResponseEntity.status(201).body(String.format(SUCCESS_MESSAGE, interrogationId)); } -/* @Operation(summary = "Deprecated") - @PutMapping(path="/lunatic-json") - @PreAuthorize("hasRole('COLLECT_PLATFORM')") - // Check version when merging - @Deprecated(since="1.13.0", forRemoval=true) - public ResponseEntity saveRawResponsesFromJsonBodyWithValidationDeprecated( - @RequestBody Map body - ) { - - SchemaRegistry schemaRegistry = SchemaRegistry.withDialect(Dialects.getDraft202012(), SchemaRegistry.Builder::build); - Schema jsonSchema = schemaRegistry - .getSchema(RawResponseController.class.getResourceAsStream("/modele-filiere-spec/RawResponse.json") - ); - try { - if (jsonSchema == null) { - throw new GenesisException(500, "No RawResponse json schema has been found"); - } - List errors = jsonSchema.validate( - new ObjectMapper().readTree( - new ObjectMapper().writeValueAsString(body) - ) - ); - // Throw Genesis exception if errors are present - validate(errors); - //Check required ids - checkRequiredIds(body); - } catch (JsonProcessingException jpe) { - return ResponseEntity.status(400).body(jpe.toString()); - } catch (GenesisException ge) { - return ResponseEntity.status(ge.getStatus()).body(ge.getMessage()); - } - - LunaticJsonRawDataModel rawData = LunaticJsonRawDataModel.builder() - .campaignId(body.get(PARTITION_ID).toString()) - .questionnaireId(body.get("questionnaireModelId").toString().toUpperCase()) - .interrogationId(body.get(INTERROGATION_ID).toString()) - .idUE(body.get("surveyUnitId").toString()) - .mode(Mode.getEnumFromJsonName(body.get("mode").toString())) - .data(body) - .recordDate(LocalDateTime.now()) - .build(); - try { - lunaticJsonRawDataApiPort.save(rawData); - } catch (Exception e) { - return ResponseEntity.status(500).body("Unexpected error"); - } - - log.info("Data saved for interrogationId {} and partition {}", body.get(INTERROGATION_ID).toString(), - body.get(PARTITION_ID).toString()); - return ResponseEntity.status(201).body(String.format(SUCCESS_MESSAGE, body.get(INTERROGATION_ID).toString())); - }*/ @Operation(summary = "Save lunatic json data from one interrogation in Genesis Database (with json " + "schema validation)") diff --git a/src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java b/src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java index e8a75af0..1e332875 100644 --- a/src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java +++ b/src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java @@ -101,6 +101,7 @@ public ResponseController(SurveyUnitApiPort surveyUnitService, } //SAVE + @Deprecated(since = "2026-01-01") @Operation(summary = "Save one file of responses to Genesis Database, passing its path as a parameter") @PutMapping(path = "/lunatic-xml/save-one") @PreAuthorize("hasRole('ADMIN')") @@ -117,6 +118,7 @@ public ResponseEntity saveResponsesFromXmlFile(@RequestParam("pathLunati return processXmlFileSequentially(filepath, modeSpecified, metadataFilePath); } + @Deprecated(since = "2026-01-01") @Operation(summary = "Save multiple files to Genesis Database from the campaign root folder") @PutMapping(path = "/lunatic-xml/save-folder") @PreAuthorize("hasRole('ADMIN')") @@ -149,6 +151,7 @@ public ResponseEntity saveResponsesFromXmlCampaignFolder(@RequestParam(" } //SAVE ALL + @Deprecated(since = "2026-01-01") @Operation(summary = "Save all files to Genesis Database (differential data folder only), regardless of the campaign") @PutMapping(path = "/lunatic-xml/save-all-campaigns") @PreAuthorize("hasRole('SCHEDULER')") @@ -211,14 +214,13 @@ public ResponseEntity> findResponsesByInterrogationAndColl * @deprecated * This endpoint is deprecated because the parameter `questionnaireId` has been renamed * to `collectionInstrumentId` in the Information System (modeled in the modelefiliere library). - * * A new endpoint using the updated parameter names will be provided to remain compliant with * the current data model. This endpoint will be removed once all dependent APIs have adopted * the new naming convention. * * Use the new endpoint with `collectionInstrumentId` for future implementations. */ - @Deprecated(forRemoval = true) + @Deprecated(forRemoval = true, since= "2026-01-01") @Operation(summary = "Retrieve responses for an interrogation, using interrogationId and questionnaireId from Genesis Database with the latest value for each available state of every variable", description = "use /by-interrogation-and-collection-instrument/latest-states instead") @GetMapping(path = "/by-ue-and-questionnaire/latest-states", From ebfa9066e0fdb64f8c996644bf4b51405ba5ce9f Mon Sep 17 00:00:00 2001 From: Alice Lambois Date: Thu, 4 Dec 2025 16:46:14 +0100 Subject: [PATCH 27/42] refactor: change requestparam to pathvariable --- .../controller/rest/responses/ResponseController.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java b/src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java index 1e332875..474c2f42 100644 --- a/src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java +++ b/src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java @@ -43,6 +43,7 @@ import org.springframework.stereotype.Controller; import org.springframework.web.bind.annotation.DeleteMapping; import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.PutMapping; import org.springframework.web.bind.annotation.RequestBody; @@ -189,9 +190,9 @@ public ResponseEntity saveResponsesFromAllCampaignFolders(){ //DELETE @Operation(summary = "Delete all responses associated with a collection instrument (formerly questionnaire)") - @DeleteMapping(path = "/delete/by-collection-instrument") + @DeleteMapping(path = "/delete/{collectionInstrumentId}") @PreAuthorize("hasRole('ADMIN')") - public ResponseEntity deleteAllResponsesByCollectionInstrument(@RequestParam("collectionInstrumentId") String collectionInstrumentId) { + public ResponseEntity deleteAllResponsesByCollectionInstrument(@PathVariable("collectionInstrumentId") String collectionInstrumentId) { log.info("Try to delete all responses of collection instrument : {}", collectionInstrumentId); Long ndDocuments = surveyUnitService.deleteByCollectionInstrumentId(collectionInstrumentId); log.info("{} responses deleted", ndDocuments); From 4182e74d751b4bd27d4f6ed6175d08fb275e6d72 Mon Sep 17 00:00:00 2001 From: Alice Lambois Date: Thu, 4 Dec 2025 16:46:51 +0100 Subject: [PATCH 28/42] style : remove unused --- .../rest/responses/RawResponseController.java | 28 ------------------- 1 file changed, 28 deletions(-) diff --git a/src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseController.java b/src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseController.java index 9ba6bc57..55317671 100644 --- a/src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseController.java +++ b/src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseController.java @@ -46,14 +46,12 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; -import java.util.stream.Collectors; @Slf4j @Controller public class RawResponseController { private static final String SUCCESS_MESSAGE = "Interrogation %s saved"; - private static final String PARTITION_ID = "partitionId"; private static final String INTERROGATION_ID = "interrogationId"; private final LunaticJsonRawDataApiPort lunaticJsonRawDataApiPort; private final RawResponseApiPort rawResponseApiPort; @@ -238,31 +236,5 @@ public ResponseEntity> getRawResponsesFromJs return ResponseEntity.status(HttpStatus.OK).body(new PagedModel<>(rawResponses)); } - private void validate(List errors) throws GenesisException { - if (!errors.isEmpty()) { - String errorMessage = errors.stream() - .map(Error::getMessage) - .collect(Collectors.joining(System.lineSeparator() + " - ")); - - throw new GenesisException( - 400, - "Input data JSON is not valid: %n - %s".formatted(errorMessage) - ); - } - } - - private void checkRequiredIds(Map body) throws GenesisException { - for (String requiredKey : List.of( - PARTITION_ID, - "questionnaireModelId", - INTERROGATION_ID, - "surveyUnitId", - "mode" - )) { - if (body.get(requiredKey) == null) { - throw new GenesisException(400, "No %s found in body".formatted(requiredKey)); - } - } - } } From 358a3af0a2e2cb2be677a5a0d6757a6ebbeed2b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Henninger?= Date: Thu, 4 Dec 2025 17:10:21 +0100 Subject: [PATCH 29/42] feat: new endpoint to list collection instrument ids containing unprocessed data --- .../controller/rest/responses/RawResponseController.java | 7 +++++++ .../genesis/domain/ports/api/RawResponseApiPort.java | 1 + .../domain/ports/spi/RawResponsePersistencePort.java | 1 + .../domain/service/rawdata/RawResponseService.java | 7 ++++++- .../infrastructure/adapter/RawResponseMongoAdapter.java | 5 +++++ .../infrastructure/repository/RawResponseRepository.java | 7 +++++++ .../cucumber/functional_tests/RawDataDefinitions.java | 5 +++++ .../rest/responses/RawResponseControllerTest.java | 9 +++++++-- 8 files changed, 39 insertions(+), 3 deletions(-) diff --git a/src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseController.java b/src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseController.java index c93651fd..9f169e80 100644 --- a/src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseController.java +++ b/src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseController.java @@ -215,6 +215,13 @@ public ResponseEntity processRawResponses( return ResponseEntity.status(e.getStatus()).body(e.getMessage()); } } + @Operation(summary = "Get the list of collection instruments containing unprocessed interrogations") + @GetMapping(path = "/raw-responses/unprocessed/collection-intrument-ids") + @PreAuthorize("hasRole('SCHEDULER')") + public ResponseEntity> getUnprocessedCollectionInstrument(){ + log.info("Try to get collection instruments containing unprocessed interrogations..."); + return ResponseEntity.ok(rawResponseApiPort.getUnprocessedCollectionInstrumentIds()); + } //GET unprocessed @Operation(summary = "Get campaign id and interrogationId from all unprocessed raw json data") diff --git a/src/main/java/fr/insee/genesis/domain/ports/api/RawResponseApiPort.java b/src/main/java/fr/insee/genesis/domain/ports/api/RawResponseApiPort.java index e5d589c5..f29d0b77 100644 --- a/src/main/java/fr/insee/genesis/domain/ports/api/RawResponseApiPort.java +++ b/src/main/java/fr/insee/genesis/domain/ports/api/RawResponseApiPort.java @@ -15,5 +15,6 @@ public interface RawResponseApiPort { List getRawResponses(String collectionInstrumentId, Mode mode, List interrogationIdList); DataProcessResult processRawResponses(String collectionInstrumentId, List interrogationIdList, List errors) throws GenesisException; List convertRawResponse(List rawResponses, VariablesMap variablesMap); + List getUnprocessedCollectionInstrumentIds(); void updateProcessDates(List surveyUnitModels); } diff --git a/src/main/java/fr/insee/genesis/domain/ports/spi/RawResponsePersistencePort.java b/src/main/java/fr/insee/genesis/domain/ports/spi/RawResponsePersistencePort.java index 3d79e88c..0d7a7932 100644 --- a/src/main/java/fr/insee/genesis/domain/ports/spi/RawResponsePersistencePort.java +++ b/src/main/java/fr/insee/genesis/domain/ports/spi/RawResponsePersistencePort.java @@ -10,4 +10,5 @@ public interface RawResponsePersistencePort { List findRawResponses(String collectionInstrumentId, Mode mode, List interrogationIdList); void updateProcessDates(String collectionInstrumentId, Set interrogationIds); + List getUnprocessedCollectionIds(); } diff --git a/src/main/java/fr/insee/genesis/domain/service/rawdata/RawResponseService.java b/src/main/java/fr/insee/genesis/domain/service/rawdata/RawResponseService.java index bfa7e924..08e36f80 100644 --- a/src/main/java/fr/insee/genesis/domain/service/rawdata/RawResponseService.java +++ b/src/main/java/fr/insee/genesis/domain/service/rawdata/RawResponseService.java @@ -127,7 +127,7 @@ public DataProcessResult processRawResponses(String collectionInstrumentId, List batchNumber++; } } - return new DataProcessResult(dataCount, formattedDataCount); + return new DataProcessResult(dataCount, formattedDataCount, List.of()); } @Override @@ -182,6 +182,11 @@ public List convertRawResponse(List rawResponses, //return List.of(); } + @Override + public List getUnprocessedCollectionInstrumentIds() { + return rawResponsePersistencePort.getUnprocessedCollectionIds(); + } + @Override public void updateProcessDates(List surveyUnitModels) { Set collectionInstrumentIds = new HashSet<>(); diff --git a/src/main/java/fr/insee/genesis/infrastructure/adapter/RawResponseMongoAdapter.java b/src/main/java/fr/insee/genesis/infrastructure/adapter/RawResponseMongoAdapter.java index aad390b5..0293894e 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/adapter/RawResponseMongoAdapter.java +++ b/src/main/java/fr/insee/genesis/infrastructure/adapter/RawResponseMongoAdapter.java @@ -46,4 +46,9 @@ public void updateProcessDates(String collectionInstrumentId, Set interr , Constants.MONGODB_RAW_RESPONSES_COLLECTION_NAME ); } + + @Override + public List getUnprocessedCollectionIds() { + return repository.findDistinctCollectionInstrumentIdByProcessDateIsNull(); + } } diff --git a/src/main/java/fr/insee/genesis/infrastructure/repository/RawResponseRepository.java b/src/main/java/fr/insee/genesis/infrastructure/repository/RawResponseRepository.java index 40e23381..6f60c1b2 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/repository/RawResponseRepository.java +++ b/src/main/java/fr/insee/genesis/infrastructure/repository/RawResponseRepository.java @@ -1,6 +1,7 @@ package fr.insee.genesis.infrastructure.repository; import fr.insee.genesis.infrastructure.document.rawdata.RawResponseDocument; +import org.springframework.data.mongodb.repository.Aggregation; import org.springframework.data.mongodb.repository.MongoRepository; import org.springframework.data.mongodb.repository.Query; import org.springframework.stereotype.Repository; @@ -12,4 +13,10 @@ public interface RawResponseRepository extends MongoRepository findByCollectionInstrumentIdAndModeAndInterrogationIdList(String questionnaireId, String mode, List interrogationIdList); + @Aggregation(pipeline = { + "{ $match: { processDate: null } }", + "{ $group: { _id: '$collectionInstrumentId' } }", + "{ $project: { _id: 0, collectionInstrumentId: '$_id' } }" + }) + List findDistinctCollectionInstrumentIdByProcessDateIsNull(); } diff --git a/src/test/java/cucumber/functional_tests/RawDataDefinitions.java b/src/test/java/cucumber/functional_tests/RawDataDefinitions.java index 40aa92fa..b8717cc2 100644 --- a/src/test/java/cucumber/functional_tests/RawDataDefinitions.java +++ b/src/test/java/cucumber/functional_tests/RawDataDefinitions.java @@ -120,6 +120,11 @@ public List convertRawResponse(List rawResponses, return List.of(); } + @Override + public List getUnprocessedCollectionInstrumentIds() { + return List.of(); + } + @Override public void updateProcessDates(List surveyUnitModels) { // Do nothing - stub for test diff --git a/src/test/java/fr/insee/genesis/controller/rest/responses/RawResponseControllerTest.java b/src/test/java/fr/insee/genesis/controller/rest/responses/RawResponseControllerTest.java index c9eac1d5..a8feefb7 100644 --- a/src/test/java/fr/insee/genesis/controller/rest/responses/RawResponseControllerTest.java +++ b/src/test/java/fr/insee/genesis/controller/rest/responses/RawResponseControllerTest.java @@ -93,6 +93,11 @@ public List convertRawResponse(List rawResponses, return List.of(); } + @Override + public List getUnprocessedCollectionInstrumentIds() { + return List.of(); + } + @Override public void updateProcessDates(List surveyUnitModels) { @@ -277,10 +282,10 @@ void processJsonRawDataV2Test(){ Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub()).isNotNull().isNotEmpty().hasSize(1); Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getFirst()).isNotNull(); Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getFirst().getCampaignId()).isEqualTo(questionnaireId); - Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getFirst().getQuestionnaireId()).isNotNull().isEqualTo(questionnaireId); + Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getFirst().getCollectionInstrumentId()).isNotNull().isEqualTo(questionnaireId); Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getFirst().getMode()).isNotNull().isEqualTo(Mode.WEB); Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getFirst().getInterrogationId()).isEqualTo(interrogationId); - Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getFirst().getIdUE()).isEqualTo(idUE); + Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getFirst().getUsualSurveyUnitId()).isEqualTo(idUE); Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getFirst().getFileDate()).isNotNull(); Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getFirst().getRecordDate()).isNotNull(); Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub().getFirst().getCollectedVariables()).isNotNull().isNotEmpty().hasSize(1); From b752d4f4523d1cde32ad3de563f6883c1c2f5782 Mon Sep 17 00:00:00 2001 From: Alice Lambois Date: Fri, 5 Dec 2025 13:33:06 +0100 Subject: [PATCH 30/42] style: sonar --- .../utils/ExtendedJsonNormalizer.java | 22 +++++++++---------- .../controller/utils/JsonSchemaValidator.java | 8 +++---- 2 files changed, 14 insertions(+), 16 deletions(-) diff --git a/src/main/java/fr/insee/genesis/controller/utils/ExtendedJsonNormalizer.java b/src/main/java/fr/insee/genesis/controller/utils/ExtendedJsonNormalizer.java index 62bbd94d..6bb1c4bd 100644 --- a/src/main/java/fr/insee/genesis/controller/utils/ExtendedJsonNormalizer.java +++ b/src/main/java/fr/insee/genesis/controller/utils/ExtendedJsonNormalizer.java @@ -1,18 +1,18 @@ package fr.insee.genesis.controller.utils; import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.*; - -import java.util.Iterator; -import java.util.Map; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fasterxml.jackson.databind.node.TextNode; public class ExtendedJsonNormalizer { + public static final String $_DATE = "$date"; + private ExtendedJsonNormalizer() {} /** * Recursively converts Mongo Extended JSON objects into simple types expected by the schema: * - {"$date": "..."} -> TextNode("...") - * * Leaves all other values untouched. Returns a structural copy of the node (does not mutate the original). */ @@ -23,8 +23,8 @@ public static JsonNode normalize(JsonNode node) { ObjectNode obj = (ObjectNode) node; if (obj.size() == 1) { - if (obj.has("$date") && obj.get("$date").isTextual()) { - return TextNode.valueOf(obj.get("$date").asText()); + if (obj.has($_DATE) && obj.get($_DATE).isTextual()) { + return TextNode.valueOf(obj.get($_DATE).asText()); } // if (obj.has("$oid") && obj.get("$oid").isTextual()) { // return TextNode.valueOf(obj.get("$oid").asText()); @@ -32,11 +32,9 @@ public static JsonNode normalize(JsonNode node) { } ObjectNode copy = obj.objectNode(); - Iterator> it = obj.fields(); - while (it.hasNext()) { - Map.Entry e = it.next(); - copy.set(e.getKey(), normalize(e.getValue())); - } + obj.fields().forEachRemaining(e -> + copy.set(e.getKey(), normalize(e.getValue())) + ); return copy; } diff --git a/src/main/java/fr/insee/genesis/controller/utils/JsonSchemaValidator.java b/src/main/java/fr/insee/genesis/controller/utils/JsonSchemaValidator.java index 68c8df0a..9a033f49 100644 --- a/src/main/java/fr/insee/genesis/controller/utils/JsonSchemaValidator.java +++ b/src/main/java/fr/insee/genesis/controller/utils/JsonSchemaValidator.java @@ -42,23 +42,23 @@ public static JsonSchema loadSchemaFromClasspath(String resourcePath, ObjectMapp } String cp = resourcePath.startsWith("/") ? resourcePath.substring(1) : resourcePath; try (InputStream in = Thread.currentThread().getContextClassLoader().getResourceAsStream(cp)) { - if (in == null) throw new IOException("Schema not found on classpath: " + resourcePath); + if (in == null){ throw new IOException("Schema not found on classpath: " + resourcePath);} JsonNode schemaNode = mapper.readTree(in); return JsonSchemaFactory.getInstance(SpecVersion.VersionFlag.V202012).getSchema(schemaNode); } } private static void ensureValid(JsonNode root, JsonSchema schema) throws SchemaValidationException { - log.info("Schéma Apply : " + schema.getSchemaNode().get("title")); + log.info("Schéma Apply : {}", schema.getSchemaNode().get("title")); java.util.Set errors = schema.validate(root); if (!errors.isEmpty()) { String formatted = errors.stream() .sorted(java.util.Comparator.comparing(ValidationMessage::getEvaluationPath)) - .map(err -> err.getMessage()) + .map(ValidationMessage::getMessage) .collect(java.util.stream.Collectors.joining("\n")); throw new SchemaValidationException( "Uploaded JSON is not correct according to the json-schema:\n" + formatted, errors); } - log.info("Schema-compliant JSON : " + schema.getSchemaNode().get("title")); + log.info("Schema-compliant JSON : {}", schema.getSchemaNode().get("title")); } } From fd64a601aaf75db3f344dc8ef819a0f45230ed63 Mon Sep 17 00:00:00 2001 From: Alice Lambois Date: Fri, 5 Dec 2025 13:53:33 +0100 Subject: [PATCH 31/42] style: change for lookup --- .../genesis/domain/model/surveyunit/Mode.java | 72 ++++++++++--------- 1 file changed, 38 insertions(+), 34 deletions(-) diff --git a/src/main/java/fr/insee/genesis/domain/model/surveyunit/Mode.java b/src/main/java/fr/insee/genesis/domain/model/surveyunit/Mode.java index ad25a45b..b3ce4303 100644 --- a/src/main/java/fr/insee/genesis/domain/model/surveyunit/Mode.java +++ b/src/main/java/fr/insee/genesis/domain/model/surveyunit/Mode.java @@ -5,6 +5,9 @@ import lombok.Getter; import org.springframework.lang.Nullable; +import java.util.HashMap; +import java.util.Map; + @Getter public enum Mode { @@ -28,41 +31,42 @@ public enum Mode { this.jsonName = jsonName; } - @JsonCreator - public static Mode fromString(String value) { - if (value == null) return null; - - for (Mode m : values()) { - if (value.equalsIgnoreCase(m.modeName) || - value.equalsIgnoreCase(m.jsonName)) { - return m; - } - } - throw new IllegalArgumentException("Invalid Mode: " + value); - } + // lookup Maps + private static final Map BY_MODE_NAME = new HashMap<>(); + private static final Map BY_JSON_NAME = new HashMap<>(); + private static final Map BY_ANY_NAME = new HashMap<>(); - public static Mode getEnumFromModeName(String modeName) { - if (modeName == null){ - return null; - } - for (Mode mode : Mode.values()) { - if (modeName.equals(mode.getModeName())) { - return mode; - } - } - return null; - } + static { + for (Mode mode : values()) { + if (mode.modeName != null && !mode.modeName.isBlank()) { + BY_MODE_NAME.put(mode.modeName.toUpperCase(), mode); + BY_ANY_NAME.put(mode.modeName.toUpperCase(), mode); + } + if (mode.jsonName != null && !mode.jsonName.isBlank()) { + BY_JSON_NAME.put(mode.jsonName.toUpperCase(), mode); + BY_ANY_NAME.put(mode.jsonName.toUpperCase(), mode); + } + } + } - public static Mode getEnumFromJsonName(String modeName) { - if (modeName == null){ - return null; - } - for (Mode mode : Mode.values()) { - if (modeName.equals(mode.getJsonName())) { - return mode; - } - } - return null; - } + @JsonCreator + public static Mode fromString(String value) { + if (value == null){ return null;} + + Mode mode = BY_ANY_NAME.get(value.trim().toUpperCase()); + if (mode != null) { return mode; } + + throw new IllegalArgumentException("Invalid Mode: " + value); + } + + public static Mode getEnumFromModeName(@Nullable String modeName) { + if (modeName == null) { return null; } + return BY_MODE_NAME.get(modeName.trim().toUpperCase()); + } + + public static Mode getEnumFromJsonName(@Nullable String jsonName) { + if (jsonName == null) { return null; } + return BY_JSON_NAME.get(jsonName.trim().toUpperCase()); + } } From 6571774dd52bc99fa49c1c6394972a48321948a7 Mon Sep 17 00:00:00 2001 From: Alice Lambois Date: Fri, 5 Dec 2025 14:29:19 +0100 Subject: [PATCH 32/42] style : add deprecated since --- .../model/surveyunit/SurveyUnitModel.java | 2 +- .../service/rawdata/RawResponseService.java | 24 +++++++++---------- .../ContextualExternalVariableDocument.java | 2 +- .../ContextualPreviousVariableDocument.java | 2 +- .../lunaticmodel/LunaticModelDocument.java | 3 +-- .../QuestionnaireMetadataDocument.java | 2 +- .../surveyunit/SurveyUnitDocument.java | 6 ++--- 7 files changed, 19 insertions(+), 22 deletions(-) diff --git a/src/main/java/fr/insee/genesis/domain/model/surveyunit/SurveyUnitModel.java b/src/main/java/fr/insee/genesis/domain/model/surveyunit/SurveyUnitModel.java index 94f0e1d2..212adb42 100644 --- a/src/main/java/fr/insee/genesis/domain/model/surveyunit/SurveyUnitModel.java +++ b/src/main/java/fr/insee/genesis/domain/model/surveyunit/SurveyUnitModel.java @@ -27,7 +27,7 @@ public class SurveyUnitModel { /** * @deprecated We will not receive this identifier anymore */ - @Deprecated(forRemoval = true) + @Deprecated(forRemoval = true, since = "2026-01-01") private String campaignId; private String interrogationId; // New name of idUE diff --git a/src/main/java/fr/insee/genesis/domain/service/rawdata/RawResponseService.java b/src/main/java/fr/insee/genesis/domain/service/rawdata/RawResponseService.java index 08e36f80..e01c3e0b 100644 --- a/src/main/java/fr/insee/genesis/domain/service/rawdata/RawResponseService.java +++ b/src/main/java/fr/insee/genesis/domain/service/rawdata/RawResponseService.java @@ -36,6 +36,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.stream.Collectors; @Service @Slf4j @@ -179,7 +180,6 @@ public List convertRawResponse(List rawResponses, } } return surveyUnitModels; - //return List.of(); } @Override @@ -195,13 +195,10 @@ public void updateProcessDates(List surveyUnitModels) { } for (String collectionInstrumentId : collectionInstrumentIds) { - Set interrogationIds = new HashSet<>(); - for (SurveyUnitModel surveyUnitModel : - surveyUnitModels.stream().filter( - surveyUnitModel -> surveyUnitModel.getCollectionInstrumentId().equals(collectionInstrumentId) - ).toList()) { - interrogationIds.add(surveyUnitModel.getInterrogationId()); - } + Set interrogationIds = surveyUnitModels.stream() + .filter(su -> su.getCollectionInstrumentId().equals(collectionInstrumentId)) + .map(SurveyUnitModel::getInterrogationId) + .collect(Collectors.toSet()); rawResponsePersistencePort.updateProcessDates(collectionInstrumentId, interrogationIds); } } @@ -218,11 +215,12 @@ private Map> getProcessedIdsMap(List survey private void sendProcessedIdsToQualityTool(List surveyUnitModels) { try { + Map> processedIdsMap = getProcessedIdsMap(surveyUnitModels); ResponseEntity response = - surveyUnitQualityToolPort.sendProcessedIds(getProcessedIdsMap(surveyUnitModels)); + surveyUnitQualityToolPort.sendProcessedIds(processedIdsMap); if (response.getStatusCode().is2xxSuccessful()) { - log.info("Successfully sent {} ids to quality tool", getProcessedIdsMap(surveyUnitModels).size()); + log.info("Successfully sent {} ids to quality tool", processedIdsMap.size()); }else{ log.warn("Survey unit quality tool responded non-2xx code {} and body {}", response.getStatusCode(), response.getBody()); @@ -237,7 +235,7 @@ private static Boolean getIsCapturedIndirectly(RawResponse rawResponse) { return rawResponse.payload().get("isCapturedIndirectly") == null ? null : Boolean.parseBoolean(rawResponse.payload().get("isCapturedIndirectly").toString()); }catch(Exception e){ - log.warn("Exception when parsing isCapturedIndirectly : {}}",e.toString()); + log.warn("Exception when parsing isCapturedIndirectly : {}",e.toString()); return Boolean.FALSE; } } @@ -247,7 +245,7 @@ private static LocalDateTime getValidationDate(RawResponse rawResponse) { return rawResponse.payload().get("validationDate") == null ? null : LocalDateTime.parse(rawResponse.payload().get("validationDate").toString()); }catch(Exception e){ - log.warn("Exception when parsing validation date : {}}",e.toString()); + log.warn("Exception when parsing validation date : {}",e.toString()); return null; } } @@ -340,7 +338,7 @@ private static void convertOneVar(Map.Entry externalVariableEntr private static String getIdLoop(VariablesMap variablesMap, String variableName) { if (variablesMap.getVariable(variableName) == null) { - log.warn("Variable {} not present in metadatas, assigning to {}", variableName, Constants.ROOT_GROUP_NAME); + log.warn("Variable {} not present in metadata, assigning to {}", variableName, Constants.ROOT_GROUP_NAME); return Constants.ROOT_GROUP_NAME; } return variablesMap.getVariable(variableName).getGroupName(); diff --git a/src/main/java/fr/insee/genesis/infrastructure/document/contextualexternal/ContextualExternalVariableDocument.java b/src/main/java/fr/insee/genesis/infrastructure/document/contextualexternal/ContextualExternalVariableDocument.java index 288e6bf7..4518ead6 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/document/contextualexternal/ContextualExternalVariableDocument.java +++ b/src/main/java/fr/insee/genesis/infrastructure/document/contextualexternal/ContextualExternalVariableDocument.java @@ -22,7 +22,7 @@ public class ContextualExternalVariableDocument { /** * @deprecated it will be replaced by collectionInstrumentId */ - @Deprecated(forRemoval = true) + @Deprecated(forRemoval = true, since = "2026-01-01") @Indexed String questionnaireId; @Indexed diff --git a/src/main/java/fr/insee/genesis/infrastructure/document/contextualprevious/ContextualPreviousVariableDocument.java b/src/main/java/fr/insee/genesis/infrastructure/document/contextualprevious/ContextualPreviousVariableDocument.java index f258e76f..63b924ac 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/document/contextualprevious/ContextualPreviousVariableDocument.java +++ b/src/main/java/fr/insee/genesis/infrastructure/document/contextualprevious/ContextualPreviousVariableDocument.java @@ -21,7 +21,7 @@ public class ContextualPreviousVariableDocument { /** * @deprecated it will be replaced by collectionInstrumentId */ - @Deprecated(forRemoval = true) + @Deprecated(forRemoval = true, since = "2026-01-01") @Indexed String questionnaireId; @Indexed diff --git a/src/main/java/fr/insee/genesis/infrastructure/document/lunaticmodel/LunaticModelDocument.java b/src/main/java/fr/insee/genesis/infrastructure/document/lunaticmodel/LunaticModelDocument.java index bbc34a1d..b884d245 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/document/lunaticmodel/LunaticModelDocument.java +++ b/src/main/java/fr/insee/genesis/infrastructure/document/lunaticmodel/LunaticModelDocument.java @@ -16,8 +16,7 @@ I choose to keep the record here (this choice can be challenged)*/ @Builder @Document(collection = "lunaticmodels") public record LunaticModelDocument ( - - @Deprecated + @Deprecated (since = "2026-01-01", forRemoval = true) @Indexed String questionnaireId, @Indexed diff --git a/src/main/java/fr/insee/genesis/infrastructure/document/metadata/QuestionnaireMetadataDocument.java b/src/main/java/fr/insee/genesis/infrastructure/document/metadata/QuestionnaireMetadataDocument.java index af045d67..8be83a6c 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/document/metadata/QuestionnaireMetadataDocument.java +++ b/src/main/java/fr/insee/genesis/infrastructure/document/metadata/QuestionnaireMetadataDocument.java @@ -17,7 +17,7 @@ I choose to keep the record here (this choice can be challenged)*/ @Document(collection = "questionnaireMetadatas") public record QuestionnaireMetadataDocument( - @Deprecated + @Deprecated (since = "2026-01-01", forRemoval = true) @Indexed String questionnaireId, @Indexed diff --git a/src/main/java/fr/insee/genesis/infrastructure/document/surveyunit/SurveyUnitDocument.java b/src/main/java/fr/insee/genesis/infrastructure/document/surveyunit/SurveyUnitDocument.java index 95503c7f..9b1e8ac5 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/document/surveyunit/SurveyUnitDocument.java +++ b/src/main/java/fr/insee/genesis/infrastructure/document/surveyunit/SurveyUnitDocument.java @@ -24,7 +24,7 @@ public class SurveyUnitDocument { /** * @deprecated This piece of information will not be available anymore in the raw responses */ - @Deprecated(forRemoval = true) + @Deprecated(forRemoval = true, since ="2026-01-01") private String campaignId; @Indexed private String interrogationId; @@ -32,7 +32,7 @@ public class SurveyUnitDocument { /** * @deprecated It will be replaced by usualSurveyUnitId */ - @Deprecated(forRemoval = true) + @Deprecated(forRemoval = true, since ="2026-01-01") private String idUE; private String usualSurveyUnitId; @@ -40,7 +40,7 @@ public class SurveyUnitDocument { /** * @deprecated It will be replaced by collectionInstrumentId */ - @Deprecated(forRemoval = true) + @Deprecated(forRemoval = true, since ="2026-01-01") private String questionnaireId; private String collectionInstrumentId; From 00d942a17e0c4418042a7df9d5f3cfae1be69216 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Henninger?= Date: Fri, 5 Dec 2025 15:07:27 +0100 Subject: [PATCH 33/42] feat: add endpoints necessary for automized processing --- .../rest/responses/RawResponseController.java | 30 ++++++- .../domain/ports/api/RawResponseApiPort.java | 2 + .../ports/spi/RawResponsePersistencePort.java | 3 +- .../service/rawdata/RawResponseService.java | 89 ++++++++++++++++--- .../adapter/RawResponseMongoAdapter.java | 7 ++ .../repository/RawResponseRepository.java | 7 ++ .../functional_tests/RawDataDefinitions.java | 5 ++ .../responses/RawResponseControllerTest.java | 5 ++ 8 files changed, 133 insertions(+), 15 deletions(-) diff --git a/src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseController.java b/src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseController.java index 9f169e80..5879da89 100644 --- a/src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseController.java +++ b/src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseController.java @@ -191,7 +191,7 @@ public ResponseEntity saveRawResponsesFromRawResponseDto( } //PROCESS - @Operation(summary = "Process raw data of a campaign") + @Operation(summary = "Process raw data for a list of interrogations") @PostMapping(path = "/raw-responses/process") @PreAuthorize("hasRole('SCHEDULER')") public ResponseEntity processRawResponses( @@ -202,9 +202,8 @@ public ResponseEntity processRawResponses( @RequestParam("collectionInstrumentId") String collectionInstrumentId, @RequestBody List interrogationIdList ) { - log.info("Try to process raw responses for questionnaireId {} and {} interrogationIds", collectionInstrumentId, interrogationIdList.size()); + log.info("Try to process raw responses for collectionInstrumentId {} and {} interrogationIds", collectionInstrumentId, interrogationIdList.size()); List errors = new ArrayList<>(); - try { DataProcessResult result = rawResponseApiPort.processRawResponses(collectionInstrumentId, interrogationIdList, errors); return result.formattedDataCount() == 0 ? @@ -215,6 +214,29 @@ public ResponseEntity processRawResponses( return ResponseEntity.status(e.getStatus()).body(e.getMessage()); } } + + @Operation(summary = "Process raw data for all data of an collection instrument") + @PostMapping(path = "/raw-responses/{collectionInstrumentId}/process") + @PreAuthorize("hasRole('SCHEDULER')") + public ResponseEntity processRawResponsesByCollectionInstrumentId( + @Parameter( + description = "Id of the collection instrument (old questionnaireId)", + example = "ENQTEST2025X00" + ) + @PathVariable("collectionInstrumentId") String collectionInstrumentId + ) { + log.info("Try to process raw responses for collectionInstrumentId {}", collectionInstrumentId); + try { + DataProcessResult result = rawResponseApiPort.processRawResponses(collectionInstrumentId); + return result.formattedDataCount() == 0 ? + ResponseEntity.ok("%d document(s) processed".formatted(result.dataCount())) + : ResponseEntity.ok("%d document(s) processed, including %d FORMATTED after data verification" + .formatted(result.dataCount(), result.formattedDataCount())); + } catch (GenesisException e) { + return ResponseEntity.status(e.getStatus()).body(e.getMessage()); + } + } + @Operation(summary = "Get the list of collection instruments containing unprocessed interrogations") @GetMapping(path = "/raw-responses/unprocessed/collection-intrument-ids") @PreAuthorize("hasRole('SCHEDULER')") @@ -269,7 +291,7 @@ public ResponseEntity processJsonRawData( } @Operation(summary = "Process raw data of a questionnaire") - @PostMapping(path = "/{collectionInstrumentId}/process") + @PostMapping(path = "/responses/raw/lunatic-json/{collectionInstrumentId}/process") @PreAuthorize("hasRole('SCHEDULER')") public ResponseEntity processJsonRawData( @PathVariable String collectionInstrumentId diff --git a/src/main/java/fr/insee/genesis/domain/ports/api/RawResponseApiPort.java b/src/main/java/fr/insee/genesis/domain/ports/api/RawResponseApiPort.java index f29d0b77..d17a1141 100644 --- a/src/main/java/fr/insee/genesis/domain/ports/api/RawResponseApiPort.java +++ b/src/main/java/fr/insee/genesis/domain/ports/api/RawResponseApiPort.java @@ -14,7 +14,9 @@ public interface RawResponseApiPort { List getRawResponses(String collectionInstrumentId, Mode mode, List interrogationIdList); DataProcessResult processRawResponses(String collectionInstrumentId, List interrogationIdList, List errors) throws GenesisException; + DataProcessResult processRawResponses(String collectionInstrumentId) throws GenesisException; List convertRawResponse(List rawResponses, VariablesMap variablesMap); List getUnprocessedCollectionInstrumentIds(); void updateProcessDates(List surveyUnitModels); + } diff --git a/src/main/java/fr/insee/genesis/domain/ports/spi/RawResponsePersistencePort.java b/src/main/java/fr/insee/genesis/domain/ports/spi/RawResponsePersistencePort.java index 0d7a7932..640bd9b3 100644 --- a/src/main/java/fr/insee/genesis/domain/ports/spi/RawResponsePersistencePort.java +++ b/src/main/java/fr/insee/genesis/domain/ports/spi/RawResponsePersistencePort.java @@ -11,4 +11,5 @@ public interface RawResponsePersistencePort { List findRawResponses(String collectionInstrumentId, Mode mode, List interrogationIdList); void updateProcessDates(String collectionInstrumentId, Set interrogationIds); List getUnprocessedCollectionIds(); -} + Set findUnprocessedInterrogationIdsByCollectionInstrumentId(String collectionInstrumentId); + } diff --git a/src/main/java/fr/insee/genesis/domain/service/rawdata/RawResponseService.java b/src/main/java/fr/insee/genesis/domain/service/rawdata/RawResponseService.java index 08e36f80..6da56aca 100644 --- a/src/main/java/fr/insee/genesis/domain/service/rawdata/RawResponseService.java +++ b/src/main/java/fr/insee/genesis/domain/service/rawdata/RawResponseService.java @@ -79,14 +79,7 @@ public DataProcessResult processRawResponses(String collectionInstrumentId, List List modesList = controllerUtils.getModesList(collectionInstrumentId, null); for (Mode mode : modesList) { //Load and save metadata into database, throw exception if none - VariablesMap variablesMap = metadataService.loadAndSaveIfNotExists(collectionInstrumentId, collectionInstrumentId, mode, fileUtils, - errors).getVariables(); - if (variablesMap == null) { - throw new GenesisException(400, - "Error during metadata parsing for mode %s :%n%s" - .formatted(mode, errors.getLast().getMessage()) - ); - } + VariablesMap variablesMap = getVariablesMap(collectionInstrumentId,mode,errors); int totalBatchs = Math.ceilDiv(interrogationIdList.size() , config.getRawDataProcessingBatchSize()); int batchNumber = 1; List interrogationIdListForMode = new ArrayList<>(interrogationIdList); @@ -127,7 +120,84 @@ public DataProcessResult processRawResponses(String collectionInstrumentId, List batchNumber++; } } - return new DataProcessResult(dataCount, formattedDataCount, List.of()); + return new DataProcessResult(dataCount, formattedDataCount, errors); + } + + @Override + public DataProcessResult processRawResponses(String collectionInstrumentId) throws GenesisException { + int dataCount=0; + int formattedDataCount=0; + DataProcessingContextModel dataProcessingContext = + dataProcessingContextService.getContextByCollectionInstrumentId(collectionInstrumentId); + List errors = new ArrayList<>(); + + List modesList = controllerUtils.getModesList(collectionInstrumentId, null); + for (Mode mode : modesList) { + //Load and save metadata into database, throw exception if none + VariablesMap variablesMap = getVariablesMap(collectionInstrumentId,mode,errors); + Set interrogationIds = + rawResponsePersistencePort.findUnprocessedInterrogationIdsByCollectionInstrumentId(collectionInstrumentId); + + int totalBatchs = Math.ceilDiv(interrogationIds.size() , config.getRawDataProcessingBatchSize()); + int batchNumber = 1; + List interrogationIdListForMode = new ArrayList<>(interrogationIds); + while(!interrogationIdListForMode.isEmpty()){ + log.info("Processing raw data batch {}/{}", batchNumber, totalBatchs); + int maxIndex = Math.min(interrogationIdListForMode.size(), config.getRawDataProcessingBatchSize()); + + List surveyUnitModels = getConvertedSurveyUnits( + collectionInstrumentId, + mode, + interrogationIdListForMode, + maxIndex, + variablesMap); + + //Save converted data + surveyUnitQualityService.verifySurveyUnits(surveyUnitModels, variablesMap); + surveyUnitService.saveSurveyUnits(surveyUnitModels); + + //Update process dates + updateProcessDates(surveyUnitModels); + + //Increment data count + dataCount += surveyUnitModels.size(); + formattedDataCount += surveyUnitModels.stream() + .filter(surveyUnitModel -> surveyUnitModel.getState().equals(DataState.FORMATTED)) + .toList() + .size(); + + //Send processed ids grouped by questionnaire (if review activated) + if(dataProcessingContext != null && dataProcessingContext.isWithReview()) { + sendProcessedIdsToQualityTool(surveyUnitModels); + } + + //Remove processed ids from list + interrogationIdListForMode = interrogationIdListForMode.subList(maxIndex, interrogationIdListForMode.size()); + batchNumber++; + } + } + return new DataProcessResult(dataCount, formattedDataCount, errors); + } + + private List getConvertedSurveyUnits(String collectionInstrumentId, Mode mode, List interrogationIdListForMode, int maxIndex, VariablesMap variablesMap) { + List interrogationIdToProcess = interrogationIdListForMode.subList(0, maxIndex); + List rawResponses = getRawResponses(collectionInstrumentId, mode, interrogationIdToProcess); + return convertRawResponse( + rawResponses, + variablesMap + ); + } + + private VariablesMap getVariablesMap(String collectionInstrumentId, Mode mode, List errors) throws GenesisException { + VariablesMap variablesMap = metadataService.loadAndSaveIfNotExists(collectionInstrumentId, collectionInstrumentId, mode, fileUtils, + errors).getVariables(); + if (variablesMap == null) { + throw new GenesisException(400, + "Error during metadata parsing for mode %s :%n%s" + .formatted(mode, errors.getLast().getMessage()) + ); + } + return variablesMap; } @Override @@ -179,7 +249,6 @@ public List convertRawResponse(List rawResponses, } } return surveyUnitModels; - //return List.of(); } @Override diff --git a/src/main/java/fr/insee/genesis/infrastructure/adapter/RawResponseMongoAdapter.java b/src/main/java/fr/insee/genesis/infrastructure/adapter/RawResponseMongoAdapter.java index 0293894e..1c529267 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/adapter/RawResponseMongoAdapter.java +++ b/src/main/java/fr/insee/genesis/infrastructure/adapter/RawResponseMongoAdapter.java @@ -16,6 +16,7 @@ import org.springframework.stereotype.Service; import java.time.LocalDateTime; +import java.util.HashSet; import java.util.List; import java.util.Set; @@ -51,4 +52,10 @@ public void updateProcessDates(String collectionInstrumentId, Set interr public List getUnprocessedCollectionIds() { return repository.findDistinctCollectionInstrumentIdByProcessDateIsNull(); } + + @Override + public Set findUnprocessedInterrogationIdsByCollectionInstrumentId(String collectionInstrumentId) { + // We remove duplicate ids + return new HashSet<>(repository.findInterrogationIdByCollectionInstrumentId(collectionInstrumentId)); + } } diff --git a/src/main/java/fr/insee/genesis/infrastructure/repository/RawResponseRepository.java b/src/main/java/fr/insee/genesis/infrastructure/repository/RawResponseRepository.java index 6f60c1b2..f58865f9 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/repository/RawResponseRepository.java +++ b/src/main/java/fr/insee/genesis/infrastructure/repository/RawResponseRepository.java @@ -19,4 +19,11 @@ public interface RawResponseRepository extends MongoRepository findDistinctCollectionInstrumentIdByProcessDateIsNull(); + + @Aggregation(pipeline = { + "{ $match: { collectionInstrumentId: ?0 } }", + "{ $project: { _id: 0, interrogationId: '$interrogationId' } }" + }) + List findInterrogationIdByCollectionInstrumentId(String collectionInstrumentId); + } diff --git a/src/test/java/cucumber/functional_tests/RawDataDefinitions.java b/src/test/java/cucumber/functional_tests/RawDataDefinitions.java index b8717cc2..c2005dba 100644 --- a/src/test/java/cucumber/functional_tests/RawDataDefinitions.java +++ b/src/test/java/cucumber/functional_tests/RawDataDefinitions.java @@ -115,6 +115,11 @@ public DataProcessResult processRawResponses(String questionnaireId, List convertRawResponse(List rawResponses, VariablesMap variablesMap) { return List.of(); diff --git a/src/test/java/fr/insee/genesis/controller/rest/responses/RawResponseControllerTest.java b/src/test/java/fr/insee/genesis/controller/rest/responses/RawResponseControllerTest.java index a8feefb7..13cd2237 100644 --- a/src/test/java/fr/insee/genesis/controller/rest/responses/RawResponseControllerTest.java +++ b/src/test/java/fr/insee/genesis/controller/rest/responses/RawResponseControllerTest.java @@ -88,6 +88,11 @@ public DataProcessResult processRawResponses(String questionnaireId, List convertRawResponse(List rawResponses, VariablesMap variablesMap) { return List.of(); From 81136c962d3f2b0d8164d7e0345608d649724cf6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Henninger?= Date: Fri, 5 Dec 2025 15:39:17 +0100 Subject: [PATCH 34/42] fix: filter on process date the raw responses tthat needs to be processed --- .../infrastructure/adapter/RawResponseMongoAdapter.java | 2 +- .../infrastructure/repository/RawResponseRepository.java | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/fr/insee/genesis/infrastructure/adapter/RawResponseMongoAdapter.java b/src/main/java/fr/insee/genesis/infrastructure/adapter/RawResponseMongoAdapter.java index 1c529267..77f2d29d 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/adapter/RawResponseMongoAdapter.java +++ b/src/main/java/fr/insee/genesis/infrastructure/adapter/RawResponseMongoAdapter.java @@ -56,6 +56,6 @@ public List getUnprocessedCollectionIds() { @Override public Set findUnprocessedInterrogationIdsByCollectionInstrumentId(String collectionInstrumentId) { // We remove duplicate ids - return new HashSet<>(repository.findInterrogationIdByCollectionInstrumentId(collectionInstrumentId)); + return new HashSet<>(repository.findInterrogationIdByCollectionInstrumentIdAndProcessDateIsNull(collectionInstrumentId)); } } diff --git a/src/main/java/fr/insee/genesis/infrastructure/repository/RawResponseRepository.java b/src/main/java/fr/insee/genesis/infrastructure/repository/RawResponseRepository.java index f58865f9..e09e9fd7 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/repository/RawResponseRepository.java +++ b/src/main/java/fr/insee/genesis/infrastructure/repository/RawResponseRepository.java @@ -21,9 +21,9 @@ public interface RawResponseRepository extends MongoRepository findDistinctCollectionInstrumentIdByProcessDateIsNull(); @Aggregation(pipeline = { - "{ $match: { collectionInstrumentId: ?0 } }", + "{ $match: { collectionInstrumentId: ?0,processDate: null } }", "{ $project: { _id: 0, interrogationId: '$interrogationId' } }" }) - List findInterrogationIdByCollectionInstrumentId(String collectionInstrumentId); + List findInterrogationIdByCollectionInstrumentIdAndProcessDateIsNull(String collectionInstrumentId); } From 70aac0c73664c072331c72b5497f323e9d447183 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Henninger?= Date: Fri, 5 Dec 2025 16:15:37 +0100 Subject: [PATCH 35/42] fix: retrieve context on new document --- .../service/context/DataProcessingContextService.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/main/java/fr/insee/genesis/domain/service/context/DataProcessingContextService.java b/src/main/java/fr/insee/genesis/domain/service/context/DataProcessingContextService.java index 245597d5..8f3b3ade 100644 --- a/src/main/java/fr/insee/genesis/domain/service/context/DataProcessingContextService.java +++ b/src/main/java/fr/insee/genesis/domain/service/context/DataProcessingContextService.java @@ -149,9 +149,13 @@ public DataProcessingContextModel getContext(String interrogationId) throws Gene throw new GenesisException(404,"No interrogation in database with id %s".formatted(interrogationId)); } Set partitionIds = new HashSet<>(); - surveyUnitModels.forEach( - surveyUnitModel -> partitionIds.add(surveyUnitModel.getCampaignId()) - ); + for (SurveyUnitModel su : surveyUnitModels){ + if (su.getCampaignId()!=null){ + partitionIds.add(su.getCampaignId()); + break; + } + partitionIds.add(su.getCollectionInstrumentId()); + } if(partitionIds.isEmpty()){ return null; } From fe12378f3cf0678944e96a54058165c7d07d255c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Henninger?= Date: Fri, 5 Dec 2025 16:51:50 +0100 Subject: [PATCH 36/42] fix: get context using either collectionInstrumentId or campaignId --- .../context/DataProcessingContextService.java | 37 ++++++++++++------- .../DataProcessingContextServiceTest.java | 2 +- 2 files changed, 25 insertions(+), 14 deletions(-) diff --git a/src/main/java/fr/insee/genesis/domain/service/context/DataProcessingContextService.java b/src/main/java/fr/insee/genesis/domain/service/context/DataProcessingContextService.java index 8f3b3ade..f78afd8f 100644 --- a/src/main/java/fr/insee/genesis/domain/service/context/DataProcessingContextService.java +++ b/src/main/java/fr/insee/genesis/domain/service/context/DataProcessingContextService.java @@ -18,7 +18,6 @@ import java.time.LocalDateTime; import java.util.ArrayList; -import java.util.Arrays; import java.util.HashSet; import java.util.List; import java.util.Set; @@ -149,26 +148,38 @@ public DataProcessingContextModel getContext(String interrogationId) throws Gene throw new GenesisException(404,"No interrogation in database with id %s".formatted(interrogationId)); } Set partitionIds = new HashSet<>(); + Set campaignIds = new HashSet<>(); + Set collectionInstrumentIds = new HashSet<>(); + for (SurveyUnitModel su : surveyUnitModels){ if (su.getCampaignId()!=null){ - partitionIds.add(su.getCampaignId()); - break; + campaignIds.add(su.getCampaignId()); + } + if (su.getCollectionInstrumentId()!=null){ + collectionInstrumentIds.add(su.getCampaignId()); } - partitionIds.add(su.getCollectionInstrumentId()); } - if(partitionIds.isEmpty()){ + if(campaignIds.size() > 1 || collectionInstrumentIds.size()>1){ + throw new GenesisException(500,"Multiple partitions for interrogation %s".formatted(interrogationId)); + } + + if(campaignIds.isEmpty() && collectionInstrumentIds.isEmpty()){ return null; } - if(partitionIds.size() > 1){ - throw new GenesisException(500,"Multiple partitions for interrogation %s %n%s".formatted( - interrogationId, - Arrays.toString(partitionIds.toArray()) - )); + + DataProcessingContextModel contextModel = new DataProcessingContextModel(); + if (!collectionInstrumentIds.isEmpty()){ + contextModel = DataProcessingContextMapper.INSTANCE.documentToModel( + dataProcessingContextPersistancePort.findByPartitionId(collectionInstrumentIds.stream().toList().getFirst()) + ); } - return DataProcessingContextMapper.INSTANCE.documentToModel( - dataProcessingContextPersistancePort.findByPartitionId(partitionIds.stream().toList().getFirst()) - ); + if (contextModel.getId()==null && !campaignIds.isEmpty()) { + contextModel = DataProcessingContextMapper.INSTANCE.documentToModel( + dataProcessingContextPersistancePort.findByPartitionId(campaignIds.stream().toList().getFirst())); + } + + return contextModel; } @Override diff --git a/src/test/java/fr/insee/genesis/domain/service/context/DataProcessingContextServiceTest.java b/src/test/java/fr/insee/genesis/domain/service/context/DataProcessingContextServiceTest.java index 6b57c18c..691171e9 100644 --- a/src/test/java/fr/insee/genesis/domain/service/context/DataProcessingContextServiceTest.java +++ b/src/test/java/fr/insee/genesis/domain/service/context/DataProcessingContextServiceTest.java @@ -209,7 +209,7 @@ void getContext_shouldThrow500IfMultiplePartitions() { //To ensure test is portable on Unix/Linux/macOS and windows systems String normalizedMessage = ex.getMessage().replaceAll("\\r?\\n", ""); Assertions.assertThat(ex.getStatus()).isEqualTo(500); - Assertions.assertThat(normalizedMessage).isEqualTo("Multiple partitions for interrogation 00001 [CAMPAIGN2, CAMPAIGN1]"); + Assertions.assertThat(normalizedMessage).isEqualTo("Multiple partitions for interrogation 00001"); } @Test From 99c29778b07e52379670ea2f9c427047cb155709 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Henninger?= Date: Fri, 5 Dec 2025 17:06:57 +0100 Subject: [PATCH 37/42] fix: typo --- .../domain/service/context/DataProcessingContextService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/fr/insee/genesis/domain/service/context/DataProcessingContextService.java b/src/main/java/fr/insee/genesis/domain/service/context/DataProcessingContextService.java index f78afd8f..380b66f3 100644 --- a/src/main/java/fr/insee/genesis/domain/service/context/DataProcessingContextService.java +++ b/src/main/java/fr/insee/genesis/domain/service/context/DataProcessingContextService.java @@ -156,7 +156,7 @@ public DataProcessingContextModel getContext(String interrogationId) throws Gene campaignIds.add(su.getCampaignId()); } if (su.getCollectionInstrumentId()!=null){ - collectionInstrumentIds.add(su.getCampaignId()); + collectionInstrumentIds.add(su.getCollectionInstrumentId()); } } if(campaignIds.size() > 1 || collectionInstrumentIds.size()>1){ From f59fb561d03758c16b826a091510ec0c0f2f8778 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Henninger?= Date: Fri, 5 Dec 2025 17:18:07 +0100 Subject: [PATCH 38/42] fix: dataProcessingContext --- .../service/context/DataProcessingContextService.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/main/java/fr/insee/genesis/domain/service/context/DataProcessingContextService.java b/src/main/java/fr/insee/genesis/domain/service/context/DataProcessingContextService.java index 380b66f3..83604765 100644 --- a/src/main/java/fr/insee/genesis/domain/service/context/DataProcessingContextService.java +++ b/src/main/java/fr/insee/genesis/domain/service/context/DataProcessingContextService.java @@ -167,16 +167,17 @@ public DataProcessingContextModel getContext(String interrogationId) throws Gene return null; } + DataProcessingContextModel contextModel = new DataProcessingContextModel(); - if (!collectionInstrumentIds.isEmpty()){ + if (!campaignIds.isEmpty()){ contextModel = DataProcessingContextMapper.INSTANCE.documentToModel( - dataProcessingContextPersistancePort.findByPartitionId(collectionInstrumentIds.stream().toList().getFirst()) + dataProcessingContextPersistancePort.findByPartitionId(campaignIds.stream().toList().getFirst()) ); } - if (contextModel.getId()==null && !campaignIds.isEmpty()) { + if (contextModel.getPartitionId()==null && !collectionInstrumentIds.isEmpty()) { contextModel = DataProcessingContextMapper.INSTANCE.documentToModel( - dataProcessingContextPersistancePort.findByPartitionId(campaignIds.stream().toList().getFirst())); + dataProcessingContextPersistancePort.findByPartitionId(collectionInstrumentIds.stream().toList().getFirst())); } return contextModel; From bf834ab35c9e4b114f13721a3397173b49c716fe Mon Sep 17 00:00:00 2001 From: QDIBYS Date: Thu, 11 Dec 2025 14:39:02 +0100 Subject: [PATCH 39/42] fix: typo --- .../controller/rest/responses/RawResponseController.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseController.java b/src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseController.java index 2011b6ac..b51aeb0e 100644 --- a/src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseController.java +++ b/src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseController.java @@ -185,7 +185,7 @@ public ResponseEntity processRawResponsesByCollectionInstrumentId( } @Operation(summary = "Get the list of collection instruments containing unprocessed interrogations") - @GetMapping(path = "/raw-responses/unprocessed/collection-intrument-ids") + @GetMapping(path = "/raw-responses/unprocessed/collection-instrument-ids") @PreAuthorize("hasRole('SCHEDULER')") public ResponseEntity> getUnprocessedCollectionInstrument(){ log.info("Try to get collection instruments containing unprocessed interrogations..."); From a6be1ef3c78669a333da22ee83bc24aa16c7dfed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Henninger?= Date: Fri, 12 Dec 2025 18:40:37 +0100 Subject: [PATCH 40/42] refactor: add collectionInstrumentId to data context --- .../DataProcessingContextMapperDto.java | 9 + .../rest/DataProcessingContextController.java | 186 +++++-- .../context/DataProcessingContextModel.java | 1 + .../api/DataProcessingContextApiPort.java | 19 +- .../DataProcessingContextPersistancePort.java | 5 +- .../context/DataProcessingContextService.java | 119 ++++- .../DataProcessingContextMongoAdapter.java | 29 +- .../DataProcessingContextDocument.java | 8 +- ...ataProcessingContextMongoDBRepository.java | 6 + .../DataProcessingContextDefinitions.java | 35 +- .../functional_tests/MainDefinitions.java | 24 +- .../DataProcessingContextControllerTest.java | 475 +++++++++++++++++- .../QuestionnaireControllerTest.java | 48 +- .../responses/ResponseControllerTest.java | 18 +- .../DataProcessingContextServiceTest.java | 27 +- .../utils/context/ContextDedupUtilsTest.java | 120 ++--- ...aProcessingContextPersistancePortStub.java | 24 +- 17 files changed, 925 insertions(+), 228 deletions(-) diff --git a/src/main/java/fr/insee/genesis/controller/mappers/DataProcessingContextMapperDto.java b/src/main/java/fr/insee/genesis/controller/mappers/DataProcessingContextMapperDto.java index faa5caf6..256660b1 100644 --- a/src/main/java/fr/insee/genesis/controller/mappers/DataProcessingContextMapperDto.java +++ b/src/main/java/fr/insee/genesis/controller/mappers/DataProcessingContextMapperDto.java @@ -23,4 +23,13 @@ public List dataProcessingContextListToScheduleDtoList(List saveContext( @Parameter(description = "Identifier of the partition", required = true) @RequestParam("partitionId") String partitionId, @@ -60,8 +51,40 @@ public ResponseEntity saveContext( return ResponseEntity.ok().build(); } + @Operation(summary = "Create or update a data processing context") + @PutMapping(path = "/contexts/{collectionInstrumentId}/review") + @PreAuthorize("hasAnyRole('USER_PLATINE', 'USER_BACK_OFFICE', 'SCHEDULER')") + public ResponseEntity saveContextWithCollectionInstrumentId( + @PathVariable("collectionInstrumentId") String collectionInstrumentId, + @Parameter(description = "Allow reviewing") @RequestParam(value = "withReview", defaultValue = "false") Boolean withReview + ){ + try { + withReview = withReview != null && withReview; //False if null + dataProcessingContextApiPort.saveContextByCollectionInstrumentId(collectionInstrumentId, withReview); + }catch (GenesisException e){ + return new ResponseEntity<>(e.getMessage(), HttpStatusCode.valueOf(e.getStatus())); + } + return ResponseEntity.ok().build(); + } + + + @Operation(summary = "Returns partition review indicator") + @GetMapping(path = "/contexts/{collectionInstrumentId}/review") + @PreAuthorize("hasAnyRole('USER_BACK_OFFICE','SCHEDULER','USER_PLATINE')") + public ResponseEntity getReviewIndicatorByCollectionInstrumentId( + @PathVariable("collectionInstrumentId") String collectionInstrumentId + ){ + try { + boolean withReview = dataProcessingContextApiPort.getReviewByCollectionInstrumentId(collectionInstrumentId); + return ResponseEntity.ok(withReview); + }catch (GenesisException e){ + return new ResponseEntity<>(e.getMessage(), HttpStatusCode.valueOf(e.getStatus())); + } + } + + @Deprecated(forRemoval = true) @Operation(summary = "Returns partition review indicator") - @GetMapping(path = "/review") + @GetMapping(path = "/context/review") @PreAuthorize("hasAnyRole('USER_BACK_OFFICE','SCHEDULER','USER_PLATINE')") public ResponseEntity getReviewIndicator( @Parameter(description = "Identifier of the partition", required = true) @RequestParam("partitionId") String partitionId @@ -74,8 +97,9 @@ public ResponseEntity getReviewIndicator( } } + @Deprecated(forRemoval = true) @Operation(summary = "Schedule a Kraftwerk execution") - @PutMapping(path = "/schedules") + @PutMapping(path = "/context/schedules") @PreAuthorize("hasRole('USER_KRAFTWERK')") public ResponseEntity saveSchedule( @Parameter(description = "Partition identifier to call Kraftwerk on") @RequestParam("partitionId") String partitionId, @@ -119,8 +143,55 @@ public ResponseEntity saveSchedule( return ResponseEntity.ok().build(); } + // Should be refactored to make it restfull + @Operation(summary = "Schedule a Kraftwerk execution using the collection instrument") + @PutMapping(path = "/contexts/schedules") + @PreAuthorize("hasRole('USER_KRAFTWERK')") + public ResponseEntity saveScheduleWithCollectionInstrumentId( + @Parameter(description = "Collection instrument to call Kraftwerk on") @RequestParam("collectionInstrumentId") String collectionInstrumentId, + @Parameter(description = "Kraftwerk endpoint") @RequestParam(value = "serviceTocall", defaultValue = Constants.KRAFTWERK_MAIN_ENDPOINT) ServiceToCall serviceToCall, + @Parameter(description = "Frequency in Spring cron format (6 inputs, go to https://crontab.cronhub.io/ for generator) \n Example : 0 0 6 * * *") @RequestParam("frequency") String frequency, + @Parameter(description = "Schedule effective date and time", example = "2024-01-01T12:00:00") @RequestParam("scheduleBeginDate") LocalDateTime scheduleBeginDate, + @Parameter(description = "Schedule end date and time", example = "2024-01-01T12:00:00") @RequestParam("scheduleEndDate") LocalDateTime scheduleEndDate, + @Parameter(description = "Encrypt after process ? Ignore next parameters if false") @RequestParam(value = + "useEncryption", + defaultValue = "false") boolean useEncryption, + @Parameter(description = "(Encryption) vault path") @RequestParam(value = "encryptionVaultPath", defaultValue = "") String encryptionVaultPath, + @Parameter(description = "(Encryption) output folder") @RequestParam(value = "encryptionOutputFolder", + defaultValue = "") String encryptionOutputFolder, + @Parameter(description = "(Encryption) Use signature system") @RequestParam(value = "useSignature", defaultValue = "false") boolean useSignature + ) { + try { + //Check frequency + if(!CronExpression.isValidExpression(frequency)) { + log.warn("Returned error for wrong frequency : {}", frequency); + throw new GenesisException(400, "Wrong frequency syntax"); + } + + TrustParameters trustParameters = null; + if(useEncryption) { + trustParameters = new TrustParameters( + fileUtils.getKraftwerkOutFolder(collectionInstrumentId), + encryptionOutputFolder, + encryptionVaultPath, + useSignature + ); + } + dataProcessingContextApiPort.saveKraftwerkExecutionScheduleByCollectionInstrumentId( + collectionInstrumentId, + serviceToCall == null ? ServiceToCall.MAIN : serviceToCall, + frequency, + scheduleBeginDate, scheduleEndDate, trustParameters + ); + }catch (GenesisException e){ + return new ResponseEntity<>(e.getMessage(), HttpStatusCode.valueOf(e.getStatus())); + } + return ResponseEntity.ok().build(); + } + + @Deprecated(forRemoval = true) @Operation(summary = "Fetch all schedules") - @GetMapping(path = "/schedules") + @GetMapping(path = "/context/schedules") @PreAuthorize("hasAnyRole('SCHEDULER','READER')") public ResponseEntity getAllSchedules() { log.debug("Got GET all schedules request"); @@ -131,8 +202,23 @@ public ResponseEntity getAllSchedules() { return ResponseEntity.ok(surveyScheduleDocumentModels); } + //It is just a change of path in the url + @Operation(summary = "Fetch all schedules") + @GetMapping(path = "/contexts/schedules") + @PreAuthorize("hasAnyRole('SCHEDULER','READER')") + public ResponseEntity getAllSchedulesV2() { + log.debug("Got GET all schedules request"); + + List surveyScheduleDocumentModels = dataProcessingContextApiPort.getAllSchedules(); + + log.info("Returning {} schedule documents...", surveyScheduleDocumentModels.size()); + return ResponseEntity.ok(surveyScheduleDocumentModels); + } + + + @Deprecated(forRemoval = true) @Operation(summary = "Set last execution date of a partition with new date or nothing") - @PostMapping(path = "/schedules/lastExecutionDate") + @PostMapping(path = "/context/schedules/lastExecutionDate") @PreAuthorize("hasRole('SCHEDULER')") public ResponseEntity setSurveyLastExecution( @Parameter(description = "Survey name to call Kraftwerk on") @RequestBody String partitionId, @@ -147,8 +233,25 @@ public ResponseEntity setSurveyLastExecution( return ResponseEntity.ok().build(); } + @Operation(summary = "Update the date of the last extraction of data corresponding to a collection instrument") + @PutMapping(path = "/contexts/{collectionInstrumentId}/lastExecutionDate") + @PreAuthorize("hasRole('SCHEDULER')") + public ResponseEntity setSurveyLastExecutionByCollectionInstrumentId( + @PathVariable("collectionInstrumentId") @RequestBody String collectionInstrumentId, + @Parameter(description = "Date to save as last execution date", example = "2024-01-01T12:00:00") @RequestParam("newDate") LocalDateTime newDate + ) { + try { + dataProcessingContextApiPort.updateLastExecutionDateByCollectionInstrumentId(collectionInstrumentId, newDate); + log.info("{} last execution updated at {} !", collectionInstrumentId, newDate); + }catch (GenesisException e){ + return new ResponseEntity<>(e.getMessage(), HttpStatusCode.valueOf(e.getStatus())); + } + return ResponseEntity.ok().build(); + } + + @Deprecated(forRemoval = true) @Operation(summary = "Delete the Kraftwerk execution schedules of a partition") - @DeleteMapping(path = "/schedules") + @DeleteMapping(path = "/context/schedules") @PreAuthorize("hasRole('USER_KRAFTWERK')") public ResponseEntity deleteSchedules( @Parameter(description = "Survey name of the schedule(s) to delete") @RequestParam("partitionId") String partitionId @@ -162,36 +265,31 @@ public ResponseEntity deleteSchedules( return ResponseEntity.ok().build(); } + @Operation(summary = "Delete the Kraftwerk execution schedules of a collection instrument id") + @DeleteMapping(path = "/context/{collectionInstrumentId}/schedules") + @PreAuthorize("hasRole('USER_KRAFTWERK')") + public ResponseEntity deleteSchedulesByCollectionInstrumentId( + @PathVariable("collectionInstrumentId") String collectionInstrumentId + ){ + try { + dataProcessingContextApiPort.deleteSchedulesByCollectionInstrumentId(collectionInstrumentId); + }catch (GenesisException e){ + return new ResponseEntity<>(e.getMessage(), HttpStatusCode.valueOf(e.getStatus())); + } + log.info("Schedule deleted for survey {}", collectionInstrumentId); + return ResponseEntity.ok().build(); + } + @Operation(summary = "Delete expired schedules") - @DeleteMapping(path = "/schedules/expired-schedules") + @DeleteMapping(path = "/context/schedules/expired-schedules") @PreAuthorize("hasRole('SCHEDULER')") - public ResponseEntity deleteExpiredSchedules() throws GenesisException, IOException { - Set storedSurveySchedulesNames = new HashSet<>(); - for(ScheduleDto scheduleDto : dataProcessingContextApiPort.getAllSchedules()){ - storedSurveySchedulesNames.add(scheduleDto.surveyName()); - } - for (String surveyScheduleName : storedSurveySchedulesNames) { - List deletedKraftwerkExecutionSchedules = dataProcessingContextApiPort.deleteExpiredSchedules(surveyScheduleName); - //Save in JSON log - if(!deletedKraftwerkExecutionSchedules.isEmpty()) { - Path jsonLogPath = Path.of(fileUtils.getLogFolder(), Constants.SCHEDULE_ARCHIVE_FOLDER_NAME, - surveyScheduleName + ".json"); - ObjectMapper objectMapper = new ObjectMapper().findAndRegisterModules(); - objectMapper.registerModule(new JavaTimeModule()); - String jsonToWrite = objectMapper.writeValueAsString(deletedKraftwerkExecutionSchedules); - if(Files.exists(jsonLogPath)){ - //Remove last ] and append survey - StringBuilder content = new StringBuilder(Files.readString(jsonLogPath)); - content.setCharAt(content.length()-1, ','); - content.append(jsonToWrite, 1, jsonToWrite.length()-1); - content.append(']'); - Files.write(jsonLogPath, content.toString().getBytes(), StandardOpenOption.TRUNCATE_EXISTING); - }else { - Files.createDirectories(jsonLogPath.getParent()); - Files.write(jsonLogPath, jsonToWrite.getBytes()); - } - } + public ResponseEntity deleteExpiredSchedules(){ + try{ + dataProcessingContextApiPort.deleteExpiredSchedules(fileUtils.getLogFolder()); + } catch (GenesisException e){ + return new ResponseEntity<>(e.getMessage(), HttpStatusCode.valueOf(e.getStatus())); } + log.info("Expired schedules deleted"); return ResponseEntity.ok().build(); } } diff --git a/src/main/java/fr/insee/genesis/domain/model/context/DataProcessingContextModel.java b/src/main/java/fr/insee/genesis/domain/model/context/DataProcessingContextModel.java index 58a19c62..d0a14746 100644 --- a/src/main/java/fr/insee/genesis/domain/model/context/DataProcessingContextModel.java +++ b/src/main/java/fr/insee/genesis/domain/model/context/DataProcessingContextModel.java @@ -20,6 +20,7 @@ public class DataProcessingContextModel { @Id private ObjectId id; //Used to remove warning + @Deprecated(forRemoval = true) private String partitionId; private String collectionInstrumentId; //QuestionnaireId diff --git a/src/main/java/fr/insee/genesis/domain/ports/api/DataProcessingContextApiPort.java b/src/main/java/fr/insee/genesis/domain/ports/api/DataProcessingContextApiPort.java index 5b8e025e..e38d4ac8 100644 --- a/src/main/java/fr/insee/genesis/domain/ports/api/DataProcessingContextApiPort.java +++ b/src/main/java/fr/insee/genesis/domain/ports/api/DataProcessingContextApiPort.java @@ -2,7 +2,6 @@ import fr.insee.genesis.controller.dto.ScheduleDto; import fr.insee.genesis.domain.model.context.DataProcessingContextModel; -import fr.insee.genesis.domain.model.context.schedule.KraftwerkExecutionSchedule; import fr.insee.genesis.domain.model.context.schedule.ServiceToCall; import fr.insee.genesis.domain.model.context.schedule.TrustParameters; import fr.insee.genesis.exceptions.GenesisException; @@ -12,7 +11,9 @@ public interface DataProcessingContextApiPort { void saveContext(String partitionId, Boolean withReview) throws GenesisException; + void saveContextByCollectionInstrumentId(String collectionInstrumentID, Boolean withReview) throws GenesisException; + @Deprecated(forRemoval = true) void saveKraftwerkExecutionSchedule(String partitionId, ServiceToCall serviceToCall, String frequency, @@ -20,13 +21,22 @@ void saveKraftwerkExecutionSchedule(String partitionId, LocalDateTime endDate, TrustParameters trustParameters) throws GenesisException; + void saveKraftwerkExecutionScheduleByCollectionInstrumentId(String collectionInstrumentId, + ServiceToCall serviceToCall, + String frequency, + LocalDateTime startDate, + LocalDateTime endDate, + TrustParameters trustParameters) throws GenesisException; + void updateLastExecutionDate(String surveyName, LocalDateTime newDate) throws GenesisException; + void updateLastExecutionDateByCollectionInstrumentId(String collectionInstrumentId, LocalDateTime newDate) throws GenesisException; void deleteSchedules(String surveyName) throws GenesisException; + void deleteSchedulesByCollectionInstrumentId(String collectionInstrumentId) throws GenesisException; List getAllSchedules(); - List deleteExpiredSchedules(String surveyScheduleName) throws GenesisException; + void deleteExpiredSchedules(String logFolder) throws GenesisException; long countSchedules(); @@ -41,5 +51,10 @@ void saveKraftwerkExecutionSchedule(String partitionId, * @param partitionId id of the partition * @return the review indicator stored in genesis */ + @Deprecated(forRemoval = true) boolean getReviewByPartitionId(String partitionId) throws GenesisException; + + boolean getReviewByCollectionInstrumentId(String collectionInstrumentId) throws GenesisException; + + } diff --git a/src/main/java/fr/insee/genesis/domain/ports/spi/DataProcessingContextPersistancePort.java b/src/main/java/fr/insee/genesis/domain/ports/spi/DataProcessingContextPersistancePort.java index 69d3d6ba..1b4b4702 100644 --- a/src/main/java/fr/insee/genesis/domain/ports/spi/DataProcessingContextPersistancePort.java +++ b/src/main/java/fr/insee/genesis/domain/ports/spi/DataProcessingContextPersistancePort.java @@ -4,6 +4,7 @@ import fr.insee.genesis.domain.model.context.schedule.KraftwerkExecutionSchedule; import fr.insee.genesis.infrastructure.document.context.DataProcessingContextDocument; +import java.io.IOException; import java.util.List; public interface DataProcessingContextPersistancePort { @@ -11,6 +12,8 @@ public interface DataProcessingContextPersistancePort { List findByPartitionIds(List partitionIds); + DataProcessingContextModel findByCollectionInstrumentId(String collectionInstrumentId); + List findByCollectionInstrumentIds(List collectionInstrumentIds); void save(DataProcessingContextDocument dataProcessingContextDocument); @@ -23,7 +26,7 @@ public interface DataProcessingContextPersistancePort { long count(); - List removeExpiredSchedules(DataProcessingContextModel dataProcessingContextModel); + List removeExpiredSchedules(DataProcessingContextModel dataProcessingContextModel) throws IOException; List findAllByReview(boolean withReview); } diff --git a/src/main/java/fr/insee/genesis/domain/service/context/DataProcessingContextService.java b/src/main/java/fr/insee/genesis/domain/service/context/DataProcessingContextService.java index 83604765..4f19be63 100644 --- a/src/main/java/fr/insee/genesis/domain/service/context/DataProcessingContextService.java +++ b/src/main/java/fr/insee/genesis/domain/service/context/DataProcessingContextService.java @@ -1,5 +1,8 @@ package fr.insee.genesis.domain.service.context; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; +import fr.insee.genesis.Constants; import fr.insee.genesis.controller.dto.ScheduleDto; import fr.insee.genesis.domain.model.context.DataProcessingContextModel; import fr.insee.genesis.domain.model.context.schedule.KraftwerkExecutionSchedule; @@ -16,6 +19,10 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; import java.time.LocalDateTime; import java.util.ArrayList; import java.util.HashSet; @@ -53,6 +60,21 @@ public void saveContext(String partitionId, Boolean withReview) throws GenesisEx dataProcessingContextPersistancePort.save(DataProcessingContextMapper.INSTANCE.modelToDocument(dataProcessingContextModel)); } + @Override + public void saveContextByCollectionInstrumentId(String collectionInstrumentId, Boolean withReview) throws GenesisException { + DataProcessingContextModel dataProcessingContextModel = dataProcessingContextPersistancePort.findByCollectionInstrumentId(collectionInstrumentId); + if(dataProcessingContextModel == null){ + //Create if not exist + dataProcessingContextModel = DataProcessingContextModel.builder() + .collectionInstrumentId(collectionInstrumentId) + .kraftwerkExecutionScheduleList(new ArrayList<>()) + .build(); + } + dataProcessingContextModel.setWithReview(withReview); + + dataProcessingContextPersistancePort.save(DataProcessingContextMapper.INSTANCE.modelToDocument(dataProcessingContextModel)); + } + @Override public void saveKraftwerkExecutionSchedule(String partitionId, ServiceToCall serviceToCall, @@ -82,6 +104,30 @@ public void saveKraftwerkExecutionSchedule(String partitionId, dataProcessingContextPersistancePort.save(DataProcessingContextMapper.INSTANCE.modelToDocument(dataProcessingContextModel)); } + @Override + public void saveKraftwerkExecutionScheduleByCollectionInstrumentId(String collectionInstrumentId, ServiceToCall serviceToCall, String frequency, LocalDateTime startDate, LocalDateTime endDate, TrustParameters trustParameters) throws GenesisException { + DataProcessingContextModel dataProcessingContextModel = dataProcessingContextPersistancePort.findByCollectionInstrumentId(collectionInstrumentId); + if(dataProcessingContextModel == null){ + //Create if not exist + dataProcessingContextModel = DataProcessingContextModel.builder() + .collectionInstrumentId(collectionInstrumentId) + .withReview(false) + .kraftwerkExecutionScheduleList(new ArrayList<>()) + .build(); + } + + dataProcessingContextModel.getKraftwerkExecutionScheduleList().add( + new KraftwerkExecutionSchedule(frequency, + serviceToCall, + startDate, + endDate, + trustParameters + ) + ); + dataProcessingContextPersistancePort.save(DataProcessingContextMapper.INSTANCE.modelToDocument(dataProcessingContextModel)); + } + + @Deprecated(forRemoval = true) @Override public void updateLastExecutionDate(String partitionId, LocalDateTime newDate) throws GenesisException { DataProcessingContextModel dataProcessingContextModel = @@ -95,6 +141,16 @@ public void updateLastExecutionDate(String partitionId, LocalDateTime newDate) t dataProcessingContextPersistancePort.save(DataProcessingContextMapper.INSTANCE.modelToDocument(dataProcessingContextModel)); } + @Override + public void updateLastExecutionDateByCollectionInstrumentId(String collectionInstrumentId, LocalDateTime newDate) throws GenesisException { + DataProcessingContextModel dataProcessingContextModel = dataProcessingContextPersistancePort.findByCollectionInstrumentId(collectionInstrumentId); + if (dataProcessingContextModel == null) { + throw new GenesisException(404, NOT_FOUND_MESSAGE); + } + dataProcessingContextModel.setLastExecution(newDate); + dataProcessingContextPersistancePort.save(DataProcessingContextMapper.INSTANCE.modelToDocument(dataProcessingContextModel)); + } + @Override public void deleteSchedules(String partitionId) throws GenesisException { DataProcessingContextModel dataProcessingContextModel = @@ -108,6 +164,17 @@ public void deleteSchedules(String partitionId) throws GenesisException { dataProcessingContextPersistancePort.save(DataProcessingContextMapper.INSTANCE.modelToDocument(dataProcessingContextModel)); } + @Override + public void deleteSchedulesByCollectionInstrumentId(String collectionInstrumentId) throws GenesisException { + DataProcessingContextModel dataProcessingContextModel = + dataProcessingContextPersistancePort.findByCollectionInstrumentId(collectionInstrumentId); + if (dataProcessingContextModel == null) { + throw new GenesisException(404, NOT_FOUND_MESSAGE); + } + dataProcessingContextModel.setKraftwerkExecutionScheduleList(new ArrayList<>()); + dataProcessingContextPersistancePort.save(DataProcessingContextMapper.INSTANCE.modelToDocument(dataProcessingContextModel)); + } + @Override public List getAllSchedules() { List scheduleDtos = new ArrayList<>(); @@ -123,17 +190,38 @@ public List getAllSchedules() { } @Override - public List deleteExpiredSchedules(String partitionId) throws GenesisException { - DataProcessingContextModel dataProcessingContextModel = - DataProcessingContextMapper.INSTANCE.documentToModel( - dataProcessingContextPersistancePort.findByPartitionId(partitionId) - ); - if (dataProcessingContextModel == null) { - throw new GenesisException(404, NOT_FOUND_MESSAGE); + public void deleteExpiredSchedules(String logFolder) throws GenesisException { + List dataProcessingContextModels = + DataProcessingContextMapper.INSTANCE.listDocumentToListModel(dataProcessingContextPersistancePort.findAll()); + for(DataProcessingContextModel context : dataProcessingContextModels){ + try { + List deletedKraftwerkExecutionSchedules = dataProcessingContextPersistancePort.removeExpiredSchedules(context); + //Save in JSON log + if(!deletedKraftwerkExecutionSchedules.isEmpty()) { + String scheduleName = context.getCollectionInstrumentId()==null ? + context.getPartitionId() : context.getCollectionInstrumentId(); + Path jsonLogPath = Path.of(logFolder, Constants.SCHEDULE_ARCHIVE_FOLDER_NAME, + scheduleName + ".json"); + ObjectMapper objectMapper = new ObjectMapper().findAndRegisterModules(); + objectMapper.registerModule(new JavaTimeModule()); + String jsonToWrite = objectMapper.writeValueAsString(deletedKraftwerkExecutionSchedules); + if(Files.exists(jsonLogPath)){ + //Remove last ] and append survey + StringBuilder content = new StringBuilder(Files.readString(jsonLogPath)); + content.setCharAt(content.length()-1, ','); + content.append(jsonToWrite, 1, jsonToWrite.length()-1); + content.append(']'); + Files.write(jsonLogPath, content.toString().getBytes(), StandardOpenOption.TRUNCATE_EXISTING); + }else { + Files.createDirectories(jsonLogPath.getParent()); + Files.write(jsonLogPath, jsonToWrite.getBytes()); + } + } + } catch (IOException e) { + String name = context.getCollectionInstrumentId()!=null?context.getCollectionInstrumentId() :context.getPartitionId(); + throw new GenesisException(500,String.format("An error occured trying to delete expired schedules for %s",name)); + } } - return new ArrayList<>( - dataProcessingContextPersistancePort.removeExpiredSchedules(dataProcessingContextModel) - ); } @Override @@ -202,6 +290,7 @@ public List getPartitionIds(boolean withReview){ return partitionIds; } + @Deprecated(forRemoval = true) @Override public boolean getReviewByPartitionId(String partitionId) throws GenesisException { DataProcessingContextDocument dataProcessingContextDocument = @@ -213,4 +302,14 @@ public boolean getReviewByPartitionId(String partitionId) throws GenesisExceptio dataProcessingContextPersistancePort.findByPartitionId(partitionId) ).isWithReview(); } + + @Override + public boolean getReviewByCollectionInstrumentId(String collectionInstrumentId) throws GenesisException { + DataProcessingContextModel dataProcessingContextModel = + dataProcessingContextPersistancePort.findByCollectionInstrumentId(collectionInstrumentId); + if(dataProcessingContextModel == null){ + throw new GenesisException(404, "Data processing context not found"); + } + return dataProcessingContextModel.isWithReview(); + } } diff --git a/src/main/java/fr/insee/genesis/infrastructure/adapter/DataProcessingContextMongoAdapter.java b/src/main/java/fr/insee/genesis/infrastructure/adapter/DataProcessingContextMongoAdapter.java index 930135dc..f2aec1af 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/adapter/DataProcessingContextMongoAdapter.java +++ b/src/main/java/fr/insee/genesis/infrastructure/adapter/DataProcessingContextMongoAdapter.java @@ -7,6 +7,7 @@ import fr.insee.genesis.infrastructure.document.context.DataProcessingContextDocument; import fr.insee.genesis.infrastructure.mappers.DataProcessingContextMapper; import fr.insee.genesis.infrastructure.repository.DataProcessingContextMongoDBRepository; +import fr.insee.genesis.infrastructure.utils.FileUtils; import fr.insee.genesis.infrastructure.utils.context.ContextDedupUtils; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; @@ -17,6 +18,7 @@ import org.springframework.data.mongodb.core.query.Update; import org.springframework.stereotype.Service; +import java.io.IOException; import java.time.LocalDateTime; import java.util.ArrayList; import java.util.List; @@ -27,11 +29,14 @@ public class DataProcessingContextMongoAdapter implements DataProcessingContextPersistancePort { private final DataProcessingContextMongoDBRepository dataProcessingContextMongoDBRepository; private final MongoTemplate mongoTemplate; + private final FileUtils fileUtils; + @Autowired - public DataProcessingContextMongoAdapter(DataProcessingContextMongoDBRepository dataProcessingContextMongoDBRepository, MongoTemplate mongoTemplate) { + public DataProcessingContextMongoAdapter(DataProcessingContextMongoDBRepository dataProcessingContextMongoDBRepository, MongoTemplate mongoTemplate, FileUtils fileUtils) { this.dataProcessingContextMongoDBRepository = dataProcessingContextMongoDBRepository; this.mongoTemplate = mongoTemplate; + this.fileUtils = fileUtils; } @@ -49,11 +54,18 @@ public List findByPartitionIds(List partitio return DataProcessingContextMapper.INSTANCE.listDocumentToListModel(ContextDedupUtils.deduplicateContexts(existingDocuments)); } + @Override + public DataProcessingContextModel findByCollectionInstrumentId(String collectionInstrumentId) { + List existingDocuments = + dataProcessingContextMongoDBRepository.findByCollectionInstrumentIdList(List.of(collectionInstrumentId)); + return DataProcessingContextMapper.INSTANCE.documentToModel(existingDocuments.isEmpty()?null:existingDocuments.getFirst()); + } + @Override public List findByCollectionInstrumentIds(List collectionInstrumentIds) { List existingDocuments = dataProcessingContextMongoDBRepository.findByCollectionInstrumentIdList(collectionInstrumentIds); - return DataProcessingContextMapper.INSTANCE.listDocumentToListModel(ContextDedupUtils.deduplicateContexts(existingDocuments)); + return DataProcessingContextMapper.INSTANCE.listDocumentToListModel(existingDocuments); } @Override @@ -73,7 +85,7 @@ public void deleteBypartitionId(String partitionId) { @Override public List findAll() { - return ContextDedupUtils.deduplicateContexts(dataProcessingContextMongoDBRepository.findAll()); + return dataProcessingContextMongoDBRepository.findAll(); } @Override @@ -82,7 +94,7 @@ public long count() { } @Override - public List removeExpiredSchedules(DataProcessingContextModel dataProcessingContextModel) { + public List removeExpiredSchedules(DataProcessingContextModel dataProcessingContextModel) throws IOException { List deletedKraftwerkExecutionSchedules = new ArrayList<>(); for (KraftwerkExecutionSchedule kraftwerkExecutionScheduleToRemove : dataProcessingContextModel.getKraftwerkExecutionScheduleList().stream().filter( @@ -91,6 +103,13 @@ public List removeExpiredSchedules(DataProcessingCon deletedKraftwerkExecutionSchedules.add(kraftwerkExecutionScheduleToRemove); Query query = Query.query(Criteria.where("scheduleEndDate").is(kraftwerkExecutionScheduleToRemove.getScheduleEndDate())); + // If collectionInstrumentId is present we use it, if not we use partitionId + if (dataProcessingContextModel.getCollectionInstrumentId()!=null){ + mongoTemplate.updateMulti(Query.query(Criteria.where("collectionInstrumentId").is(dataProcessingContextModel.getCollectionInstrumentId())), new Update().pull( + "kraftwerkExecutionScheduleList", query), + Constants.MONGODB_SCHEDULE_COLLECTION_NAME); + continue; + } mongoTemplate.updateMulti(Query.query(Criteria.where("surveyName").is(dataProcessingContextModel.getPartitionId())), new Update().pull( "kraftwerkExecutionScheduleList", query), Constants.MONGODB_SCHEDULE_COLLECTION_NAME); @@ -102,7 +121,7 @@ public List removeExpiredSchedules(DataProcessingCon @Override public List findAllByReview(boolean withReview) { - return ContextDedupUtils.deduplicateContexts(dataProcessingContextMongoDBRepository.findAll()) + return dataProcessingContextMongoDBRepository.findAll() .stream().filter(doc -> doc.isWithReview() == withReview ).toList(); diff --git a/src/main/java/fr/insee/genesis/infrastructure/document/context/DataProcessingContextDocument.java b/src/main/java/fr/insee/genesis/infrastructure/document/context/DataProcessingContextDocument.java index 662bc2f9..079886f7 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/document/context/DataProcessingContextDocument.java +++ b/src/main/java/fr/insee/genesis/infrastructure/document/context/DataProcessingContextDocument.java @@ -14,18 +14,12 @@ @Data @Document(collection = Constants.MONGODB_CONTEXT_COLLECTION_NAME) public class DataProcessingContextDocument{ - public DataProcessingContextDocument(String partitionId, - List kraftwerkExecutionScheduleList, - boolean withReview) { - this.partitionId = partitionId; - this.kraftwerkExecutionScheduleList = kraftwerkExecutionScheduleList; - this.withReview = withReview; - } @Id private ObjectId id; @Indexed private String partitionId; //ex Survey Name, campaignId + @Indexed private String collectionInstrumentId; // QuestionnaireId private LocalDateTime lastExecution; private List kraftwerkExecutionScheduleList; diff --git a/src/main/java/fr/insee/genesis/infrastructure/repository/DataProcessingContextMongoDBRepository.java b/src/main/java/fr/insee/genesis/infrastructure/repository/DataProcessingContextMongoDBRepository.java index b0fe29fd..0f7e2d72 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/repository/DataProcessingContextMongoDBRepository.java +++ b/src/main/java/fr/insee/genesis/infrastructure/repository/DataProcessingContextMongoDBRepository.java @@ -9,12 +9,18 @@ @Repository public interface DataProcessingContextMongoDBRepository extends MongoRepository { + @Deprecated(forRemoval = true) @Query(value = "{ 'partitionId' : {$in: ?0} }") List findByPartitionIdList(List partitionIds); + @Query(value = "{ 'collectionInstrumentId' : {$in: ?0} }") List findByCollectionInstrumentIdList(List collectionInstrumentIds); + @Deprecated(forRemoval = true) @Query(value = "{ 'partitionId' : ?0 }", delete = true) void deleteByPartitionId(String partitionId); + + @Query(value = "{ 'collectionInstrumentId' : ?0 }", delete = true) + void deleteByCollectionInstrumentId(String partitionId); } diff --git a/src/test/java/cucumber/functional_tests/DataProcessingContextDefinitions.java b/src/test/java/cucumber/functional_tests/DataProcessingContextDefinitions.java index 90a488fc..35011b41 100644 --- a/src/test/java/cucumber/functional_tests/DataProcessingContextDefinitions.java +++ b/src/test/java/cucumber/functional_tests/DataProcessingContextDefinitions.java @@ -46,27 +46,29 @@ public void init() { @Given("We have a context in database with partition {string}") public void add_context_to_database(String partitionId) { - dataProcessingContextPersistancePortStub.getMongoStub().add( - new DataProcessingContextDocument(partitionId, new ArrayList<>(), false) - ); + DataProcessingContextDocument doc = new DataProcessingContextDocument(); + doc.setPartitionId(partitionId); + doc.setKraftwerkExecutionScheduleList(new ArrayList<>()); + doc.setWithReview(false); + dataProcessingContextPersistancePortStub.getMongoStub().add(doc); } @Given("We have a context in database with partition {string} and review indicator to {string}") public void add_context_with_review_indicator(String partitionId, String withReviewString) { - boolean withReview = Boolean.parseBoolean(withReviewString); - dataProcessingContextPersistancePortStub.getMongoStub().add( - new DataProcessingContextDocument(partitionId, new ArrayList<>(), withReview) - ); + DataProcessingContextDocument doc = new DataProcessingContextDocument(); + doc.setPartitionId(partitionId); + doc.setKraftwerkExecutionScheduleList(new ArrayList<>()); + doc.setWithReview(Boolean.parseBoolean(withReviewString)); + dataProcessingContextPersistancePortStub.getMongoStub().add(doc); } @Given("We have a context in database with partition {string} and {int} valid schedule\\(s)") public void add_context_with_schedule(String partitionId, int expectedScheduleNumber) { - DataProcessingContextDocument dataProcessingContextDocument = new DataProcessingContextDocument( - partitionId, - new ArrayList<>(), - false - ); + DataProcessingContextDocument dataProcessingContextDocument = new DataProcessingContextDocument(); + dataProcessingContextDocument.setPartitionId(partitionId); + dataProcessingContextDocument.setKraftwerkExecutionScheduleList(new ArrayList<>()); + dataProcessingContextDocument.setWithReview(false); for(int i = 0; i < expectedScheduleNumber; i++){ dataProcessingContextDocument.getKraftwerkExecutionScheduleList().add( @@ -85,11 +87,10 @@ public void add_context_with_schedule(String partitionId, int expectedScheduleNu @Given("We have a context in database with partition {string} and {int} expired schedule\\(s)") public void add_context_with_expired_schedule(String partitionId, int expectedScheduleNumber) { - DataProcessingContextDocument dataProcessingContextDocument = new DataProcessingContextDocument( - partitionId, - new ArrayList<>(), - false - ); + DataProcessingContextDocument dataProcessingContextDocument = new DataProcessingContextDocument(); + dataProcessingContextDocument.setPartitionId(partitionId); + dataProcessingContextDocument.setKraftwerkExecutionScheduleList(new ArrayList<>()); + dataProcessingContextDocument.setWithReview(false); for(int i = 0; i < expectedScheduleNumber; i++){ dataProcessingContextDocument.getKraftwerkExecutionScheduleList().add( diff --git a/src/test/java/cucumber/functional_tests/MainDefinitions.java b/src/test/java/cucumber/functional_tests/MainDefinitions.java index 2a2d524e..4f92755d 100644 --- a/src/test/java/cucumber/functional_tests/MainDefinitions.java +++ b/src/test/java/cucumber/functional_tests/MainDefinitions.java @@ -122,24 +122,20 @@ public void copy_data_file(String dataFile) throws IOException { @Given("We have a context in database for that data with review {string}") public void create_context(String withReviewString) { - dataProcessingContextPersistancePortStub.getMongoStub().add( - new DataProcessingContextDocument( - directory, - new ArrayList<>(), - Boolean.parseBoolean(withReviewString) - ) - ); + DataProcessingContextDocument doc = new DataProcessingContextDocument(); + doc.setPartitionId(directory); + doc.setKraftwerkExecutionScheduleList(new ArrayList<>()); + doc.setWithReview(Boolean.parseBoolean(withReviewString)); + dataProcessingContextPersistancePortStub.getMongoStub().add(doc); } @Given("We have a context in database for partitionId {string} with review {string}") public void create_context(String partitionId, String withReviewString) { - dataProcessingContextPersistancePortStub.getMongoStub().add( - new DataProcessingContextDocument( - partitionId, - new ArrayList<>(), - Boolean.parseBoolean(withReviewString) - ) - ); + DataProcessingContextDocument doc = new DataProcessingContextDocument(); + doc.setPartitionId(partitionId); + doc.setKraftwerkExecutionScheduleList(new ArrayList<>()); + doc.setWithReview(Boolean.parseBoolean(withReviewString)); + dataProcessingContextPersistancePortStub.getMongoStub().add(doc); } @Given("We have a survey unit with campaignId {string} and interrogationId {string}") diff --git a/src/test/java/fr/insee/genesis/controller/rest/DataProcessingContextControllerTest.java b/src/test/java/fr/insee/genesis/controller/rest/DataProcessingContextControllerTest.java index f4594a98..4f08ec28 100644 --- a/src/test/java/fr/insee/genesis/controller/rest/DataProcessingContextControllerTest.java +++ b/src/test/java/fr/insee/genesis/controller/rest/DataProcessingContextControllerTest.java @@ -6,7 +6,6 @@ import fr.insee.genesis.domain.model.context.schedule.KraftwerkExecutionSchedule; import fr.insee.genesis.domain.model.context.schedule.ServiceToCall; import fr.insee.genesis.domain.service.context.DataProcessingContextService; -import fr.insee.genesis.exceptions.GenesisException; import fr.insee.genesis.infrastructure.document.context.DataProcessingContextDocument; import fr.insee.genesis.infrastructure.mappers.DataProcessingContextMapper; import fr.insee.genesis.infrastructure.utils.FileUtils; @@ -64,6 +63,15 @@ void getAllSchedulesTest() { Assertions.assertThat(response.getStatusCode().is2xxSuccessful()).isTrue(); } + @Test + void getAllSchedulesV2Test() { + //When + ResponseEntity response = dataProcessingContextController.getAllSchedulesV2(); + + //Then + Assertions.assertThat(response.getStatusCode().is2xxSuccessful()).isTrue(); + } + @Test void addScheduleWithoutEncryptionTest() { //When @@ -91,6 +99,33 @@ void addScheduleWithoutEncryptionTest() { Assertions.assertThat(dataProcessingContextDocument.getKraftwerkExecutionScheduleList().getFirst().getTrustParameters()).isNull(); } + @Test + void addScheduleWithoutEncryptionTestUsingCollectionInstrumentId() { + //When + String collectionInstrumentId = "TESTADDSURVEY_CI"; + ServiceToCall serviceToCall = ServiceToCall.MAIN; + String frequency = "0 0 6 * * *"; + LocalDateTime scheduleBeginDate = LocalDateTime.now(); + LocalDateTime scheduleEndDate = LocalDateTime.now().plusMonths(1); + + dataProcessingContextController.saveScheduleWithCollectionInstrumentId(collectionInstrumentId, serviceToCall, frequency, scheduleBeginDate, scheduleEndDate, + false, "TEST", "TEST", false); + + //Then + Assertions.assertThat(dataProcessingContextPersistancePortStub.getMongoStub()).filteredOn(dataProcessingContextDocument -> + dataProcessingContextDocument.getCollectionInstrumentId().equals(collectionInstrumentId) + ).isNotEmpty(); + + List mongoStubFiltered = dataProcessingContextPersistancePortStub.getMongoStub().stream().filter(dataProcessingContextDocument -> + dataProcessingContextDocument.getCollectionInstrumentId().equals(collectionInstrumentId)).toList(); + + DataProcessingContextDocument dataProcessingContextDocument = mongoStubFiltered.getFirst(); + + Assertions.assertThat(dataProcessingContextDocument.getKraftwerkExecutionScheduleList()).isNotEmpty(); + Assertions.assertThat(dataProcessingContextDocument.getKraftwerkExecutionScheduleList().getFirst().getFrequency()).isEqualTo(frequency); + Assertions.assertThat(dataProcessingContextDocument.getKraftwerkExecutionScheduleList().getFirst().getTrustParameters()).isNull(); + } + @Test void addScheduleWithoutEncryptionTest_nullServiceToCall() { //When @@ -119,6 +154,34 @@ void addScheduleWithoutEncryptionTest_nullServiceToCall() { Assertions.assertThat(dataProcessingContextDocument.getKraftwerkExecutionScheduleList().getFirst().getTrustParameters()).isNull(); } + @Test + void addScheduleWithoutEncryptionTest_nullServiceToCall_collectionInstrumentId() { + //When + String collectionInstrumentId = "TESTADDSURVEY_CI"; + ServiceToCall serviceToCall = null; + String frequency = "0 0 6 * * *"; + LocalDateTime scheduleBeginDate = LocalDateTime.now(); + LocalDateTime scheduleEndDate = LocalDateTime.now().plusMonths(1); + + dataProcessingContextController.saveScheduleWithCollectionInstrumentId(collectionInstrumentId, serviceToCall, frequency, scheduleBeginDate, scheduleEndDate, + false, "TEST", "TEST", false); + + //Then + Assertions.assertThat(dataProcessingContextPersistancePortStub.getMongoStub()).filteredOn(dataProcessingContextDocument -> + dataProcessingContextDocument.getCollectionInstrumentId().equals(collectionInstrumentId) + ).isNotEmpty(); + + List mongoStubFiltered = dataProcessingContextPersistancePortStub.getMongoStub().stream().filter(dataProcessingContextDocument -> + dataProcessingContextDocument.getCollectionInstrumentId().equals(collectionInstrumentId)).toList(); + + DataProcessingContextDocument dataProcessingContextDocument = mongoStubFiltered.getFirst(); + + Assertions.assertThat(dataProcessingContextDocument.getKraftwerkExecutionScheduleList()).isNotEmpty(); + Assertions.assertThat(dataProcessingContextDocument.getKraftwerkExecutionScheduleList().getFirst().getServiceToCall()).isEqualTo(ServiceToCall.MAIN); + Assertions.assertThat(dataProcessingContextDocument.getKraftwerkExecutionScheduleList().getFirst().getFrequency()).isEqualTo(frequency); + Assertions.assertThat(dataProcessingContextDocument.getKraftwerkExecutionScheduleList().getFirst().getTrustParameters()).isNull(); + } + @Test void addScheduleWithEncryptionTest() { //When @@ -157,6 +220,44 @@ void addScheduleWithEncryptionTest() { Assertions.assertThat(dataProcessingContextDocument.getKraftwerkExecutionScheduleList().getFirst().getTrustParameters().isUseSignature()).isFalse(); } + @Test + void addScheduleWithEncryptionTest_collectionInstrumentId() { + //When + String collectionInstrumentId = "TESTADDSURVEY_CI"; + ServiceToCall serviceToCall = ServiceToCall.MAIN; + String frequency = "0 0 6 * * *"; + LocalDateTime scheduleBeginDate = LocalDateTime.now(); + LocalDateTime scheduleEndDate = LocalDateTime.now().plusMonths(1); + + + dataProcessingContextController.saveScheduleWithCollectionInstrumentId(collectionInstrumentId, serviceToCall, frequency, scheduleBeginDate, scheduleEndDate, true, + "testvault/testkey", + Path.of(TestConstants.TEST_RESOURCES_DIRECTORY).resolve("OUT_ENCRYPTED").resolve(collectionInstrumentId).toString(), + false + ); + + //Then + Assertions.assertThat(dataProcessingContextPersistancePortStub.getMongoStub()).filteredOn(dataProcessingContextDocument -> + dataProcessingContextDocument.getCollectionInstrumentId().equals(collectionInstrumentId) + ).isNotEmpty(); + + List mongoStubFiltered = dataProcessingContextPersistancePortStub.getMongoStub().stream().filter(dataProcessingContextDocument -> + dataProcessingContextDocument.getCollectionInstrumentId().equals(collectionInstrumentId)).toList(); + + DataProcessingContextDocument dataProcessingContextDocument = mongoStubFiltered.getFirst(); + + Assertions.assertThat(dataProcessingContextDocument.getKraftwerkExecutionScheduleList()).isNotEmpty(); + Assertions.assertThat(dataProcessingContextDocument.getKraftwerkExecutionScheduleList().getFirst().getFrequency()).isEqualTo(frequency); + Assertions.assertThat(dataProcessingContextDocument.getKraftwerkExecutionScheduleList().getFirst().getTrustParameters()).isNotNull(); + Assertions.assertThat(dataProcessingContextDocument.getKraftwerkExecutionScheduleList().getFirst().getTrustParameters().getInputPath()).contains( + "TESTADDSURVEY"); + Assertions.assertThat(dataProcessingContextDocument.getKraftwerkExecutionScheduleList().getFirst().getTrustParameters().getOutputFolder()).contains( + "TESTADDSURVEY"); + Assertions.assertThat(dataProcessingContextDocument.getKraftwerkExecutionScheduleList().getFirst().getTrustParameters().getVaultPath()).isEqualTo( + "testvault/testkey"); + Assertions.assertThat(dataProcessingContextDocument.getKraftwerkExecutionScheduleList().getFirst().getTrustParameters().isUseSignature()).isFalse(); + } + @Test void addAdditionnalScheduleTest() { //When @@ -185,6 +286,34 @@ void addAdditionnalScheduleTest() { Assertions.assertThat(dataProcessingContextDocument.getKraftwerkExecutionScheduleList().getFirst().getFrequency()).isEqualTo(frequency); } + @Test + void addAdditionnalScheduleTest_collectionInstrumentId() { + //When + String collectionInstrumentId = "TESTADDSURVEY_CI"; //Already exists in stub + ServiceToCall serviceToCall = ServiceToCall.MAIN; + String frequency = "0 0 6 * * *"; + LocalDateTime scheduleBeginDate = LocalDateTime.now(); + LocalDateTime scheduleEndDate = LocalDateTime.now().plusMonths(1); + + + dataProcessingContextController.saveScheduleWithCollectionInstrumentId(collectionInstrumentId, serviceToCall, frequency, scheduleBeginDate, scheduleEndDate, + false, "", "", false); + + //Then + Assertions.assertThat(dataProcessingContextPersistancePortStub.getMongoStub()).filteredOn(dataProcessingContextDocument -> + dataProcessingContextDocument.getCollectionInstrumentId().equals(collectionInstrumentId) + ).isNotEmpty().hasSize(1); + + List mongoStubFiltered = dataProcessingContextPersistancePortStub.getMongoStub().stream().filter(dataProcessingContextDocument -> + dataProcessingContextDocument.getCollectionInstrumentId().equals(collectionInstrumentId)).toList(); + + DataProcessingContextDocument dataProcessingContextDocument = mongoStubFiltered.getFirst(); + Assertions.assertThat(dataProcessingContextDocument.getLastExecution()).isNull(); + + Assertions.assertThat(dataProcessingContextDocument.getKraftwerkExecutionScheduleList()).isNotEmpty(); + Assertions.assertThat(dataProcessingContextDocument.getKraftwerkExecutionScheduleList().getFirst().getFrequency()).isEqualTo(frequency); + } + @Test void addScheduleDedupTest() { //Given @@ -215,6 +344,36 @@ void addScheduleDedupTest() { Assertions.assertThat(dataProcessingContextDocument.getKraftwerkExecutionScheduleList().getFirst().getFrequency()).isEqualTo(frequency); } + @Test + void addScheduleDedupTest_collectionInstrumentId() { + //Given + addNewDocumentToStubWithCollectionInstrumentId(); + + //When + String collectionInstrumentId = "TESTSURVEY_CI"; //Already exists in stub + ServiceToCall serviceToCall = ServiceToCall.MAIN; + String frequency = "0 0 6 * * *"; + LocalDateTime scheduleBeginDate = LocalDateTime.now(); + LocalDateTime scheduleEndDate = LocalDateTime.now().plusMonths(1); + + + dataProcessingContextController.saveScheduleWithCollectionInstrumentId(collectionInstrumentId, serviceToCall, frequency, scheduleBeginDate, scheduleEndDate, + false, "", "", false); + //Then + Assertions.assertThat(dataProcessingContextPersistancePortStub.getMongoStub()).filteredOn(dataProcessingContextDocument -> + dataProcessingContextDocument.getCollectionInstrumentId().equals(collectionInstrumentId) + ).isNotEmpty().hasSize(1); + + List mongoStubFiltered = dataProcessingContextPersistancePortStub.getMongoStub().stream().filter(dataProcessingContextDocument -> + dataProcessingContextDocument.getCollectionInstrumentId().equals(collectionInstrumentId)).toList(); + + DataProcessingContextDocument dataProcessingContextDocument = mongoStubFiltered.getFirst(); + Assertions.assertThat(dataProcessingContextDocument.getLastExecution()).isNull(); + + Assertions.assertThat(dataProcessingContextDocument.getKraftwerkExecutionScheduleList()).isNotEmpty(); + Assertions.assertThat(dataProcessingContextDocument.getKraftwerkExecutionScheduleList().getFirst().getFrequency()).isEqualTo(frequency); + } + @Test void updateLastExecutionTest(){ //Given @@ -229,6 +388,20 @@ void updateLastExecutionTest(){ Assertions.assertThat(mongoStubFiltered.getFirst().getLastExecution()).isNotNull(); } + @Test + void updateLastExecutionTest_collectionInstrumentId(){ + //Given + addNewDocumentToStubWithCollectionInstrumentId(); + + //When + dataProcessingContextController.setSurveyLastExecutionByCollectionInstrumentId("TESTSURVEY_CI", LocalDateTime.now()); + + //Then + List mongoStubFiltered = dataProcessingContextPersistancePortStub.getMongoStub().stream().filter(dataProcessingContextDocument -> + dataProcessingContextDocument.getCollectionInstrumentId().equals("TESTSURVEY_CI")).toList(); + Assertions.assertThat(mongoStubFiltered.getFirst().getLastExecution()).isNotNull(); + } + @Test void setLastExecutionTestToNull(){ //Given @@ -243,6 +416,20 @@ void setLastExecutionTestToNull(){ Assertions.assertThat(mongoStubFiltered.getFirst().getLastExecution()).isNull(); } + @Test + void setLastExecutionTestToNull_collectionInstrumentId(){ + //Given + addNewDocumentToStubWithCollectionInstrumentId(); + + //When + dataProcessingContextController.setSurveyLastExecutionByCollectionInstrumentId("TESTSURVEY_CI", null); + + //Then + List mongoStubFiltered = dataProcessingContextPersistancePortStub.getMongoStub().stream().filter(dataProcessingContextDocument -> + dataProcessingContextDocument.getCollectionInstrumentId().equals("TESTSURVEY_CI")).toList(); + Assertions.assertThat(mongoStubFiltered.getFirst().getLastExecution()).isNull(); + } + @Test void setLastExecutionTest(){ //Given @@ -258,6 +445,21 @@ void setLastExecutionTest(){ Assertions.assertThat(mongoStubFiltered.getFirst().getLastExecution()).isEqualTo(date); } + @Test + void setLastExecutionTest_collectionInstrumentId(){ + //Given + LocalDateTime date = LocalDateTime.now(); + addNewDocumentToStubWithCollectionInstrumentId(); + + //When + dataProcessingContextController.setSurveyLastExecutionByCollectionInstrumentId("TESTSURVEY_CI", date); + + //Then + List mongoStubFiltered = dataProcessingContextPersistancePortStub.getMongoStub().stream().filter(dataProcessingContextDocument -> + dataProcessingContextDocument.getCollectionInstrumentId().equals("TESTSURVEY_CI")).toList(); + Assertions.assertThat(mongoStubFiltered.getFirst().getLastExecution()).isEqualTo(date); + } + @Test void wrongFrequencyTest(){ //When+Then @@ -272,6 +474,20 @@ void wrongFrequencyTest(){ Assertions.assertThat(response.getStatusCode().is4xxClientError()).isTrue(); } + @Test + void wrongFrequencyTest_collectionInstrumentId(){ + //When+Then + String collectionInstrumentId = "TESTSURVEY_CI"; //Already exists in stub + ServiceToCall serviceToCall = ServiceToCall.MAIN; + String frequency = "ERROR"; + LocalDateTime scheduleBeginDate = LocalDateTime.now(); + LocalDateTime scheduleEndDate = LocalDateTime.now().plusMonths(1); + + ResponseEntity response = dataProcessingContextController.saveScheduleWithCollectionInstrumentId(collectionInstrumentId, serviceToCall, frequency, scheduleBeginDate, scheduleEndDate, + false, "", "", false); + Assertions.assertThat(response.getStatusCode().is4xxClientError()).isTrue(); + } + @Test void notFoundTest(){ //When+Then @@ -279,6 +495,13 @@ void notFoundTest(){ Assertions.assertThat(response.getStatusCode().is4xxClientError()).isTrue(); } + @Test + void notFoundTest_collectionInstrumentId(){ + //When+Then + ResponseEntity response = dataProcessingContextController.setSurveyLastExecutionByCollectionInstrumentId("ERROR", LocalDateTime.now()); + Assertions.assertThat(response.getStatusCode().is4xxClientError()).isTrue(); + } + @Test void deleteScheduleTest(){ //When @@ -291,12 +514,23 @@ void deleteScheduleTest(){ } @Test - void deleteExpiredScheduleTest_execution() throws IOException, GenesisException { + void deleteScheduleTest_collectionInstrumentId(){ + //When + dataProcessingContextController.deleteSchedulesByCollectionInstrumentId("TESTSURVEY_CI"); + + //Then + Assertions.assertThat(dataProcessingContextPersistancePortStub.getMongoStub()).filteredOn(dataProcessingContextDocument -> + dataProcessingContextDocument.getPartitionId().equals("TESTSURVEY_CI") + ).isEmpty(); + } + + @Test + void deleteExpiredScheduleTest_execution() { //Given DataProcessingContextModel dataProcessingContextModel = new DataProcessingContextModel( null, "TESTSURVEYADDED", - "TESTSURVEYADDED", + null, null, new ArrayList<>(), false @@ -339,7 +573,55 @@ void deleteExpiredScheduleTest_execution() throws IOException, GenesisException } @Test - void deleteExpiredScheduleTest_wholeSurvey() throws IOException, GenesisException { + void deleteExpiredScheduleTest_execution_collectionInstrumentId() { + //Given + DataProcessingContextModel dataProcessingContextModel = new DataProcessingContextModel( + null, + null, + "TESTSURVEYADDED_CI", + null, + new ArrayList<>(), + false + ); + KraftwerkExecutionSchedule kraftwerkExecutionSchedule = new KraftwerkExecutionSchedule( + "0 0 6 * * *", + ServiceToCall.MAIN, + LocalDateTime.of(2000, Month.JANUARY, 1, 1, 1, 1), + LocalDateTime.of(2000, Month.DECEMBER, 1, 1, 1, 1), + null + ); + dataProcessingContextModel.getKraftwerkExecutionScheduleList().add(kraftwerkExecutionSchedule); + kraftwerkExecutionSchedule = new KraftwerkExecutionSchedule( + "0 0 6 * * *", + ServiceToCall.MAIN, + LocalDateTime.of(2023, Month.FEBRUARY, 1, 1, 1, 1), + LocalDateTime.of(5023, Month.DECEMBER, 1, 1, 1, 1), + null + ); + dataProcessingContextModel.getKraftwerkExecutionScheduleList().add(kraftwerkExecutionSchedule); + dataProcessingContextPersistancePortStub.getMongoStub().add(DataProcessingContextMapper.INSTANCE.modelToDocument(dataProcessingContextModel)); + + //When + dataProcessingContextController.deleteExpiredSchedules(); + + //Then + //Expired schedule deleted + Assertions.assertThat(dataProcessingContextPersistancePortStub.getMongoStub()).filteredOn(dataProcessingContextDocument -> + dataProcessingContextDocument.getCollectionInstrumentId().equals("TESTSURVEYADDED_CI") + ).isNotEmpty(); + Assertions.assertThat(dataProcessingContextPersistancePortStub.getMongoStub().stream().filter(dataProcessingContextDocument -> + dataProcessingContextDocument.getCollectionInstrumentId().equals("TESTSURVEYADDED_CI")).toList().getFirst().getKraftwerkExecutionScheduleList() + ).isNotEmpty().hasSize(1); + + //Expired schedule to log json file + Assertions.assertThat(Path.of(TestConstants.TEST_RESOURCES_DIRECTORY) + .resolve(Constants.SCHEDULE_ARCHIVE_FOLDER_NAME) + .resolve("TESTSURVEYADDED_CI.json") + .toFile()).exists().content().isNotEmpty().contains("2000").doesNotContain("5023"); + } + + @Test + void deleteExpiredScheduleTest_wholeSurvey() { //Given DataProcessingContextModel dataProcessingContextModel = new DataProcessingContextModel( null, @@ -388,7 +670,56 @@ void deleteExpiredScheduleTest_wholeSurvey() throws IOException, GenesisExceptio } @Test - void deleteExpiredScheduleTest_appendLog() throws IOException, GenesisException { + void deleteExpiredScheduleTest_wholeSurvey_collectionInstrumentId() { + //Given + DataProcessingContextModel dataProcessingContextModel = new DataProcessingContextModel( + null, + null, + "TESTSURVEYADDED_CI", + null, + new ArrayList<>(), + false + ); + KraftwerkExecutionSchedule kraftwerkExecutionSchedule = new KraftwerkExecutionSchedule( + "0 0 6 * * *", + ServiceToCall.MAIN, + LocalDateTime.of(2001, Month.JANUARY, 1, 1, 1, 1), + LocalDateTime.of(2001, Month.DECEMBER, 1, 1, 1, 1), + null + ); + dataProcessingContextModel.getKraftwerkExecutionScheduleList().add(kraftwerkExecutionSchedule); + kraftwerkExecutionSchedule = new KraftwerkExecutionSchedule( + "0 0 6 * * *", + ServiceToCall.MAIN, + LocalDateTime.of(2002, Month.FEBRUARY, 1, 1, 1, 1), + LocalDateTime.of(2002, Month.DECEMBER, 1, 1, 1, 1), + null + ); + dataProcessingContextModel.getKraftwerkExecutionScheduleList().add(kraftwerkExecutionSchedule); + dataProcessingContextPersistancePortStub.getMongoStub().add(DataProcessingContextMapper.INSTANCE.modelToDocument(dataProcessingContextModel)); + + //When + dataProcessingContextController.deleteExpiredSchedules(); + + //Then + //Expired schedule deleted + Assertions.assertThat(dataProcessingContextPersistancePortStub.getMongoStub()).filteredOn(dataProcessingContextDocument -> + dataProcessingContextDocument.getCollectionInstrumentId().equals("TESTSURVEYADDED_CI") + ).hasSize(1); + Assertions.assertThat(dataProcessingContextPersistancePortStub.getMongoStub().stream().filter(dataProcessingContextDocument -> + dataProcessingContextDocument.getCollectionInstrumentId().equals("TESTSURVEYADDED_CI")).toList().getFirst().getKraftwerkExecutionScheduleList() + ).isEmpty(); + + + //Expired schedule to log json file + Assertions.assertThat(Path.of(TestConstants.TEST_RESOURCES_DIRECTORY) + .resolve(Constants.SCHEDULE_ARCHIVE_FOLDER_NAME) + .resolve("TESTSURVEYADDED_CI.json") + .toFile()).exists().content().isNotEmpty().contains("2001","2002"); + } + + @Test + void deleteExpiredScheduleTest_appendLog() { //Given DataProcessingContextModel dataProcessingContextModel = new DataProcessingContextModel( null, @@ -446,18 +777,75 @@ void deleteExpiredScheduleTest_appendLog() throws IOException, GenesisException .toFile()).exists().content().isNotEmpty().contains("2000","2001"); } + @Test + void deleteExpiredScheduleTest_appendLog_collectionInstrumentId() { + //Given + DataProcessingContextModel dataProcessingContextModel = new DataProcessingContextModel( + null, + null, + "TESTSURVEYADDED2_CI", + null, + new ArrayList<>(), + false + ); + KraftwerkExecutionSchedule kraftwerkExecutionSchedule = new KraftwerkExecutionSchedule( + "0 0 6 * * *", + ServiceToCall.MAIN, + LocalDateTime.of(2000, Month.JANUARY, 1, 1, 1, 1), + LocalDateTime.of(2000, Month.DECEMBER, 1, 1, 1, 1), + null + ); + dataProcessingContextModel.getKraftwerkExecutionScheduleList().add(kraftwerkExecutionSchedule); + kraftwerkExecutionSchedule = new KraftwerkExecutionSchedule( + "0 0 6 * * *", + ServiceToCall.MAIN, + LocalDateTime.of(2023, Month.FEBRUARY, 1, 1, 1, 1), + LocalDateTime.of(5023, Month.DECEMBER, 1, 1, 1, 1), + null + ); + dataProcessingContextModel.getKraftwerkExecutionScheduleList().add(kraftwerkExecutionSchedule); + dataProcessingContextPersistancePortStub.getMongoStub().add(DataProcessingContextMapper.INSTANCE.modelToDocument(dataProcessingContextModel)); + + //When + dataProcessingContextController.deleteExpiredSchedules(); + kraftwerkExecutionSchedule = new KraftwerkExecutionSchedule( + "0 0 6 * * *", + ServiceToCall.MAIN, + LocalDateTime.of(2001, Month.FEBRUARY, 1, 1, 1, 1), + LocalDateTime.of(2001, Month.DECEMBER, 1, 1, 1, 1), + null + ); + dataProcessingContextModel.getKraftwerkExecutionScheduleList().add(kraftwerkExecutionSchedule); + dataProcessingContextPersistancePortStub.getMongoStub().add(DataProcessingContextMapper.INSTANCE.modelToDocument(dataProcessingContextModel)); + dataProcessingContextController.deleteExpiredSchedules(); + + //Then + //Expired schedules deleted + Assertions.assertThat(dataProcessingContextPersistancePortStub.getMongoStub()).filteredOn(dataProcessingContextDocument -> + dataProcessingContextDocument.getCollectionInstrumentId().equals("TESTSURVEYADDED2_CI") + ).isNotEmpty(); + Assertions.assertThat(dataProcessingContextPersistancePortStub.getMongoStub().stream().filter(dataProcessingContextDocument -> + dataProcessingContextDocument + .getCollectionInstrumentId().equals("TESTSURVEYADDED2_CI")).toList().getFirst().getKraftwerkExecutionScheduleList() + ).isNotEmpty().hasSize(1); + + //Expired schedules to only one log json file + Assertions.assertThat(Path.of(TestConstants.TEST_RESOURCES_DIRECTORY) + .resolve(Constants.SCHEDULE_ARCHIVE_FOLDER_NAME) + .resolve("TESTSURVEYADDED2_CI.json") + .toFile()).exists().content().isNotEmpty().contains("2000","2001"); + } + @ParameterizedTest @ValueSource(booleans = {false, true}) void getReview_test(boolean withReview){ //GIVEN String partitionId = "TESTPARTITION"; - dataProcessingContextPersistancePortStub.getMongoStub().add( - new DataProcessingContextDocument( - "TESTPARTITION", - new ArrayList<>(), - withReview - ) - ); + DataProcessingContextDocument doc = new DataProcessingContextDocument(); + doc.setPartitionId("TESTPARTITION"); + doc.setKraftwerkExecutionScheduleList(new ArrayList<>()); + doc.setWithReview(withReview); + dataProcessingContextPersistancePortStub.getMongoStub().add(doc); //WHEN ResponseEntity response = dataProcessingContextController.getReviewIndicator(partitionId); @@ -468,6 +856,26 @@ void getReview_test(boolean withReview){ Assertions.assertThat((Boolean) response.getBody()).isEqualTo(withReview); } + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void getReview_test_collectionInstrumentId(boolean withReview){ + //GIVEN + String collectionInstrumentId = "TESTPARTITION_CI"; + DataProcessingContextDocument doc = new DataProcessingContextDocument(); + doc.setCollectionInstrumentId("TESTPARTITION_CI"); + doc.setKraftwerkExecutionScheduleList(new ArrayList<>()); + doc.setWithReview(withReview); + dataProcessingContextPersistancePortStub.getMongoStub().add(doc); + + //WHEN + ResponseEntity response = dataProcessingContextController.getReviewIndicatorByCollectionInstrumentId(collectionInstrumentId); + + //THEN + Assertions.assertThat(response.getStatusCode().value()).isEqualTo(200); + Assertions.assertThat(response.getBody().getClass()).isEqualTo(Boolean.class); + Assertions.assertThat((Boolean) response.getBody()).isEqualTo(withReview); + } + @Test void getReview_no_context_test(){ //WHEN @@ -479,14 +887,49 @@ void getReview_no_context_test(){ Assertions.assertThat(response.getStatusCode().value()).isEqualTo(404); } + @Test + void getReview_no_context_test_collectionInstrumentId(){ + //WHEN + ResponseEntity response = dataProcessingContextController.getReviewIndicatorByCollectionInstrumentId( + "TESTPARTITIONIDNOCONTEXT_CI" + ); + + //THEN + Assertions.assertThat(response.getStatusCode().value()).isEqualTo(404); + } + //UTILITY private void addNewDocumentToStub() { - DataProcessingContextDocument dataProcessingContextDocumentTest = new DataProcessingContextDocument( - "TESTSURVEY", - new ArrayList<>(), - false + DataProcessingContextDocument dataProcessingContextDocumentTest = new DataProcessingContextDocument(); + dataProcessingContextDocumentTest.setPartitionId("TESTSURVEY"); + dataProcessingContextDocumentTest.setKraftwerkExecutionScheduleList(new ArrayList<>()); + dataProcessingContextDocumentTest.setWithReview(false); + + KraftwerkExecutionSchedule kraftwerkExecutionSchedule = new KraftwerkExecutionSchedule( + "0 0 6 * * *", + ServiceToCall.MAIN, + LocalDateTime.of(2023, Month.JANUARY, 1, 1, 1, 1), + LocalDateTime.of(2023, Month.DECEMBER, 1, 1, 1, 1), + null ); + dataProcessingContextDocumentTest.getKraftwerkExecutionScheduleList().add(kraftwerkExecutionSchedule); + kraftwerkExecutionSchedule = new KraftwerkExecutionSchedule( + "0 0 6 * * *", + ServiceToCall.MAIN, + LocalDateTime.of(2023, Month.FEBRUARY, 1, 1, 1, 1), + LocalDateTime.of(2023, Month.DECEMBER, 1, 1, 1, 1), + null + ); + dataProcessingContextDocumentTest.getKraftwerkExecutionScheduleList().add(kraftwerkExecutionSchedule); + dataProcessingContextPersistancePortStub.getMongoStub().add(dataProcessingContextDocumentTest); + } + + private void addNewDocumentToStubWithCollectionInstrumentId() { + DataProcessingContextDocument dataProcessingContextDocumentTest = new DataProcessingContextDocument(); + dataProcessingContextDocumentTest.setCollectionInstrumentId("TESTSURVEY_CI"); + dataProcessingContextDocumentTest.setKraftwerkExecutionScheduleList(new ArrayList<>()); + dataProcessingContextDocumentTest.setWithReview(false); KraftwerkExecutionSchedule kraftwerkExecutionSchedule = new KraftwerkExecutionSchedule( "0 0 6 * * *", ServiceToCall.MAIN, diff --git a/src/test/java/fr/insee/genesis/controller/rest/responses/QuestionnaireControllerTest.java b/src/test/java/fr/insee/genesis/controller/rest/responses/QuestionnaireControllerTest.java index 3906a388..6e5e24ea 100644 --- a/src/test/java/fr/insee/genesis/controller/rest/responses/QuestionnaireControllerTest.java +++ b/src/test/java/fr/insee/genesis/controller/rest/responses/QuestionnaireControllerTest.java @@ -77,26 +77,22 @@ void getQuestionnairesWithReviewTest() { Assertions.assertThat(response.getStatusCode().is2xxSuccessful()).isTrue(); Assertions.assertThat(response.getBody()).isNotNull().isEmpty(); - dataProcessingContextPersistancePortStub.getMongoStub().add( - new DataProcessingContextDocument( - DEFAULT_QUESTIONNAIRE_ID, - new ArrayList<>(), - true - ) - ); + DataProcessingContextDocument doc = new DataProcessingContextDocument(); + doc.setPartitionId(DEFAULT_QUESTIONNAIRE_ID); + doc.setKraftwerkExecutionScheduleList(new ArrayList<>()); + doc.setWithReview(true); + dataProcessingContextPersistancePortStub.getMongoStub().add(doc); response = questionnaireControllerStatic.getQuestionnairesWithReview(true); Assertions.assertThat(response.getStatusCode().is2xxSuccessful()).isTrue(); Assertions.assertThat(response.getBody()).isNotNull().isNotEmpty().containsOnly( DEFAULT_QUESTIONNAIRE_ID); - dataProcessingContextPersistancePortStub.getMongoStub().add( - new DataProcessingContextDocument( - questionnaireId, - new ArrayList<>(), - false - ) - ); + DataProcessingContextDocument doc2 = new DataProcessingContextDocument(); + doc2.setPartitionId(questionnaireId); + doc2.setKraftwerkExecutionScheduleList(new ArrayList<>()); + doc2.setWithReview(false); + dataProcessingContextPersistancePortStub.getMongoStub().add(doc2); response = questionnaireControllerStatic.getQuestionnairesWithReview(true); Assertions.assertThat(response.getStatusCode().is2xxSuccessful()).isTrue(); @@ -114,26 +110,22 @@ void getQuestionnairesWithoutReviewTest() { Assertions.assertThat(response.getStatusCode().is2xxSuccessful()).isTrue(); Assertions.assertThat(response.getBody()).isNotNull().isEmpty(); - dataProcessingContextPersistancePortStub.getMongoStub().add( - new DataProcessingContextDocument( - DEFAULT_QUESTIONNAIRE_ID, - new ArrayList<>(), - false - ) - ); + DataProcessingContextDocument doc = new DataProcessingContextDocument(); + doc.setPartitionId(DEFAULT_QUESTIONNAIRE_ID); + doc.setKraftwerkExecutionScheduleList(new ArrayList<>()); + doc.setWithReview(false); + dataProcessingContextPersistancePortStub.getMongoStub().add(doc); response = questionnaireControllerStatic.getQuestionnairesWithReview(false); Assertions.assertThat(response.getStatusCode().is2xxSuccessful()).isTrue(); Assertions.assertThat(response.getBody()).isNotNull().isNotEmpty().containsOnly( DEFAULT_QUESTIONNAIRE_ID); - dataProcessingContextPersistancePortStub.getMongoStub().add( - new DataProcessingContextDocument( - questionnaireId, - new ArrayList<>(), - true - ) - ); + DataProcessingContextDocument doc2 = new DataProcessingContextDocument(); + doc2.setPartitionId(questionnaireId); + doc2.setKraftwerkExecutionScheduleList(new ArrayList<>()); + doc2.setWithReview(true); + dataProcessingContextPersistancePortStub.getMongoStub().add(doc2); response = questionnaireControllerStatic.getQuestionnairesWithReview(false); Assertions.assertThat(response.getStatusCode().is2xxSuccessful()).isTrue(); Assertions.assertThat(response.getBody()).isNotNull().isNotEmpty().containsOnly( diff --git a/src/test/java/fr/insee/genesis/controller/rest/responses/ResponseControllerTest.java b/src/test/java/fr/insee/genesis/controller/rest/responses/ResponseControllerTest.java index eee52879..0a01d0a0 100644 --- a/src/test/java/fr/insee/genesis/controller/rest/responses/ResponseControllerTest.java +++ b/src/test/java/fr/insee/genesis/controller/rest/responses/ResponseControllerTest.java @@ -228,10 +228,11 @@ void getLatestByStatesSurveyDataTest() throws GenesisException { surveyUnitPersistencePortStub ); - dataProcessingContextPersistancePortStub.getMongoStub().add(new DataProcessingContextDocument( - "TEST-TABLEAUX", new ArrayList<>(), true - - )); + DataProcessingContextDocument doc = new DataProcessingContextDocument(); + doc.setPartitionId("TEST-TABLEAUX"); + doc.setKraftwerkExecutionScheduleList(new ArrayList<>()); + doc.setWithReview(true); + dataProcessingContextPersistancePortStub.getMongoStub().add(doc); //WHEN @@ -322,10 +323,11 @@ void getLatestByStatesSurveyDataTest_invalid_collected() throws GenesisException surveyUnitPersistencePortStub ); - dataProcessingContextPersistancePortStub.getMongoStub().add(new DataProcessingContextDocument( - "TEST-TABLEAUX", new ArrayList<>(), true - - )); + DataProcessingContextDocument doc = new DataProcessingContextDocument(); + doc.setPartitionId("TEST-TABLEAUX"); + doc.setKraftwerkExecutionScheduleList(new ArrayList<>()); + doc.setWithReview(true); + dataProcessingContextPersistancePortStub.getMongoStub().add(doc); //WHEN diff --git a/src/test/java/fr/insee/genesis/domain/service/context/DataProcessingContextServiceTest.java b/src/test/java/fr/insee/genesis/domain/service/context/DataProcessingContextServiceTest.java index 691171e9..c7e93a97 100644 --- a/src/test/java/fr/insee/genesis/domain/service/context/DataProcessingContextServiceTest.java +++ b/src/test/java/fr/insee/genesis/domain/service/context/DataProcessingContextServiceTest.java @@ -6,6 +6,8 @@ import fr.insee.genesis.domain.model.surveyunit.SurveyUnitModel; import fr.insee.genesis.exceptions.GenesisException; import fr.insee.genesis.infrastructure.document.context.DataProcessingContextDocument; +import fr.insee.genesis.infrastructure.utils.FileUtils; +import fr.insee.genesis.stubs.ConfigStub; import fr.insee.genesis.stubs.DataProcessingContextPersistancePortStub; import fr.insee.genesis.stubs.SurveyUnitPersistencePortStub; import org.assertj.core.api.Assertions; @@ -24,6 +26,7 @@ class DataProcessingContextServiceTest { static SurveyUnitPersistencePortStub surveyUnitPersistencePortStub; static DataProcessingContextService dataProcessingContextService; static DataProcessingContextPersistancePortStub dataProcessingContextPersistencePortStub; + FileUtils fileUtils = new FileUtils(new ConfigStub()); @BeforeAll static void init(){ @@ -46,11 +49,11 @@ void reset(){ LocalDateTime.MAX, null )); - dataProcessingContextPersistencePortStub.getMongoStub().add(new DataProcessingContextDocument( - "TEST", - kraftwerkExecutionScheduleList, - false - )); + DataProcessingContextDocument doc = new DataProcessingContextDocument(); + doc.setPartitionId("TEST"); + doc.setKraftwerkExecutionScheduleList(kraftwerkExecutionScheduleList); + doc.setWithReview(false); + dataProcessingContextPersistencePortStub.getMongoStub().add(doc); } @Test @@ -146,7 +149,7 @@ void removeExpiredSchedules_test_existing_schedule() throws GenesisException { )); //When - dataProcessingContextService.deleteExpiredSchedules("TEST"); + dataProcessingContextService.deleteExpiredSchedules(fileUtils.getLogFolder()); //Then //Execution schedule deleted @@ -167,14 +170,14 @@ void removeExpiredSchedules_test_delete_document() throws GenesisException { LocalDateTime.of(2000,1,1,1,1,1), null )); - dataProcessingContextPersistencePortStub.getMongoStub().add(new DataProcessingContextDocument( - "TEST2", - kraftwerkExecutionScheduleList, - false - )); + DataProcessingContextDocument doc = new DataProcessingContextDocument(); + doc.setPartitionId("TEST2"); + doc.setKraftwerkExecutionScheduleList(kraftwerkExecutionScheduleList); + doc.setWithReview(false); + dataProcessingContextPersistencePortStub.getMongoStub().add(doc); //When - dataProcessingContextService.deleteExpiredSchedules("TEST2"); + dataProcessingContextService.deleteExpiredSchedules(fileUtils.getLogFolder()); //Then //Survey schedule document deleted diff --git a/src/test/java/fr/insee/genesis/infrastructure/utils/context/ContextDedupUtilsTest.java b/src/test/java/fr/insee/genesis/infrastructure/utils/context/ContextDedupUtilsTest.java index e62ecf0a..daabe6c7 100644 --- a/src/test/java/fr/insee/genesis/infrastructure/utils/context/ContextDedupUtilsTest.java +++ b/src/test/java/fr/insee/genesis/infrastructure/utils/context/ContextDedupUtilsTest.java @@ -31,11 +31,11 @@ void emptyListTest() { void oneElementListTest() { //Given List dataProcessingContextDocuments = new ArrayList<>(); - DataProcessingContextDocument existingDocument = new DataProcessingContextDocument( - partitionId, - new ArrayList<>(), - false - ); + DataProcessingContextDocument existingDocument = new DataProcessingContextDocument(); + existingDocument.setPartitionId(partitionId); + existingDocument.setKraftwerkExecutionScheduleList(new ArrayList<>()); + existingDocument.setWithReview(false); + dataProcessingContextDocuments.add(existingDocument); existingDocument.getKraftwerkExecutionScheduleList().add( new KraftwerkExecutionSchedule( @@ -61,11 +61,11 @@ void oneElementListTest() { void multipleElementsListTest_both() { //Given List dataProcessingContextDocuments = new ArrayList<>(); - DataProcessingContextDocument existingDocument = new DataProcessingContextDocument( - partitionId, - new ArrayList<>(), - false - ); + DataProcessingContextDocument existingDocument = new DataProcessingContextDocument(); + existingDocument.setPartitionId(partitionId); + existingDocument.setKraftwerkExecutionScheduleList(new ArrayList<>()); + existingDocument.setWithReview(false); + existingDocument.getKraftwerkExecutionScheduleList().add( new KraftwerkExecutionSchedule( "0 0 0 * * *", @@ -77,11 +77,11 @@ void multipleElementsListTest_both() { ); dataProcessingContextDocuments.add(existingDocument); - existingDocument = new DataProcessingContextDocument( - partitionId, - new ArrayList<>(), - true - ); + existingDocument = new DataProcessingContextDocument(); + existingDocument.setPartitionId(partitionId); + existingDocument.setKraftwerkExecutionScheduleList(new ArrayList<>()); + existingDocument.setWithReview(true); + existingDocument.getKraftwerkExecutionScheduleList().add( new KraftwerkExecutionSchedule( "0 0 6 * * *", @@ -109,11 +109,11 @@ void multipleElementsListTest_both() { void multipleElementsListTest_true() { //Given List dataProcessingContextDocuments = new ArrayList<>(); - DataProcessingContextDocument existingDocument = new DataProcessingContextDocument( - partitionId, - new ArrayList<>(), - true - ); + DataProcessingContextDocument existingDocument = new DataProcessingContextDocument(); + existingDocument.setPartitionId(partitionId); + existingDocument.setKraftwerkExecutionScheduleList(new ArrayList<>()); + existingDocument.setWithReview(true); + existingDocument.getKraftwerkExecutionScheduleList().add( new KraftwerkExecutionSchedule( "0 0 0 * * *", @@ -125,11 +125,11 @@ void multipleElementsListTest_true() { ); dataProcessingContextDocuments.add(existingDocument); - existingDocument = new DataProcessingContextDocument( - partitionId, - new ArrayList<>(), - true - ); + existingDocument = new DataProcessingContextDocument(); + existingDocument.setPartitionId(partitionId); + existingDocument.setKraftwerkExecutionScheduleList(new ArrayList<>()); + existingDocument.setWithReview(true); + existingDocument.getKraftwerkExecutionScheduleList().add( new KraftwerkExecutionSchedule( "0 0 6 * * *", @@ -157,11 +157,11 @@ void multipleElementsListTest_true() { void duplicateScheduleListTest() { //Given List dataProcessingContextDocuments = new ArrayList<>(); - DataProcessingContextDocument existingDocument = new DataProcessingContextDocument( - partitionId, - new ArrayList<>(), - false - ); + DataProcessingContextDocument existingDocument = new DataProcessingContextDocument(); + existingDocument.setPartitionId(partitionId); + existingDocument.setKraftwerkExecutionScheduleList(new ArrayList<>()); + existingDocument.setWithReview(false); + existingDocument.getKraftwerkExecutionScheduleList().add( new KraftwerkExecutionSchedule( "0 0 0 * * *", @@ -182,11 +182,11 @@ void duplicateScheduleListTest() { ); dataProcessingContextDocuments.add(existingDocument); - existingDocument = new DataProcessingContextDocument( - partitionId, - new ArrayList<>(), - false - ); + existingDocument = new DataProcessingContextDocument(); + existingDocument.setPartitionId(partitionId); + existingDocument.setKraftwerkExecutionScheduleList(new ArrayList<>()); + existingDocument.setWithReview(false); + existingDocument.getKraftwerkExecutionScheduleList().add( new KraftwerkExecutionSchedule( "0 0 0 * * *", @@ -212,11 +212,11 @@ void duplicateScheduleListTest() { void duplicateAllContextsTest() { //Given List dataProcessingContextDocuments = new ArrayList<>(); - DataProcessingContextDocument existingDocument = new DataProcessingContextDocument( - partitionId, - new ArrayList<>(), - false - ); + DataProcessingContextDocument existingDocument = new DataProcessingContextDocument(); + existingDocument.setPartitionId(partitionId); + existingDocument.setKraftwerkExecutionScheduleList(new ArrayList<>()); + existingDocument.setWithReview(false); + existingDocument.getKraftwerkExecutionScheduleList().add( new KraftwerkExecutionSchedule( "0 0 0 * * *", @@ -246,11 +246,11 @@ void duplicateAllContextsTest() { ); dataProcessingContextDocuments.add(existingDocument); - existingDocument = new DataProcessingContextDocument( - partitionId, - new ArrayList<>(), - false - ); + existingDocument = new DataProcessingContextDocument(); + existingDocument.setPartitionId(partitionId); + existingDocument.setKraftwerkExecutionScheduleList(new ArrayList<>()); + existingDocument.setWithReview(false); + existingDocument.getKraftwerkExecutionScheduleList().add( new KraftwerkExecutionSchedule( "0 0 0 * * *", @@ -262,11 +262,11 @@ void duplicateAllContextsTest() { ); dataProcessingContextDocuments.add(existingDocument); - existingDocument = new DataProcessingContextDocument( - partitionId + "_2", - new ArrayList<>(), - false - ); + existingDocument = new DataProcessingContextDocument(); + existingDocument.setPartitionId(partitionId + "_2"); + existingDocument.setKraftwerkExecutionScheduleList(new ArrayList<>()); + existingDocument.setWithReview(false); + existingDocument.getKraftwerkExecutionScheduleList().add( new KraftwerkExecutionSchedule( "0 0 0 * * *", @@ -286,11 +286,11 @@ void duplicateAllContextsTest() { ) ); dataProcessingContextDocuments.add(existingDocument); - existingDocument = new DataProcessingContextDocument( - partitionId + "_3", - new ArrayList<>(), - false - ); + existingDocument = new DataProcessingContextDocument(); + existingDocument.setPartitionId(partitionId + "_3"); + existingDocument.setKraftwerkExecutionScheduleList(new ArrayList<>()); + existingDocument.setWithReview(false); + existingDocument.getKraftwerkExecutionScheduleList().add( new KraftwerkExecutionSchedule( "0 0 6 * * *", @@ -301,11 +301,11 @@ void duplicateAllContextsTest() { ) ); dataProcessingContextDocuments.add(existingDocument); - existingDocument = new DataProcessingContextDocument( - partitionId + "_4", - new ArrayList<>(), - false - ); + existingDocument = new DataProcessingContextDocument(); + existingDocument.setPartitionId(partitionId + "_4"); + existingDocument.setKraftwerkExecutionScheduleList(new ArrayList<>()); + existingDocument.setWithReview(false); + dataProcessingContextDocuments.add(existingDocument); diff --git a/src/test/java/fr/insee/genesis/stubs/DataProcessingContextPersistancePortStub.java b/src/test/java/fr/insee/genesis/stubs/DataProcessingContextPersistancePortStub.java index 6326e619..528e4a33 100644 --- a/src/test/java/fr/insee/genesis/stubs/DataProcessingContextPersistancePortStub.java +++ b/src/test/java/fr/insee/genesis/stubs/DataProcessingContextPersistancePortStub.java @@ -38,15 +38,31 @@ public List findByPartitionIds(List partitio ); } + @Override + public DataProcessingContextModel findByCollectionInstrumentId(String collectionInstrumentId) { + return DataProcessingContextMapper.INSTANCE.documentToModel(ContextDedupUtils.deduplicateContexts(collectionInstrumentId, + mongoStub.stream().filter( + dataProcessingContextDocument -> dataProcessingContextDocument.getCollectionInstrumentId().equals(collectionInstrumentId) + ).toList()) + ); + } + @Override public List findByCollectionInstrumentIds(List collectionInstrumentIds) { - return List.of(); + return DataProcessingContextMapper.INSTANCE.listDocumentToListModel( + ContextDedupUtils.deduplicateContexts( + mongoStub.stream().filter( + dataProcessingContextDocument -> collectionInstrumentIds.contains(dataProcessingContextDocument.getCollectionInstrumentId()) + ).toList() + ) + ); } @Override public void save(DataProcessingContextDocument dataProcessingContextDocument) { - mongoStub.removeIf(existingDoc -> existingDoc.getPartitionId().equals( - dataProcessingContextDocument.getPartitionId()) + mongoStub.removeIf(existingDoc -> (existingDoc.getCollectionInstrumentId() != null &&existingDoc.getCollectionInstrumentId().equals( + dataProcessingContextDocument.getCollectionInstrumentId())) ||(existingDoc.getPartitionId()!= null &&existingDoc.getPartitionId().equals( + dataProcessingContextDocument.getPartitionId())) ); mongoStub.add(dataProcessingContextDocument); } @@ -83,7 +99,7 @@ public List removeExpiredSchedules(DataProcessingCon kraftwerkExecutionScheduleToRemove.getScheduleEndDate()); } //Update mongo stub - mongoStub.removeIf(scheduleDocument -> scheduleDocument.getPartitionId().equals(dataProcessingContextModel.getPartitionId())); + mongoStub.removeIf(scheduleDocument -> (scheduleDocument.getCollectionInstrumentId()!=null && scheduleDocument.getCollectionInstrumentId().equals(dataProcessingContextModel.getCollectionInstrumentId())) ||(scheduleDocument.getPartitionId()!=null && scheduleDocument.getPartitionId().equals(dataProcessingContextModel.getPartitionId()))); mongoStub.add(DataProcessingContextMapper.INSTANCE.modelToDocument(dataProcessingContextModel)); return kraftwerkExecutionSchedulesToRemove; } From 89ce666408b1c5d075faa73f240b5cc17c07509a Mon Sep 17 00:00:00 2001 From: QDIBYS Date: Wed, 17 Dec 2025 10:20:48 +0100 Subject: [PATCH 41/42] fix: continue if states null or empty --- .../genesis/domain/service/rawdata/RawResponseService.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/main/java/fr/insee/genesis/domain/service/rawdata/RawResponseService.java b/src/main/java/fr/insee/genesis/domain/service/rawdata/RawResponseService.java index 002db961..8fcdcded 100644 --- a/src/main/java/fr/insee/genesis/domain/service/rawdata/RawResponseService.java +++ b/src/main/java/fr/insee/genesis/domain/service/rawdata/RawResponseService.java @@ -366,6 +366,10 @@ private static void convertToCollectedVar( Map states = JsonUtils.asMap(collectedVariable.getValue()); // nothing if no state + if (states == null || states.isEmpty()) { + continue; + } + if (states.containsKey(stateKey)) { Object value = states.get(stateKey); From 0742eda4afcfe6234353d48b79ebc22163877640 Mon Sep 17 00:00:00 2001 From: QDIBYS Date: Wed, 17 Dec 2025 10:24:31 +0100 Subject: [PATCH 42/42] refactor : simplify condition --- .../genesis/domain/service/rawdata/RawResponseService.java | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/main/java/fr/insee/genesis/domain/service/rawdata/RawResponseService.java b/src/main/java/fr/insee/genesis/domain/service/rawdata/RawResponseService.java index 8fcdcded..6f343237 100644 --- a/src/main/java/fr/insee/genesis/domain/service/rawdata/RawResponseService.java +++ b/src/main/java/fr/insee/genesis/domain/service/rawdata/RawResponseService.java @@ -365,12 +365,7 @@ private static void convertToCollectedVar( // Map for this variable (COLLECTED/EDITED -> value) Map states = JsonUtils.asMap(collectedVariable.getValue()); - // nothing if no state - if (states == null || states.isEmpty()) { - continue; - } - - if (states.containsKey(stateKey)) { + if (states != null && states.containsKey(stateKey)) { Object value = states.get(stateKey); // liste ?