Integration of the modelefiliere library

.mvn/wrapper/maven-wrapper.properties

@@ -0,0 +1,2 @@
+distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.9.11/apache-maven-3.9.11-bin.zip
+wrapperUrl=https://repo.maven.apache.org/maven2/org/apache/maven/wrapper/maven-wrapper/3.3.4/maven-wrapper-3.3.4.jar

pom.xml

@@ -98,7 +98,16 @@
             <artifactId>springdoc-openapi-starter-webmvc-ui</artifactId>
             <version>${springdoc.version}</version>
         </dependency>
-
+        <dependency>
+            <groupId>fr.insee</groupId>
+            <artifactId>modelefiliere</artifactId>
+            <version>2.0.0</version>
+        </dependency>
+        <dependency>
+            <groupId>com.networknt</groupId>
+            <artifactId>json-schema-validator</artifactId>
+            <version>1.5.9</version>
+        </dependency>
         <!-- XML libraries -->
         <!-- XML-XSLT with Saxon -->
         <dependency>
@@ -107,13 +116,6 @@
             <version>12.9</version>
         </dependency>
 
-        <!-- JSON -->
-        <dependency>
-            <groupId>com.networknt</groupId>
-            <artifactId>json-schema-validator</artifactId>
-            <version>2.0.0</version>
-        </dependency>
-
         <!-- generate implementation auto -->
         <dependency>
             <groupId>org.mapstruct</groupId>

src/main/java/fr/insee/genesis/Constants.java

@@ -23,8 +23,7 @@ public class Constants {
     public static final String MONGODB_CONTEXTUAL_EXTERNAL_COLLECTION_NAME = "editedExternal";
     public static final String LOOP_NAME_PREFIX = "BOUCLE";
     public static final String MONGODB_RESPONSE_COLLECTION_NAME = "responses";
-    public static final String MONGODB_RESPONSE_RAW_COLLECTION_NAME = "lunaticjsondata";
-    public static final String MONGODB_VARIABLETYPE_COLLECTION_NAME = "variabletypes";
+    public static final String MONGODB_RAW_RESPONSES_COLLECTION_NAME = "rawResponses";
     public static final String VOLUMETRY_FOLDER_NAME = "genesis_volumetries";
     public static final String VOLUMETRY_FILE_SUFFIX = "_VOLUMETRY";
     public static final String VOLUMETRY_RAW_FILE_SUFFIX = "_RAW_VOLUMETRY";

src/main/java/fr/insee/genesis/controller/adapter/LunaticJsonAdapter.java

@@ -13,7 +13,7 @@ public class LunaticJsonAdapter {
 
     public SurveyUnitModel convert(LunaticJsonSurveyUnit su){
         return SurveyUnitModel.builder()
-                .questionnaireId(su.getQuestionnaireId())
+                .collectionInstrumentId(su.getQuestionnaireId())
                 .campaignId("")
                 .interrogationId(su.getInterrogationId())
                 .state(DataState.COLLECTED)

src/main/java/fr/insee/genesis/controller/adapter/LunaticXmlAdapter.java

@@ -69,7 +69,7 @@ public static List<SurveyUnitModel> convert(LunaticXmlSurveyUnit su, VariablesMa
      */
     private static SurveyUnitModel getStateDataFromSurveyUnit(LunaticXmlSurveyUnit su, VariablesMap variablesMap, String campaignId, DataState dataState, Mode mode) {
         SurveyUnitModel surveyUnitModel = SurveyUnitModel.builder()
-                .questionnaireId(su.getQuestionnaireModelId().toUpperCase())
+                .collectionInstrumentId(su.getQuestionnaireModelId().toUpperCase())
                 .campaignId(campaignId)
                 .interrogationId(su.getId())
                 .state(dataState)

src/main/java/fr/insee/genesis/controller/dto/ScheduleDto.java

@@ -8,6 +8,7 @@
 
 @Builder
 public record ScheduleDto (String surveyName,
+						   String collectionInstrumentId,
 						   LocalDateTime lastExecution,
 						   List<KraftwerkExecutionSchedule> kraftwerkExecutionScheduleList
 ){}

src/main/java/fr/insee/genesis/controller/dto/SurveyUnitSimplified.java

@@ -11,10 +11,14 @@
 @Data
 public class SurveyUnitSimplified {
 
-	private String questionnaireId;
+	private String collectionInstrumentId;
+	/**
+	 * @deprecated We will not reveive this piece of information anymore
+	 */
+	@Deprecated(forRemoval = true, since =  "2026-01-01")
 	private String campaignId;
 	private String interrogationId;
-	private String surveyUnitId;
+	private String usualSurveyUnitId;
 	private Mode mode;
 	private List<VariableModel> variablesUpdate;
 	private List<VariableModel> externalVariables;

src/main/java/fr/insee/genesis/controller/mappers/DataProcessingContextMapperDto.java

@@ -23,4 +23,13 @@ public List<ScheduleDto> dataProcessingContextListToScheduleDtoList(List<DataPro
         }
         return dtos;
     }
+
+    public DataProcessingContextModel scheduleDtoToDataProcessingContext (ScheduleDto schedule){
+        return DataProcessingContextModel.builder()
+                .partitionId(schedule.surveyName())
+                .collectionInstrumentId(schedule.collectionInstrumentId())
+                .lastExecution(schedule.lastExecution())
+                .kraftwerkExecutionScheduleList(schedule.kraftwerkExecutionScheduleList())
+                .build();
+    }
 }

src/main/java/fr/insee/genesis/controller/rest/DataProcessingContextController.java

@@ -1,10 +1,7 @@
 package fr.insee.genesis.controller.rest;
 
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
 import fr.insee.genesis.Constants;
 import fr.insee.genesis.controller.dto.ScheduleDto;
-import fr.insee.genesis.domain.model.context.schedule.KraftwerkExecutionSchedule;
 import fr.insee.genesis.domain.model.context.schedule.ServiceToCall;
 import fr.insee.genesis.domain.model.context.schedule.TrustParameters;
 import fr.insee.genesis.domain.ports.api.DataProcessingContextApiPort;
@@ -21,31 +18,25 @@
 import org.springframework.stereotype.Controller;
 import org.springframework.web.bind.annotation.DeleteMapping;
 import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.PathVariable;
 import org.springframework.web.bind.annotation.PostMapping;
 import org.springframework.web.bind.annotation.PutMapping;
 import org.springframework.web.bind.annotation.RequestBody;
-import org.springframework.web.bind.annotation.RequestMapping;
 import org.springframework.web.bind.annotation.RequestParam;
 
-import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.StandardOpenOption;
 import java.time.LocalDateTime;
-import java.util.HashSet;
 import java.util.List;
-import java.util.Set;
 
-@RequestMapping(path = "/context" )
 @Controller
 @AllArgsConstructor
 @Slf4j
 public class DataProcessingContextController {
     private DataProcessingContextApiPort dataProcessingContextApiPort;
     private final FileUtils fileUtils;
 
+    @Deprecated(forRemoval = true)
     @Operation(summary = "Create or update a data processing context")
-    @PutMapping(path = "/review")
+    @PutMapping(path = "/context/review")
     @PreAuthorize("hasAnyRole('USER_PLATINE', 'USER_BACK_OFFICE', 'SCHEDULER')")
     public ResponseEntity<Object> saveContext(
             @Parameter(description = "Identifier of the partition", required = true) @RequestParam("partitionId") String partitionId,
@@ -60,8 +51,40 @@ public ResponseEntity<Object> saveContext(
         return ResponseEntity.ok().build();
     }
 
+    @Operation(summary = "Create or update a data processing context")
+    @PutMapping(path = "/contexts/{collectionInstrumentId}/review")
+    @PreAuthorize("hasAnyRole('USER_PLATINE', 'USER_BACK_OFFICE', 'SCHEDULER')")
+    public ResponseEntity<Object> saveContextWithCollectionInstrumentId(
+            @PathVariable("collectionInstrumentId") String collectionInstrumentId,
+            @Parameter(description = "Allow reviewing") @RequestParam(value = "withReview", defaultValue = "false") Boolean withReview
+    ){
+        try {
+            withReview = withReview != null && withReview; //False if null
+            dataProcessingContextApiPort.saveContextByCollectionInstrumentId(collectionInstrumentId, withReview);
+        }catch (GenesisException e){
+            return new ResponseEntity<>(e.getMessage(), HttpStatusCode.valueOf(e.getStatus()));
+        }
+        return ResponseEntity.ok().build();
+    }
+
+
+    @Operation(summary = "Returns partition review indicator")
+    @GetMapping(path = "/contexts/{collectionInstrumentId}/review")
+    @PreAuthorize("hasAnyRole('USER_BACK_OFFICE','SCHEDULER','USER_PLATINE')")
+    public ResponseEntity<Object> getReviewIndicatorByCollectionInstrumentId(
+            @PathVariable("collectionInstrumentId") String collectionInstrumentId
+    ){
+        try {
+            boolean withReview = dataProcessingContextApiPort.getReviewByCollectionInstrumentId(collectionInstrumentId);
+            return ResponseEntity.ok(withReview);
+        }catch (GenesisException e){
+            return new ResponseEntity<>(e.getMessage(), HttpStatusCode.valueOf(e.getStatus()));
+        }
+    }
+
+    @Deprecated(forRemoval = true)
     @Operation(summary = "Returns partition review indicator")
-    @GetMapping(path = "/review")
+    @GetMapping(path = "/context/review")
     @PreAuthorize("hasAnyRole('USER_BACK_OFFICE','SCHEDULER','USER_PLATINE')")
     public ResponseEntity<Object> getReviewIndicator(
             @Parameter(description = "Identifier of the partition", required = true) @RequestParam("partitionId") String partitionId
@@ -74,8 +97,9 @@ public ResponseEntity<Object> getReviewIndicator(
         }
     }
 
+    @Deprecated(forRemoval = true)
     @Operation(summary = "Schedule a Kraftwerk execution")
-    @PutMapping(path = "/schedules")
+    @PutMapping(path = "/context/schedules")
     @PreAuthorize("hasRole('USER_KRAFTWERK')")
     public ResponseEntity<Object> saveSchedule(
             @Parameter(description = "Partition identifier to call Kraftwerk on") @RequestParam("partitionId") String partitionId,
@@ -119,8 +143,55 @@ public ResponseEntity<Object> saveSchedule(
         return ResponseEntity.ok().build();
     }
 
+    // Should be refactored to make it restfull
+    @Operation(summary = "Schedule a Kraftwerk execution using the collection instrument")
+    @PutMapping(path = "/contexts/schedules")
+    @PreAuthorize("hasRole('USER_KRAFTWERK')")
+    public ResponseEntity<Object> saveScheduleWithCollectionInstrumentId(
+            @Parameter(description = "Collection instrument to call Kraftwerk on") @RequestParam("collectionInstrumentId") String collectionInstrumentId,
+            @Parameter(description = "Kraftwerk endpoint") @RequestParam(value = "serviceTocall", defaultValue = Constants.KRAFTWERK_MAIN_ENDPOINT) ServiceToCall serviceToCall,
+            @Parameter(description = "Frequency in Spring cron format (6 inputs, go to https://crontab.cronhub.io/ for generator)  \n Example : 0 0 6 * * *") @RequestParam("frequency") String frequency,
+            @Parameter(description = "Schedule effective date and time", example = "2024-01-01T12:00:00") @RequestParam("scheduleBeginDate") LocalDateTime scheduleBeginDate,
+            @Parameter(description = "Schedule end date and time", example = "2024-01-01T12:00:00") @RequestParam("scheduleEndDate") LocalDateTime scheduleEndDate,
+            @Parameter(description = "Encrypt after process ? Ignore next parameters if false") @RequestParam(value =
+                    "useEncryption",
+                    defaultValue = "false") boolean useEncryption,
+            @Parameter(description = "(Encryption) vault path") @RequestParam(value = "encryptionVaultPath", defaultValue = "") String encryptionVaultPath,
+            @Parameter(description = "(Encryption) output folder") @RequestParam(value = "encryptionOutputFolder",
+                    defaultValue = "") String encryptionOutputFolder,
+            @Parameter(description = "(Encryption) Use signature system") @RequestParam(value = "useSignature", defaultValue = "false") boolean useSignature
+    ) {
+        try {
+            //Check frequency
+            if(!CronExpression.isValidExpression(frequency)) {
+                log.warn("Returned error for wrong frequency : {}", frequency);
+                throw new GenesisException(400, "Wrong frequency syntax");
+            }
+
+            TrustParameters trustParameters = null;
+            if(useEncryption) {
+                trustParameters = new TrustParameters(
+                        fileUtils.getKraftwerkOutFolder(collectionInstrumentId),
+                        encryptionOutputFolder,
+                        encryptionVaultPath,
+                        useSignature
+                );
+            }
+            dataProcessingContextApiPort.saveKraftwerkExecutionScheduleByCollectionInstrumentId(
+                    collectionInstrumentId,
+                    serviceToCall == null ? ServiceToCall.MAIN : serviceToCall,
+                    frequency,
+                    scheduleBeginDate, scheduleEndDate, trustParameters
+            );
+        }catch (GenesisException e){
+            return new ResponseEntity<>(e.getMessage(), HttpStatusCode.valueOf(e.getStatus()));
+        }
+        return ResponseEntity.ok().build();
+    }
+
+    @Deprecated(forRemoval = true)
     @Operation(summary = "Fetch all schedules")
-    @GetMapping(path = "/schedules")
+    @GetMapping(path = "/context/schedules")
     @PreAuthorize("hasAnyRole('SCHEDULER','READER')")
     public ResponseEntity<Object> getAllSchedules() {
         log.debug("Got GET all schedules request");
@@ -131,8 +202,23 @@ public ResponseEntity<Object> getAllSchedules() {
         return ResponseEntity.ok(surveyScheduleDocumentModels);
     }
 
+    //It is just a change of path in the url
+    @Operation(summary = "Fetch all schedules")
+    @GetMapping(path = "/contexts/schedules")
+    @PreAuthorize("hasAnyRole('SCHEDULER','READER')")
+    public ResponseEntity<Object> getAllSchedulesV2() {
+        log.debug("Got GET all schedules request");
+
+        List<ScheduleDto> surveyScheduleDocumentModels = dataProcessingContextApiPort.getAllSchedules();
+
+        log.info("Returning {} schedule documents...", surveyScheduleDocumentModels.size());
+        return ResponseEntity.ok(surveyScheduleDocumentModels);
+    }
+
+
+    @Deprecated(forRemoval = true)
     @Operation(summary = "Set last execution date of a partition with new date or nothing")
-    @PostMapping(path = "/schedules/lastExecutionDate")
+    @PostMapping(path = "/context/schedules/lastExecutionDate")
     @PreAuthorize("hasRole('SCHEDULER')")
     public ResponseEntity<Object> setSurveyLastExecution(
             @Parameter(description = "Survey name to call Kraftwerk on") @RequestBody String partitionId,
@@ -147,8 +233,25 @@ public ResponseEntity<Object> setSurveyLastExecution(
         return ResponseEntity.ok().build();
     }
 
+    @Operation(summary = "Update the date of the last extraction of data corresponding to a collection instrument")
+    @PutMapping(path = "/contexts/{collectionInstrumentId}/lastExecutionDate")
+    @PreAuthorize("hasRole('SCHEDULER')")
+    public ResponseEntity<Object> setSurveyLastExecutionByCollectionInstrumentId(
+            @PathVariable("collectionInstrumentId") @RequestBody String collectionInstrumentId,
+            @Parameter(description = "Date to save as last execution date", example = "2024-01-01T12:00:00") @RequestParam("newDate") LocalDateTime newDate
+    ) {
+        try {
+            dataProcessingContextApiPort.updateLastExecutionDateByCollectionInstrumentId(collectionInstrumentId, newDate);
+            log.info("{} last execution updated at {} !", collectionInstrumentId, newDate);
+        }catch (GenesisException e){
+            return new ResponseEntity<>(e.getMessage(), HttpStatusCode.valueOf(e.getStatus()));
+        }
+        return ResponseEntity.ok().build();
+    }
+
+    @Deprecated(forRemoval = true)
     @Operation(summary = "Delete the Kraftwerk execution schedules of a partition")
-    @DeleteMapping(path = "/schedules")
+    @DeleteMapping(path = "/context/schedules")
     @PreAuthorize("hasRole('USER_KRAFTWERK')")
     public ResponseEntity<Object> deleteSchedules(
             @Parameter(description = "Survey name of the schedule(s) to delete") @RequestParam("partitionId") String partitionId
@@ -162,36 +265,31 @@ public ResponseEntity<Object> deleteSchedules(
         return ResponseEntity.ok().build();
     }
 
+    @Operation(summary = "Delete the Kraftwerk execution schedules of a collection instrument id")
+    @DeleteMapping(path = "/context/{collectionInstrumentId}/schedules")
+    @PreAuthorize("hasRole('USER_KRAFTWERK')")
+    public ResponseEntity<Object> deleteSchedulesByCollectionInstrumentId(
+            @PathVariable("collectionInstrumentId") String collectionInstrumentId
+    ){
+        try {
+            dataProcessingContextApiPort.deleteSchedulesByCollectionInstrumentId(collectionInstrumentId);
+        }catch (GenesisException e){
+            return new ResponseEntity<>(e.getMessage(), HttpStatusCode.valueOf(e.getStatus()));
+        }
+        log.info("Schedule deleted for survey {}", collectionInstrumentId);
+        return ResponseEntity.ok().build();
+    }
+
     @Operation(summary = "Delete expired schedules")
-    @DeleteMapping(path = "/schedules/expired-schedules")
+    @DeleteMapping(path = "/context/schedules/expired-schedules")
     @PreAuthorize("hasRole('SCHEDULER')")
-    public ResponseEntity<Object> deleteExpiredSchedules() throws GenesisException, IOException {
-        Set<String> storedSurveySchedulesNames = new HashSet<>();
-        for(ScheduleDto scheduleDto : dataProcessingContextApiPort.getAllSchedules()){
-            storedSurveySchedulesNames.add(scheduleDto.surveyName());
-        }
-        for (String surveyScheduleName : storedSurveySchedulesNames) {
-            List<KraftwerkExecutionSchedule> deletedKraftwerkExecutionSchedules = dataProcessingContextApiPort.deleteExpiredSchedules(surveyScheduleName);
-            //Save in JSON log
-            if(!deletedKraftwerkExecutionSchedules.isEmpty()) {
-                Path jsonLogPath = Path.of(fileUtils.getLogFolder(), Constants.SCHEDULE_ARCHIVE_FOLDER_NAME,
-                        surveyScheduleName + ".json");
-                ObjectMapper objectMapper = new ObjectMapper().findAndRegisterModules();
-                objectMapper.registerModule(new JavaTimeModule());
-                String jsonToWrite = objectMapper.writeValueAsString(deletedKraftwerkExecutionSchedules);
-                if(Files.exists(jsonLogPath)){
-                    //Remove last ] and append survey
-                    StringBuilder content = new StringBuilder(Files.readString(jsonLogPath));
-                    content.setCharAt(content.length()-1, ',');
-                    content.append(jsonToWrite, 1, jsonToWrite.length()-1);
-                    content.append(']');
-                    Files.write(jsonLogPath, content.toString().getBytes(), StandardOpenOption.TRUNCATE_EXISTING);
-                }else {
-                    Files.createDirectories(jsonLogPath.getParent());
-                    Files.write(jsonLogPath, jsonToWrite.getBytes());
-                }
-            }
+    public ResponseEntity<Object> deleteExpiredSchedules(){
+        try{
+            dataProcessingContextApiPort.deleteExpiredSchedules(fileUtils.getLogFolder());
+        } catch (GenesisException e){
+            return new ResponseEntity<>(e.getMessage(), HttpStatusCode.valueOf(e.getStatus()));
         }
+        log.info("Expired schedules deleted");
         return ResponseEntity.ok().build();
     }
 }