usualSurveyUnitId bug fix + json export modele filiere fix + unit tests fix for modele filiere

.github/workflows/create-release.yaml

@@ -69,6 +69,9 @@ jobs:
           distribution: 'temurin'
           java-version: '21'
 
+      - name: install model filiere
+        run: mvn install:install-file -Dfile="$(pwd)/libs/modelefiliere-2.1.0.jar" -DgroupId=fr.insee -DartifactId=modelefiliere -Dversion=2.1.0 -Dpackaging=jar -DgeneratePom=true
+
       - name: Clone BPM
         uses: actions/checkout@v6
         with:

.github/workflows/docker.yaml

@@ -26,6 +26,9 @@ jobs:
           distribution: "temurin"
           java-version: "21"
 
+      - name: install model filiere
+        run: mvn install:install-file -Dfile="$(pwd)/libs/modelefiliere-2.1.0.jar" -DgroupId=fr.insee -DartifactId=modelefiliere -Dversion=2.1.0 -Dpackaging=jar -DgeneratePom=true
+
       - name: Clone BPM
         uses: actions/checkout@master
         with:

.github/workflows/maven.yml

@@ -29,6 +29,9 @@ jobs:
         distribution: 'temurin'
         cache: maven
 
+    - name: install model filiere
+      run: mvn install:install-file -Dfile="$(pwd)/libs/modelefiliere-2.1.0.jar" -DgroupId=fr.insee -DartifactId=modelefiliere -Dversion=2.1.0 -Dpackaging=jar -DgeneratePom=true
+
     - name: Clone BPM
       uses: actions/checkout@master
       with:

.github/workflows/scheduled-version-update.yaml

@@ -85,6 +85,9 @@ jobs:
           fi
           echo "new_version=$NEW_VERSION" >> $GITHUB_OUTPUT
 
+      - name: install model filiere
+        run: mvn install:install-file -Dfile="$(pwd)/libs/modelefiliere-2.1.0.jar" -DgroupId=fr.insee -DartifactId=modelefiliere -Dversion=2.1.0 -Dpackaging=jar -DgeneratePom=true
+
       - name: Clone BPM
         uses: actions/checkout@master
         with:

.runtests/Unit tests GENESIS.run.xml

@@ -0,0 +1,13 @@
+<component name="ProjectRunConfigurationManager">
+  <configuration default="false" name="Unit tests GENESIS" type="JUnit" factoryName="JUnit">
+    <module name="genesis-api" />
+    <option name="PACKAGE_NAME" value="fr.insee.genesis" />
+    <option name="MAIN_CLASS_NAME" value="" />
+    <option name="METHOD_NAME" value="" />
+    <option name="TEST_OBJECT" value="directory" />
+    <dir value="$PROJECT_DIR$/src/test/java/fr/insee/genesis" />
+    <method v="2">
+      <option name="Make" enabled="true" />
+    </method>
+  </configuration>
+</component>

CHANGELOG.md

@@ -1,4 +1,8 @@
 # Changelog
+## 2.0.0 [TODO]
+### Changed
+- Use filière model
+
 ## 1.13.0 [2025-12-04]
 ### Changed
 - New raw data process endpoint

pom.xml

@@ -101,7 +101,7 @@
         <dependency>
             <groupId>fr.insee</groupId>
             <artifactId>modelefiliere</artifactId>
-            <version>2.0.0</version>
+            <version>2.1.0</version>
         </dependency>
         <dependency>
             <groupId>com.networknt</groupId>

src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseController.java

@@ -237,15 +237,15 @@ public ResponseEntity<String> processJsonRawData(
         }
     }
 
-    @Operation(summary = "Process raw data of a questionnaire")
-    @PostMapping(path = "/responses/raw/lunatic-json/{collectionInstrumentId}/process")
+    @Operation(summary = "Process raw data of a questionnaire (old raw model)")
+    @PostMapping(path = "/responses/raw/lunatic-json/{questionnaireId}/process")
     @PreAuthorize("hasRole('SCHEDULER')")
     public ResponseEntity<String> processJsonRawData(
-            @PathVariable String collectionInstrumentId
+            @PathVariable String questionnaireId
     ) {
-        log.info("Try to process raw JSON datas for questionnaire {}",collectionInstrumentId);
+        log.info("Try to process raw JSON datas for questionnaire {}",questionnaireId);
         try {
-            DataProcessResult result = lunaticJsonRawDataApiPort.processRawData(collectionInstrumentId);
+            DataProcessResult result = lunaticJsonRawDataApiPort.processRawData(questionnaireId);
             return result.formattedDataCount() == 0 ?
                     ResponseEntity.ok("%d document(s) processed".formatted(result.dataCount()))
                     : ResponseEntity.ok("%d document(s) processed, including %d FORMATTED after data verification"

src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java

@@ -53,7 +53,11 @@
 
 import javax.xml.parsers.ParserConfigurationException;
 import javax.xml.stream.XMLStreamException;
-import java.io.*;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
 import java.net.MalformedURLException;
 import java.nio.file.Files;
 import java.nio.file.Path;
@@ -296,6 +300,7 @@ public ResponseEntity<SurveyUnitSimplified> getLatestByInterrogationOneObject(@R
                     "For a given id, the endpoint returns a document by collection mode (if there is more than one).")
     @PostMapping(path = "/simplified/by-list-interrogation-and-collection-instrument/latest")
     @PreAuthorize("hasRole('USER_KRAFTWERK')")
+    //TODO move logic and unit test to surveyUnitService (also extract some methods instead of multiple lambdas)
     public ResponseEntity<List<SurveyUnitSimplified>> getLatestForInterrogationListAndCollectionInstrument(
             @RequestParam("collectionInstrumentId") String collectionInstrumentId,
             @RequestBody List<InterrogationId> interrogationIds)
@@ -307,16 +312,20 @@ public ResponseEntity<List<SurveyUnitSimplified>> getLatestForInterrogationListA
             modes.forEach(mode -> {
                 List<VariableModel> outputVariables = new ArrayList<>();
                 List<VariableModel> outputExternalVariables = new ArrayList<>();
+                List<String> usualSurveyUnitIds = new ArrayList<>();
                 responses.stream().filter(rep -> rep.getMode().equals(mode)).forEach(response -> {
                     outputVariables.addAll(response.getCollectedVariables());
                     outputExternalVariables.addAll(response.getExternalVariables());
+                    if(response.getUsualSurveyUnitId() != null){
+                        usualSurveyUnitIds.add(response.getUsualSurveyUnitId());
+                    }
                 });
                 if (!outputVariables.isEmpty() || !outputExternalVariables.isEmpty()) {
                     results.add(SurveyUnitSimplified.builder()
                             .collectionInstrumentId(responses.getFirst().getCollectionInstrumentId())
                             .campaignId(responses.getFirst().getCampaignId())
-                            .interrogationId(responses.getFirst().getInterrogationId())
-                            .usualSurveyUnitId(responses.getFirst().getUsualSurveyUnitId())
+                            .interrogationId(interrogationId.getInterrogationId())
+                            .usualSurveyUnitId(!usualSurveyUnitIds.isEmpty() ? usualSurveyUnitIds.getFirst() : null)
                             .mode(mode)
                             .variablesUpdate(outputVariables)
                             .externalVariables(outputExternalVariables)

src/main/java/fr/insee/genesis/domain/service/contextualvariable/ContextualVariableJsonService.java

@@ -79,9 +79,7 @@ public int saveContextualVariableFiles(String collectionInstrumentId, FileUtils
         int fileCount = 0;
 
         for (Mode mode : Mode.values()) {
-            try (Stream<Path> filePaths = Files.list(Path.of(fileUtils.getDataFolder(collectionInstrumentId,
-                    mode.getFolder()
-                    , null)))) {
+            try (Stream<Path> filePaths = Files.list(Path.of(contextualFolderPath))) {
                 Iterator<Path> it = filePaths
                         .filter(path -> path.toString().endsWith(".json"))
                         .iterator();

src/main/java/fr/insee/genesis/domain/service/metadata/QuestionnaireMetadataService.java

@@ -50,17 +50,11 @@ public MetadataModel loadAndSaveIfNotExists(String campaignName, String collecti
         List<QuestionnaireMetadataModel> questionnaireMetadataModels =
                 questionnaireMetadataPersistencePort.find(collectionInstrumentId.toUpperCase(), mode);
         if(questionnaireMetadataModels.isEmpty() || questionnaireMetadataModels.getFirst().metadataModel() == null){
-            MetadataModel metadataModel = readMetadatas(campaignName, mode.getModeName(), fileUtils, errors);
+            MetadataModel metadataModel = readMetadatas(collectionInstrumentId, mode.getModeName(), fileUtils, errors);
             saveMetadata(collectionInstrumentId.toUpperCase(), mode, metadataModel);
             return metadataModel;
         }
-
-        MetadataModel metadataModel =
-                readMetadatas(campaignName, mode.getModeName(), fileUtils, errors);
-
-        saveMetadata(questionnaireId.toUpperCase(), mode, metadataModel);
-
-        return metadataModel;
+        return questionnaireMetadataModels.getFirst().metadataModel();
     }
 
     private void saveMetadata(String collectionInstrumentId, Mode mode, MetadataModel metadataModel) {
@@ -87,11 +81,12 @@ private MetadataModel readMetadatas(String campaignName, String modeName, FileUt
                                   List<GenesisError> errors) throws GenesisException{
 
         Path ddiFilePath;
+        Path lunaticFilePath;
         MetadataModel metadataModel = null;
         try {
             ddiFilePath = fileUtils.findFile(String.format("%s/%s", fileUtils.getSpecFolder(campaignName), modeName), DDI_FILE_PATTERN);
-            metadataModel = parseMetadata(ddiFilePath.toString(), true);
-
+            lunaticFilePath = fileUtils.findFile(String.format("%s/%s", fileUtils.getSpecFolder(campaignName), modeName), LUNATIC_FILE_PATTERN);
+            metadataModel = parseMetadata(lunaticFilePath, ddiFilePath);
         } catch (RuntimeException e) {
             //DDI file not found and already log - Go to next step
         } catch (IOException e) {
@@ -100,8 +95,8 @@ private MetadataModel readMetadatas(String campaignName, String modeName, FileUt
         if(metadataModel == null ){
             log.warn("DDI not found or error occurred. Trying Lunatic metadata...for {}, {} mode", campaignName, modeName);
             try {
-                Path lunaticFilePath = fileUtils.findFile(String.format("%s/%s", fileUtils.getSpecFolder(campaignName), modeName), LUNATIC_FILE_PATTERN);
-                return parseMetadata(lunaticFilePath.toString(), false);
+                lunaticFilePath = fileUtils.findFile(String.format("%s/%s", fileUtils.getSpecFolder(campaignName), modeName), LUNATIC_FILE_PATTERN);
+                return parseMetadata(lunaticFilePath, null);
             } catch (Exception ex) {
                 log.error("Error reading Lunatic metadata file", ex);
                 errors.add(new GenesisError(ex.toString()));
@@ -116,32 +111,33 @@ private MetadataModel readMetadatas(String campaignName, String modeName, FileUt
     }
 
     /**
-     * Parse metadata file, either DDI or Lunatic, depending on the withDDI flag.
+     * Parse metadata file
      *
-     * @param metadataFilePath path to the metadata file
-     * @param withDDI          true for DDI parsing, false for Lunatic parsing
+     * @param lunaticFilePath path to the DDI metadata file
+     * @param ddiFilePath path to the DDI metadata file, will parse only lunatic if null
      * @return VariablesMap or null if an error occurs
      */
-    private MetadataModel parseMetadata(String metadataFilePath, boolean withDDI) {
+    private MetadataModel parseMetadata(Path lunaticFilePath, Path ddiFilePath) {
         try {
-            log.info("Try to read {} file: {}", withDDI ? "DDI" : "Lunatic", metadataFilePath);
-            if (withDDI) {
-                InputStream metadataInputStream = new FileInputStream(metadataFilePath);
+            log.info("Try to read {} file: {}", ddiFilePath != null ? "DDI" : "Lunatic", ddiFilePath);
+
+            if (ddiFilePath != null) {
+                InputStream metadataInputStream = new FileInputStream(ddiFilePath.toFile());
+                InputStream lunaticInputStream = new FileInputStream(lunaticFilePath.toFile());
                 MetadataModel metadataModel = ReaderUtils.getMetadataFromDDIAndLunatic(
-                        Path.of(metadataFilePath).toFile().toURI().toURL().toString(),
-                        metadataInputStream,metadataInputStream);
+                        ddiFilePath.toFile().toURI().toURL().toString(),
+                        metadataInputStream,
+                        lunaticInputStream
+                );
                 // Temporary solution
                 // the logic of adding variables from lunatic to the ones present in the DDI needs to be implemented in BPM
                 // (only in Kraftwerk for the moment)
                 for (String enoVar : Constants.getEnoVariables()){
                     metadataModel.getVariables().putVariable(new Variable(enoVar, metadataModel.getRootGroup(), VariableType.STRING));
                 }
                 return metadataModel;
-
-            } else {
-                return LunaticReader.getMetadataFromLunatic(
-                        new FileInputStream(metadataFilePath));
             }
+            return LunaticReader.getMetadataFromLunatic(new FileInputStream(lunaticFilePath.toFile()));
         } catch (MetadataParserException | IOException e) {
             log.error("Error reading metadata file", e);
             return null;

src/main/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitService.java

@@ -248,7 +248,7 @@ public SurveyUnitDto findLatestValuesByStateByIdAndByCollectionInstrumentId(
             for (SurveyUnitModel surveyUnitModel : suByMode) {
                 if (variablesMap == null) {
                     variablesMap = metadataService.loadAndSaveIfNotExists(
-                            surveyUnitModel.getCampaignId(),
+                            surveyUnitModel.getCollectionInstrumentId(),
                             surveyUnitModel.getCollectionInstrumentId(),
                             surveyUnitModel.getMode(),
                             fileUtils,

src/main/java/fr/insee/genesis/infrastructure/adapter/SurveyUnitMongoAdapter.java

@@ -161,8 +161,10 @@ public List<SurveyUnitModel> findInterrogationIdsByCollectionInstrumentId(String
 
 	@Override
 	public List<SurveyUnitModel> findInterrogationIdsByQuestionnaireIdAndDateAfter(String questionnaireId, LocalDateTime since) {
-		List<SurveyUnitDocument> surveyUnits = mongoRepository.findInterrogationIdsByQuestionnaireIdAndDateAfter(questionnaireId, since);
-		return surveyUnits.isEmpty() ? Collections.emptyList() : SurveyUnitDocumentMapper.INSTANCE.listDocumentToListModel(surveyUnits);
+		List<SurveyUnitDocument> results =  new ArrayList<>();
+		results.addAll(mongoRepository.findInterrogationIdsByQuestionnaireIdAndDateAfter(questionnaireId, since));
+		results.addAll(mongoRepository.findInterrogationIdsByCollectionInstrumentIdAndDateAfter(questionnaireId, since));
+		return results.isEmpty() ? Collections.emptyList() : SurveyUnitDocumentMapper.INSTANCE.listDocumentToListModel(results);
 	}
 
 	//========== OPTIMISATIONS PERFS (START) ===========

src/main/java/fr/insee/genesis/infrastructure/mappers/SurveyUnitDocumentMapper.java

@@ -22,7 +22,8 @@ public interface SurveyUnitDocumentMapper {
 	List<SurveyUnitDocument> listModelToListDocument(List<SurveyUnitModel> surveyUnitModels);
 
 	@AfterMapping
-	default void fillModelAfterRead(SurveyUnitDocument doc,
+	@SuppressWarnings("deprecation")
+	default void handleDeprecatedFields(SurveyUnitDocument doc,
 									@MappingTarget SurveyUnitModel model) {
 
 		if (model.getUsualSurveyUnitId() == null) {
@@ -34,5 +35,4 @@ default void fillModelAfterRead(SurveyUnitDocument doc,
 		}
 
 	}
-
 }

src/main/java/fr/insee/genesis/infrastructure/repository/SurveyUnitMongoDBRepository.java

@@ -37,6 +37,9 @@ public interface SurveyUnitMongoDBRepository extends MongoRepository<SurveyUnitD
 	@Query(value = "{ 'questionnaireId' : ?0, 'recordDate': { $gte: ?1 } }", fields = "{ 'interrogationId' : 1, 'mode' :  1 }")
 	List<SurveyUnitDocument> findInterrogationIdsByQuestionnaireIdAndDateAfter(String questionnaireId, LocalDateTime since);
 
+	@Query(value = "{ 'collectionInstrumentId' : ?0, 'recordDate': { $gte: ?1 } }", fields = "{ 'interrogationId' : 1, 'mode' :  1 }")
+	List<SurveyUnitDocument> findInterrogationIdsByCollectionInstrumentIdAndDateAfter(String collectionInstrumentId, LocalDateTime since);
+
 	//========= OPTIMISATIONS PERFS (START) ==========
 	/**
 	 * @author Adrien Marchal

src/test/resources/fr/insee/genesis/features/do_we_extract_data.feature → src/test/java/cucumber/features/do_we_extract_data.feature

@@ -34,7 +34,7 @@ Feature: Do we extract data ?
     Examples:
       | CampaignId                | QuestionnaireId             | InterrogationId  | ExpectedCollectedVariablesCount   | ExpectedExternalVariablesCount    | ExpectedVariableType | ExpectedVariableName | ExpectedValue |
       | TEST-TABLEAUX             | TEST-TABLEAUX               | AUTO11000        | 49                                | 4                                 | Integer              | TABLEAUTIC21         | 5             |
-      | SAMPLETEST-PARADATA-v2    | quest_model_famille_AD_ttp  | 0000007          | 96                                | 17                                | Boolean              | AVIS_SUPPORT1        | true          |
+      | SAMPLETEST-PARADATA-V2    | quest_model_famille_AD_ttp  | 0000007          | 96                                | 17                                | Boolean              | AVIS_SUPPORT1        | true          |
 
   Scenario Outline: Multiple different Contexts for one partitionId
     Given We have data in directory "<CampaignId>"