Get unprocessed questionnaire ids from old raw responses model

src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseController.java

@@ -46,6 +46,7 @@
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 
 @Slf4j
 @Controller
@@ -201,6 +202,14 @@ public ResponseEntity<List<LunaticJsonRawDataUnprocessedDto>> getUnprocessedJson
         return ResponseEntity.ok(lunaticJsonRawDataApiPort.getUnprocessedDataIds());
     }
 
+    @Operation(summary = "Get campaign id and interrogationId from all unprocessed raw json data")
+    @GetMapping(path = "/responses/raw/lunatic-json/get/unprocessed/questionnaireIds")
+    @PreAuthorize("hasRole('SCHEDULER')")
+    public ResponseEntity<Set<String>> getUnprocessedJsonRawDataQuestionnairesIds() {
+        log.info("Try to get unprocessed raw JSON datas questionniares...");
+        return ResponseEntity.ok(lunaticJsonRawDataApiPort.getUnprocessedDataQuestionnaireIds());
+    }
+
     @Hidden
     @GetMapping(path = "/responses/raw/lunatic-json/get/by-interrogation-mode-and-campaign")
     @PreAuthorize("hasRole('ADMIN')")

src/main/java/fr/insee/genesis/domain/ports/api/LunaticJsonRawDataApiPort.java

@@ -23,6 +23,7 @@ public interface LunaticJsonRawDataApiPort {
     List<LunaticJsonRawDataModel> getRawData(String campaignName, Mode mode, List<String> interrogationIdList);
     List<SurveyUnitModel> convertRawData(List<LunaticJsonRawDataModel> rawData, VariablesMap variablesMap);
     List<LunaticJsonRawDataUnprocessedDto> getUnprocessedDataIds();
+    Set<String> getUnprocessedDataQuestionnaireIds();
     void updateProcessDates(List<SurveyUnitModel> surveyUnitModels);
     Set<String> findDistinctQuestionnaireIds();
     long countResponsesByQuestionnaireId(String campaignId);

src/main/java/fr/insee/genesis/domain/ports/spi/LunaticJsonRawDataPersistencePort.java

@@ -18,6 +18,7 @@ public interface LunaticJsonRawDataPersistencePort {
     List<LunaticJsonRawDataModel> getAllUnprocessedData();
     void updateProcessDates(String campaignId, Set<String> interrogationIds);
     Set<String> findDistinctQuestionnaireIds();
+    Set<String> findDistinctQuestionnaireIdsByNullProcessDate();
     Page<LunaticJsonRawDataModel> findByCampaignIdAndDate(String campaignId, Instant startDt, Instant endDt, Pageable pageable);
     long countResponsesByQuestionnaireId(String questionnaireId);
     List<GroupedInterrogation> findProcessedIdsGroupedByQuestionnaireSince(LocalDateTime since);

src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java

@@ -353,6 +353,11 @@ public List<LunaticJsonRawDataUnprocessedDto> getUnprocessedDataIds() {
         return dtos;
     }
 
+    @Override
+    public Set<String> getUnprocessedDataQuestionnaireIds() {
+        return lunaticJsonRawDataPersistencePort.findDistinctQuestionnaireIdsByNullProcessDate();
+    }
+
     @SuppressWarnings("unchecked")
     private static void convertRawDataExternalVariables(
             LunaticJsonRawDataModel srcRawData,

src/main/java/fr/insee/genesis/infrastructure/adapter/LunaticJsonRawDataMongoAdapter.java

@@ -6,7 +6,6 @@
 import fr.insee.genesis.domain.model.surveyunit.rawdata.LunaticJsonRawDataModel;
 import fr.insee.genesis.domain.ports.spi.LunaticJsonRawDataPersistencePort;
 import fr.insee.genesis.infrastructure.document.rawdata.LunaticJsonRawDataDocument;
-import fr.insee.genesis.infrastructure.document.surveyunit.GroupedInterrogationDocument;
 import fr.insee.genesis.infrastructure.mappers.GroupedInterrogationDocumentMapper;
 import fr.insee.genesis.infrastructure.mappers.LunaticJsonRawDataDocumentMapper;
 import fr.insee.genesis.infrastructure.repository.LunaticJsonMongoDBRepository;
@@ -50,6 +49,11 @@ public List<LunaticJsonRawDataModel> getAllUnprocessedData() {
         return LunaticJsonRawDataDocumentMapper.INSTANCE.listDocumentToListModel(repository.findByNullProcessDate());
     }
 
+    @Override
+    public Set<String> findDistinctQuestionnaireIdsByNullProcessDate(){
+        return getDistinctQuestionnaireIdsInCollection(Constants.MONGODB_LUNATIC_RAWDATA_COLLECTION_NAME);
+    }
+
     @Override
     public List<LunaticJsonRawDataModel> findRawData(String campaignName, Mode mode, List<String> interrogationIdList) {
         List<LunaticJsonRawDataDocument> rawDataDocs = repository.findModesByCampaignIdAndByModeAndinterrogationIdIninterrogationIdList(campaignName, mode, interrogationIdList);return LunaticJsonRawDataDocumentMapper.INSTANCE.listDocumentToListModel(rawDataDocs);
@@ -66,13 +70,7 @@ public void updateProcessDates(String campaignId, Set<String> interrogationIds)
 
     @Override
     public Set<String> findDistinctQuestionnaireIds() {
-        Set<String> questionnaireIds = new HashSet<>();
-        for(String questionnaireId : mongoTemplate.getCollection(Constants.MONGODB_RAW_RESPONSES_COLLECTION_NAME).distinct(
-                "questionnaireId",
-                String.class)){
-            questionnaireIds.add(questionnaireId);
-        }
-        return questionnaireIds;
+        return getDistinctQuestionnaireIdsInCollection(Constants.MONGODB_RAW_RESPONSES_COLLECTION_NAME);
     }
 
     @Override
@@ -107,4 +105,14 @@ public Set<String> findUnprocessedInterrogationIdsByCollectionInstrumentId(Strin
 
         return interrogationIds;
     }
+
+    private Set<String> getDistinctQuestionnaireIdsInCollection(String collectionName) {
+        Set<String> questionnaireIds = new HashSet<>();
+        for (String questionnaireId : mongoTemplate.getCollection(collectionName).distinct(
+                "questionnaireId",
+                String.class)) {
+            questionnaireIds.add(questionnaireId);
+        }
+        return questionnaireIds;
+    }
 }

src/test/java/fr/insee/genesis/controller/rest/responses/RawResponseControllerUnitTest.java

@@ -0,0 +1,49 @@
+package fr.insee.genesis.controller.rest.responses;
+
+import fr.insee.genesis.domain.ports.api.LunaticJsonRawDataApiPort;
+import fr.insee.genesis.domain.ports.api.RawResponseApiPort;
+import fr.insee.genesis.infrastructure.repository.RawResponseInputRepository;
+import org.assertj.core.api.Assertions;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.springframework.http.ResponseEntity;
+
+import java.util.HashSet;
+import java.util.Set;
+
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+
+class RawResponseControllerUnitTest {
+
+    static RawResponseController rawResponseController;
+    static LunaticJsonRawDataApiPort lunaticJsonRawDataStub;
+
+    @BeforeEach
+    void init(){
+        lunaticJsonRawDataStub = mock(LunaticJsonRawDataApiPort.class);
+        rawResponseController = new RawResponseController(
+            lunaticJsonRawDataStub,
+                mock(RawResponseApiPort.class),
+                mock(RawResponseInputRepository.class)
+        );
+    }
+
+
+    @Test
+    void getUnprocessedJsonRawDataQuestionnairesIds_test(){
+        // GIVEN
+        Set<String> questionnaireIds = new HashSet<>();
+        questionnaireIds.add("QUEST1");
+        questionnaireIds.add("QUEST2");
+        doReturn(questionnaireIds).when(lunaticJsonRawDataStub).getUnprocessedDataQuestionnaireIds();
+
+        // WHEN
+        ResponseEntity<Set<String>> response = rawResponseController.getUnprocessedJsonRawDataQuestionnairesIds();
+
+        // THEN
+        Assertions.assertThat(response.getStatusCode().is2xxSuccessful()).isTrue();
+        Assertions.assertThat(response.getBody()).isNotNull().containsExactlyInAnyOrder("QUEST1","QUEST2");
+    }
+
+}

src/test/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataServiceUnitTest.java

@@ -0,0 +1,59 @@
+package fr.insee.genesis.domain.service.rawdata;
+
+import fr.insee.genesis.controller.utils.ControllerUtils;
+import fr.insee.genesis.domain.ports.spi.DataProcessingContextPersistancePort;
+import fr.insee.genesis.domain.ports.spi.LunaticJsonRawDataPersistencePort;
+import fr.insee.genesis.domain.ports.spi.SurveyUnitQualityToolPort;
+import fr.insee.genesis.domain.service.context.DataProcessingContextService;
+import fr.insee.genesis.domain.service.metadata.QuestionnaireMetadataService;
+import fr.insee.genesis.domain.service.surveyunit.SurveyUnitQualityService;
+import fr.insee.genesis.domain.service.surveyunit.SurveyUnitService;
+import fr.insee.genesis.infrastructure.utils.FileUtils;
+import fr.insee.genesis.stubs.ConfigStub;
+import org.assertj.core.api.Assertions;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.util.HashSet;
+import java.util.Set;
+
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+
+
+class LunaticJsonRawDataServiceUnitTest {
+
+    static LunaticJsonRawDataService lunaticJsonRawDataService;
+
+    static LunaticJsonRawDataPersistencePort lunaticJsonRawDataPersistencePort;
+
+    @BeforeEach
+    void init() {
+        lunaticJsonRawDataPersistencePort = mock(LunaticJsonRawDataPersistencePort.class);
+        lunaticJsonRawDataService = new LunaticJsonRawDataService(
+                lunaticJsonRawDataPersistencePort,
+                new ControllerUtils(new FileUtils(new ConfigStub())),
+                mock(QuestionnaireMetadataService.class),
+                mock(SurveyUnitService.class),
+                mock(SurveyUnitQualityService.class),
+                new FileUtils(new ConfigStub()),
+                mock(DataProcessingContextService.class),
+                mock(SurveyUnitQualityToolPort.class),
+                new ConfigStub(),
+                mock(DataProcessingContextPersistancePort.class)
+        );
+    }
+
+    @Test
+    void getUnprocessedDataQuestionnaireIds() {
+        //GIVEN
+        Set<String> questionnaireIds = new HashSet<>();
+        questionnaireIds.add("QUEST1");
+        questionnaireIds.add("QUEST2");
+        doReturn(questionnaireIds).when(lunaticJsonRawDataPersistencePort).findDistinctQuestionnaireIdsByNullProcessDate();
+
+        //WHEN + THEN
+        Assertions.assertThat(lunaticJsonRawDataService.getUnprocessedDataQuestionnaireIds())
+                .containsExactlyInAnyOrder("QUEST1","QUEST2");
+    }
+}

src/test/java/fr/insee/genesis/infrastructure/adapter/LunaticJsonRawDataMongoAdapterTest.java

@@ -74,5 +74,4 @@ void findRawDataTest(){
         List<LunaticJsonRawDataModel> rawdatas = adapter.findRawData("campaign01",Mode.WEB,List.of("interrogation01"));
         Assertions.assertThat(rawdatas).hasSize(1);
     }
-
 }

src/test/java/fr/insee/genesis/stubs/LunaticJsonRawDataPersistanceStub.java

@@ -42,6 +42,19 @@ public List<LunaticJsonRawDataModel> getAllUnprocessedData() {
         );
     }
 
+    @Override
+    public Set<String> findDistinctQuestionnaireIdsByNullProcessDate() {
+        Set<String> questionnaireIds = new HashSet<>();
+        mongoStub.stream().filter(
+                lunaticJsonDataDocument -> lunaticJsonDataDocument.processDate() == null
+        ).forEach(doc -> {
+            if(doc.questionnaireId() != null){
+                questionnaireIds.add(doc.questionnaireId());
+            }
+        });
+        return questionnaireIds;
+    }
+
     @Override
     public List<LunaticJsonRawDataModel> findRawData(String campaignName, Mode mode, List<String> interrogationIdList) {
         List<LunaticJsonRawDataDocument> docs = mongoStub.stream().filter(lunaticJsonRawDataDocument ->