diff --git a/src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseController.java b/src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseController.java index ec4d9aa0..846cd5aa 100644 --- a/src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseController.java +++ b/src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseController.java @@ -47,6 +47,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Set; @Slf4j @Controller @@ -202,6 +203,14 @@ public ResponseEntity> getUnprocessedJson return ResponseEntity.ok(lunaticJsonRawDataApiPort.getUnprocessedDataIds()); } + @Operation(summary = "Get campaign id and interrogationId from all unprocessed raw json data") + @GetMapping(path = "/responses/raw/lunatic-json/get/unprocessed/questionnaireIds") + @PreAuthorize("hasRole('SCHEDULER')") + public ResponseEntity> getUnprocessedJsonRawDataQuestionnairesIds() { + log.info("Try to get unprocessed raw JSON datas questionniares..."); + return ResponseEntity.ok(lunaticJsonRawDataApiPort.getUnprocessedDataQuestionnaireIds()); + } + @Hidden @GetMapping(path = "/responses/raw/lunatic-json/get/by-interrogation-mode-and-campaign") @PreAuthorize("hasRole('ADMIN')") diff --git a/src/main/java/fr/insee/genesis/domain/ports/api/LunaticJsonRawDataApiPort.java b/src/main/java/fr/insee/genesis/domain/ports/api/LunaticJsonRawDataApiPort.java index d928a00e..ba3e28a8 100644 --- a/src/main/java/fr/insee/genesis/domain/ports/api/LunaticJsonRawDataApiPort.java +++ b/src/main/java/fr/insee/genesis/domain/ports/api/LunaticJsonRawDataApiPort.java @@ -23,6 +23,7 @@ public interface LunaticJsonRawDataApiPort { List getRawData(String campaignName, Mode mode, List interrogationIdList); List convertRawData(List rawData, VariablesMap variablesMap); List getUnprocessedDataIds(); + Set getUnprocessedDataQuestionnaireIds(); void updateProcessDates(List surveyUnitModels); Set findDistinctQuestionnaireIds(); long countResponsesByQuestionnaireId(String campaignId); diff --git a/src/main/java/fr/insee/genesis/domain/ports/spi/LunaticJsonRawDataPersistencePort.java b/src/main/java/fr/insee/genesis/domain/ports/spi/LunaticJsonRawDataPersistencePort.java index 5116271b..bbbf2951 100644 --- a/src/main/java/fr/insee/genesis/domain/ports/spi/LunaticJsonRawDataPersistencePort.java +++ b/src/main/java/fr/insee/genesis/domain/ports/spi/LunaticJsonRawDataPersistencePort.java @@ -19,6 +19,8 @@ public interface LunaticJsonRawDataPersistencePort { List getAllUnprocessedData(); void updateProcessDates(String campaignId, Set interrogationIds); Set findDistinctQuestionnaireIds(); + Set findDistinctQuestionnaireIdsByNullProcessDate(); + Set findModesByQuestionnaire(String questionnaireId); Page findByCampaignIdAndDate(String campaignId, Instant startDt, Instant endDt, Pageable pageable); long countResponsesByQuestionnaireId(String questionnaireId); List findProcessedIdsGroupedByQuestionnaireSince(LocalDateTime since); diff --git a/src/main/java/fr/insee/genesis/domain/ports/spi/RawResponsePersistencePort.java b/src/main/java/fr/insee/genesis/domain/ports/spi/RawResponsePersistencePort.java index 17c25dce..14b30043 100644 --- a/src/main/java/fr/insee/genesis/domain/ports/spi/RawResponsePersistencePort.java +++ b/src/main/java/fr/insee/genesis/domain/ports/spi/RawResponsePersistencePort.java @@ -16,5 +16,6 @@ public interface RawResponsePersistencePort { void updateProcessDates(String collectionInstrumentId, Set interrogationIds); List getUnprocessedCollectionIds(); Set findUnprocessedInterrogationIdsByCollectionInstrumentId(String collectionInstrumentId); + List findModesByCollectionInstrument(String collectionInstrumentId); Page findByCampaignIdAndDate(String campaignId, Instant startDate, Instant endDate, Pageable pageable); } diff --git a/src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java b/src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java index 09c9e1ea..abb3df06 100644 --- a/src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java +++ b/src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java @@ -1,5 +1,6 @@ package fr.insee.genesis.domain.service.rawdata; +import fr.insee.bpm.metadata.model.MetadataModel; import fr.insee.bpm.metadata.model.VariablesMap; import fr.insee.genesis.Constants; import fr.insee.genesis.configuration.Config; @@ -358,6 +359,48 @@ public List getUnprocessedDataIds() { return dtos; } + @Override + public Set getUnprocessedDataQuestionnaireIds() { + Set unprocessedQuestionnaireIds = lunaticJsonRawDataPersistencePort.findDistinctQuestionnaireIdsByNullProcessDate(); + Set unprocessedQuestionnaireIdsWithSpecs = new HashSet<>(); + for (String unprocessedQuestionnaireId : unprocessedQuestionnaireIds){ + Set modes = lunaticJsonRawDataPersistencePort.findModesByQuestionnaire(unprocessedQuestionnaireId); + if (modes.isEmpty()){ + continue; + } + + boolean areAllSpecsOK = true; + for(Mode mode : modes){ + if(!isSpecsPresentForQuestionnaireAndMode(unprocessedQuestionnaireId, mode)){ + areAllSpecsOK = false; + } + } + if(areAllSpecsOK){ + unprocessedQuestionnaireIdsWithSpecs.add(unprocessedQuestionnaireId); + } + } + + return unprocessedQuestionnaireIdsWithSpecs; + } + + private boolean isSpecsPresentForQuestionnaireAndMode(String unprocessedQuestionnaireId, Mode mode) { + List genesisErrors = new ArrayList<>(); + MetadataModel metadataModel; + try { + metadataModel = metadataService.loadAndSaveIfNotExists( + unprocessedQuestionnaireId, + unprocessedQuestionnaireId, + mode, + fileUtils, + genesisErrors + ); + } catch (GenesisException ge) { + log.warn("Genesis exception thrown for questionnaire %s and mode %s, excluding from get questionnaire ids...".formatted(unprocessedQuestionnaireId, mode)); + return false; + } + return metadataModel != null && genesisErrors.isEmpty(); + } + @SuppressWarnings("unchecked") private static void convertRawDataExternalVariables( LunaticJsonRawDataModel srcRawData, diff --git a/src/main/java/fr/insee/genesis/domain/service/rawdata/RawResponseService.java b/src/main/java/fr/insee/genesis/domain/service/rawdata/RawResponseService.java index ca1fbc29..17911b1f 100644 --- a/src/main/java/fr/insee/genesis/domain/service/rawdata/RawResponseService.java +++ b/src/main/java/fr/insee/genesis/domain/service/rawdata/RawResponseService.java @@ -1,5 +1,6 @@ package fr.insee.genesis.domain.service.rawdata; +import fr.insee.bpm.metadata.model.MetadataModel; import fr.insee.bpm.metadata.model.VariablesMap; import fr.insee.genesis.Constants; import fr.insee.genesis.configuration.Config; @@ -272,7 +273,45 @@ public List convertRawResponse(List rawRespon @Override public List getUnprocessedCollectionInstrumentIds() { - return rawResponsePersistencePort.getUnprocessedCollectionIds(); + List unprocessedCollectionInstrumentIds = rawResponsePersistencePort.getUnprocessedCollectionIds(); + List unprocessedCollectionInstrumentIdsWithSpecs = new ArrayList<>(); + for (String unprocessedCollectionInstrumentId : unprocessedCollectionInstrumentIds){ + Set modes = new HashSet<>(rawResponsePersistencePort.findModesByCollectionInstrument(unprocessedCollectionInstrumentId)); + if (modes.isEmpty()){ + continue; + } + + boolean areAllSpecsOK = true; + for(Mode mode : modes){ + if(!isSpecsPresentForCollectionInstrumentAndMode(unprocessedCollectionInstrumentId, mode)){ + areAllSpecsOK = false; + } + } + if(areAllSpecsOK){ + unprocessedCollectionInstrumentIdsWithSpecs.add(unprocessedCollectionInstrumentId); + } + } + + return unprocessedCollectionInstrumentIdsWithSpecs; + } + + private boolean isSpecsPresentForCollectionInstrumentAndMode(String unprocessedCollectionInstrumentId, Mode mode) { + List genesisErrors = new ArrayList<>(); + MetadataModel metadataModel; + try { + metadataModel = metadataService.loadAndSaveIfNotExists( + unprocessedCollectionInstrumentId, + unprocessedCollectionInstrumentId, + mode, + fileUtils, + genesisErrors + ); + } catch (GenesisException ge) { + log.warn("Genesis exception thrown for collection instrument %s and mode %s, excluding from get collection instrument ids..." + .formatted(unprocessedCollectionInstrumentId, mode)); + return false; + } + return metadataModel != null && genesisErrors.isEmpty(); } @Override diff --git a/src/main/java/fr/insee/genesis/infrastructure/adapter/LunaticJsonRawDataMongoAdapter.java b/src/main/java/fr/insee/genesis/infrastructure/adapter/LunaticJsonRawDataMongoAdapter.java index 11b65311..a913f526 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/adapter/LunaticJsonRawDataMongoAdapter.java +++ b/src/main/java/fr/insee/genesis/infrastructure/adapter/LunaticJsonRawDataMongoAdapter.java @@ -49,6 +49,20 @@ public List getAllUnprocessedData() { return LunaticJsonRawDataDocumentMapper.INSTANCE.listDocumentToListModel(repository.findByNullProcessDate()); } + @Override + public Set findDistinctQuestionnaireIdsByNullProcessDate(){ + Set questionnaireIds = new HashSet<>(); + for(LunaticJsonRawDataDocument lunaticJsonRawDataDocument : repository.findByNullProcessDate()){ + questionnaireIds.add(lunaticJsonRawDataDocument.questionnaireId()); + } + return questionnaireIds; + } + + @Override + public Set findModesByQuestionnaire(String questionnaireId) { + return new HashSet<>(repository.findModesByQuestionnaireId(questionnaireId)); + } + @Override public List findRawData(String campaignName, Mode mode, List interrogationIdList) { List rawDataDocs = repository.findModesByCampaignIdAndByModeAndinterrogationIdIninterrogationIdList(campaignName, mode, interrogationIdList);return LunaticJsonRawDataDocumentMapper.INSTANCE.listDocumentToListModel(rawDataDocs); @@ -72,9 +86,9 @@ public void updateProcessDates(String campaignId, Set interrogationIds) @Override public Set findDistinctQuestionnaireIds() { Set questionnaireIds = new HashSet<>(); - for(String questionnaireId : mongoTemplate.getCollection(Constants.MONGODB_RAW_RESPONSES_COLLECTION_NAME).distinct( + for (String questionnaireId : mongoTemplate.getCollection(Constants.MONGODB_RAW_RESPONSES_COLLECTION_NAME).distinct( "questionnaireId", - String.class)){ + String.class)) { questionnaireIds.add(questionnaireId); } return questionnaireIds; diff --git a/src/main/java/fr/insee/genesis/infrastructure/adapter/RawResponseMongoAdapter.java b/src/main/java/fr/insee/genesis/infrastructure/adapter/RawResponseMongoAdapter.java index 1c95f507..f3fb2c6e 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/adapter/RawResponseMongoAdapter.java +++ b/src/main/java/fr/insee/genesis/infrastructure/adapter/RawResponseMongoAdapter.java @@ -69,6 +69,11 @@ public Set findUnprocessedInterrogationIdsByCollectionInstrumentId(Strin return new HashSet<>(repository.findInterrogationIdByCollectionInstrumentIdAndProcessDateIsNull(collectionInstrumentId)); } + @Override + public List findModesByCollectionInstrument(String collectionInstrumentId) { + return repository.findModesByCollectionInstrumentId(collectionInstrumentId); + } + @Override public Page findByCampaignIdAndDate(String campaignId, Instant startDate, Instant endDate, Pageable pageable) { Page rawDataDocs = repository.findByCampaignIdAndDate(campaignId, startDate, endDate, pageable); diff --git a/src/main/java/fr/insee/genesis/infrastructure/repository/LunaticJsonMongoDBRepository.java b/src/main/java/fr/insee/genesis/infrastructure/repository/LunaticJsonMongoDBRepository.java index 3ae9f676..5959f002 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/repository/LunaticJsonMongoDBRepository.java +++ b/src/main/java/fr/insee/genesis/infrastructure/repository/LunaticJsonMongoDBRepository.java @@ -23,6 +23,12 @@ public interface LunaticJsonMongoDBRepository extends MongoRepository findModesByCampaignId(String campaignId); + @Aggregation(pipeline = { + "{ '$match': { 'questionnaireId': ?0 } }", + "{ '$project': { '_id': 0, 'mode': 1 } }" + }) + List findModesByQuestionnaireId(String questionnaireId); + @Query(value = "{ 'campaignId' : ?0, 'mode' : ?1, 'interrogationId': {$in: ?2} }") List findModesByCampaignIdAndByModeAndinterrogationIdIninterrogationIdList(String campaignName, Mode mode, List interrogationIdList); diff --git a/src/main/java/fr/insee/genesis/infrastructure/repository/RawResponseRepository.java b/src/main/java/fr/insee/genesis/infrastructure/repository/RawResponseRepository.java index 699a9c97..894f1f48 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/repository/RawResponseRepository.java +++ b/src/main/java/fr/insee/genesis/infrastructure/repository/RawResponseRepository.java @@ -1,5 +1,6 @@ package fr.insee.genesis.infrastructure.repository; +import fr.insee.genesis.domain.model.surveyunit.Mode; import fr.insee.genesis.infrastructure.document.rawdata.RawResponseDocument; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; @@ -29,8 +30,15 @@ public interface RawResponseRepository extends MongoRepository findInterrogationIdByCollectionInstrumentIdAndProcessDateIsNull(String collectionInstrumentId); + @Aggregation(pipeline = { + "{ '$match': { 'collectionInstrumentId': ?0 } }", + "{ '$project': { '_id': 0, 'mode': 1 } }" + }) + List findModesByCollectionInstrumentId(String collectionInstrumentId); + @Query(value = "{ 'payload.campaignId' : ?0, 'recordDate' : { $gte: ?1, $lte: ?2 } }") Page findByCampaignIdAndDate(String campaignId, Instant startDate, Instant endDate, Pageable pageable); + @Query(value = "{ 'interrogationId': ?0}") List findByInterrogationId(String interrogationId); } diff --git a/src/test/java/fr/insee/genesis/controller/rest/responses/RawResponseControllerUnitTest.java b/src/test/java/fr/insee/genesis/controller/rest/responses/RawResponseControllerUnitTest.java new file mode 100644 index 00000000..9d23152e --- /dev/null +++ b/src/test/java/fr/insee/genesis/controller/rest/responses/RawResponseControllerUnitTest.java @@ -0,0 +1,49 @@ +package fr.insee.genesis.controller.rest.responses; + +import fr.insee.genesis.domain.ports.api.LunaticJsonRawDataApiPort; +import fr.insee.genesis.domain.ports.api.RawResponseApiPort; +import fr.insee.genesis.infrastructure.repository.RawResponseInputRepository; +import org.assertj.core.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.http.ResponseEntity; + +import java.util.HashSet; +import java.util.Set; + +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; + +class RawResponseControllerUnitTest { + + static RawResponseController rawResponseController; + static LunaticJsonRawDataApiPort lunaticJsonRawDataStub; + + @BeforeEach + void init(){ + lunaticJsonRawDataStub = mock(LunaticJsonRawDataApiPort.class); + rawResponseController = new RawResponseController( + lunaticJsonRawDataStub, + mock(RawResponseApiPort.class), + mock(RawResponseInputRepository.class) + ); + } + + + @Test + void getUnprocessedJsonRawDataQuestionnairesIds_test(){ + // GIVEN + Set questionnaireIds = new HashSet<>(); + questionnaireIds.add("QUEST1"); + questionnaireIds.add("QUEST2"); + doReturn(questionnaireIds).when(lunaticJsonRawDataStub).getUnprocessedDataQuestionnaireIds(); + + // WHEN + ResponseEntity> response = rawResponseController.getUnprocessedJsonRawDataQuestionnairesIds(); + + // THEN + Assertions.assertThat(response.getStatusCode().is2xxSuccessful()).isTrue(); + Assertions.assertThat(response.getBody()).isNotNull().containsExactlyInAnyOrder("QUEST1","QUEST2"); + } + +} diff --git a/src/test/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataServiceUnitTest.java b/src/test/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataServiceUnitTest.java new file mode 100644 index 00000000..68482047 --- /dev/null +++ b/src/test/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataServiceUnitTest.java @@ -0,0 +1,103 @@ +package fr.insee.genesis.domain.service.rawdata; + +import fr.insee.bpm.metadata.model.MetadataModel; +import fr.insee.genesis.controller.utils.ControllerUtils; +import fr.insee.genesis.domain.model.surveyunit.Mode; +import fr.insee.genesis.domain.ports.spi.DataProcessingContextPersistancePort; +import fr.insee.genesis.domain.ports.spi.LunaticJsonRawDataPersistencePort; +import fr.insee.genesis.domain.ports.spi.QuestionnaireMetadataPersistencePort; +import fr.insee.genesis.domain.ports.spi.SurveyUnitQualityToolPort; +import fr.insee.genesis.domain.service.context.DataProcessingContextService; +import fr.insee.genesis.domain.service.metadata.QuestionnaireMetadataService; +import fr.insee.genesis.domain.service.surveyunit.SurveyUnitQualityService; +import fr.insee.genesis.domain.service.surveyunit.SurveyUnitService; +import fr.insee.genesis.infrastructure.utils.FileUtils; +import fr.insee.genesis.stubs.ConfigStub; +import lombok.SneakyThrows; +import org.assertj.core.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.util.HashSet; +import java.util.Set; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; + + +class LunaticJsonRawDataServiceUnitTest { + + static LunaticJsonRawDataService lunaticJsonRawDataService; + + static LunaticJsonRawDataPersistencePort lunaticJsonRawDataPersistencePort; + + static QuestionnaireMetadataService metadataService; + + @BeforeEach + void init() { + lunaticJsonRawDataPersistencePort = mock(LunaticJsonRawDataPersistencePort.class); + metadataService = mock(QuestionnaireMetadataService.class); + lunaticJsonRawDataService = new LunaticJsonRawDataService( + lunaticJsonRawDataPersistencePort, + new ControllerUtils(new FileUtils(new ConfigStub())), + metadataService, + mock(SurveyUnitService.class), + mock(SurveyUnitQualityService.class), + new FileUtils(new ConfigStub()), + mock(DataProcessingContextService.class), + mock(SurveyUnitQualityToolPort.class), + new ConfigStub(), + mock(DataProcessingContextPersistancePort.class) + ); + } + + @Test + @SneakyThrows + void getUnprocessedDataQuestionnaireIds() { + //GIVEN + Set questionnaireIds = new HashSet<>(); + questionnaireIds.add("QUEST1"); + questionnaireIds.add("QUEST2"); + doReturn(questionnaireIds).when(lunaticJsonRawDataPersistencePort).findDistinctQuestionnaireIdsByNullProcessDate(); + doReturn(Set.of(Mode.WEB)).when(lunaticJsonRawDataPersistencePort).findModesByQuestionnaire(any()); + doReturn(new MetadataModel()).when(metadataService).loadAndSaveIfNotExists(any(), any(), any(), any(), any()); + + + //WHEN + THEN + Assertions.assertThat(lunaticJsonRawDataService.getUnprocessedDataQuestionnaireIds()) + .containsExactlyInAnyOrder("QUEST1","QUEST2"); + } + + @Test + @SneakyThrows + void getUnprocessedDataQuestionnaireIds_shouldnt_return_if_no_spec() { + //GIVEN + Set questionnaireIds = new HashSet<>(); + questionnaireIds.add("QUEST1"); //No spec + questionnaireIds.add("TEST-TABLEAUX"); + doReturn(questionnaireIds).when(lunaticJsonRawDataPersistencePort).findDistinctQuestionnaireIdsByNullProcessDate(); + doReturn(Set.of(Mode.WEB)).when(lunaticJsonRawDataPersistencePort).findModesByQuestionnaire(any()); + //No mock for metadataservice this time + metadataService = new QuestionnaireMetadataService( + mock(QuestionnaireMetadataPersistencePort.class) + ); + lunaticJsonRawDataService = new LunaticJsonRawDataService( + lunaticJsonRawDataPersistencePort, + new ControllerUtils(new FileUtils(new ConfigStub())), + metadataService, + mock(SurveyUnitService.class), + mock(SurveyUnitQualityService.class), + new FileUtils(new ConfigStub()), + mock(DataProcessingContextService.class), + mock(SurveyUnitQualityToolPort.class), + new ConfigStub(), + mock(DataProcessingContextPersistancePort.class) + ); + + + //WHEN + THEN + Assertions.assertThat(lunaticJsonRawDataService.getUnprocessedDataQuestionnaireIds()) + .containsExactly("TEST-TABLEAUX"); + } +} \ No newline at end of file diff --git a/src/test/java/fr/insee/genesis/domain/service/rawdata/RawResponseServiceUnitTest.java b/src/test/java/fr/insee/genesis/domain/service/rawdata/RawResponseServiceUnitTest.java index 8bacf28d..39d210dc 100644 --- a/src/test/java/fr/insee/genesis/domain/service/rawdata/RawResponseServiceUnitTest.java +++ b/src/test/java/fr/insee/genesis/domain/service/rawdata/RawResponseServiceUnitTest.java @@ -7,6 +7,7 @@ import fr.insee.genesis.domain.model.surveyunit.Mode; import fr.insee.genesis.domain.model.surveyunit.SurveyUnitModel; import fr.insee.genesis.domain.model.surveyunit.rawdata.RawResponseModel; +import fr.insee.genesis.domain.ports.spi.QuestionnaireMetadataPersistencePort; import fr.insee.genesis.domain.ports.spi.RawResponsePersistencePort; import fr.insee.genesis.domain.ports.spi.SurveyUnitQualityToolPort; import fr.insee.genesis.domain.service.context.DataProcessingContextService; @@ -76,6 +77,54 @@ void init() { surveyUnitModelsCaptor = ArgumentCaptor.forClass(List.class); } + @Test + @SneakyThrows + void getUnprocessedCollectionInstrumentIds_test() { + //GIVEN + List collectionInstrumentIds = new ArrayList<>(); + collectionInstrumentIds.add("QUEST1"); + collectionInstrumentIds.add("QUEST2"); + doReturn(collectionInstrumentIds).when(rawResponsePersistencePort).getUnprocessedCollectionIds(); + doReturn(List.of(Mode.WEB)).when(rawResponsePersistencePort).findModesByCollectionInstrument(any()); + doReturn(new MetadataModel()).when(metadataService).loadAndSaveIfNotExists(any(), any(), any(), any(), any()); + + + //WHEN + THEN + Assertions.assertThat(rawResponseService.getUnprocessedCollectionInstrumentIds()) + .containsExactlyInAnyOrder("QUEST1","QUEST2"); + } + + @Test + @SneakyThrows + void getUnprocessedCollectionInstrumentIds_shouldnt_return_if_no_spec() { + //GIVEN + List questionnaireIds = new ArrayList<>(); + questionnaireIds.add("QUEST1"); //No spec + questionnaireIds.add("TEST-TABLEAUX"); + doReturn(questionnaireIds).when(rawResponsePersistencePort).getUnprocessedCollectionIds(); + doReturn(List.of(Mode.WEB)).when(rawResponsePersistencePort).findModesByCollectionInstrument(any()); + //No mock for metadataservice this time + metadataService = new QuestionnaireMetadataService( + mock(QuestionnaireMetadataPersistencePort.class) + ); + rawResponseService = new RawResponseService( + new ControllerUtils(new FileUtils(new ConfigStub())), + metadataService, + mock(SurveyUnitService.class), + mock(SurveyUnitQualityService.class), + mock(SurveyUnitQualityToolPort.class), + mock(DataProcessingContextService.class), + new FileUtils(new ConfigStub()), + new ConfigStub(), + rawResponsePersistencePort + ); + + + //WHEN + THEN + Assertions.assertThat(rawResponseService.getUnprocessedCollectionInstrumentIds()) + .containsExactly("TEST-TABLEAUX"); + } + @Nested @DisplayName("Non regression tests of #22875 : validation date and questionnaire state in processed responses") class ValidationDateAndQuestionnaireStateTests{ diff --git a/src/test/java/fr/insee/genesis/infrastructure/adapter/LunaticJsonRawDataMongoAdapterTest.java b/src/test/java/fr/insee/genesis/infrastructure/adapter/LunaticJsonRawDataMongoAdapterTest.java index 80627450..58d6ceef 100644 --- a/src/test/java/fr/insee/genesis/infrastructure/adapter/LunaticJsonRawDataMongoAdapterTest.java +++ b/src/test/java/fr/insee/genesis/infrastructure/adapter/LunaticJsonRawDataMongoAdapterTest.java @@ -74,5 +74,4 @@ void findRawDataTest(){ List rawdatas = adapter.findRawData("campaign01",Mode.WEB,List.of("interrogation01")); Assertions.assertThat(rawdatas).hasSize(1); } - } \ No newline at end of file diff --git a/src/test/java/fr/insee/genesis/stubs/LunaticJsonMongoDBRepositoryStub.java b/src/test/java/fr/insee/genesis/stubs/LunaticJsonMongoDBRepositoryStub.java index 76822695..efe0a4d9 100644 --- a/src/test/java/fr/insee/genesis/stubs/LunaticJsonMongoDBRepositoryStub.java +++ b/src/test/java/fr/insee/genesis/stubs/LunaticJsonMongoDBRepositoryStub.java @@ -46,6 +46,15 @@ public List findModesByCampaignId(String campaignId) { .toList(); } + @Override + public List findModesByQuestionnaireId(String questionnaireId) { + return documents.stream() + .filter(doc -> Objects.equals(doc.questionnaireId(), questionnaireId)) + .map(LunaticJsonRawDataDocument::mode) + .distinct() + .toList(); + } + @Override public List findModesByCampaignIdAndByModeAndinterrogationIdIninterrogationIdList( String campaignName, Mode mode, List interrogationIdList) { diff --git a/src/test/java/fr/insee/genesis/stubs/LunaticJsonRawDataPersistanceStub.java b/src/test/java/fr/insee/genesis/stubs/LunaticJsonRawDataPersistanceStub.java index d00e7040..abaa729b 100644 --- a/src/test/java/fr/insee/genesis/stubs/LunaticJsonRawDataPersistanceStub.java +++ b/src/test/java/fr/insee/genesis/stubs/LunaticJsonRawDataPersistanceStub.java @@ -21,6 +21,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; @Getter @@ -42,6 +43,28 @@ public List getAllUnprocessedData() { ); } + @Override + public Set findDistinctQuestionnaireIdsByNullProcessDate() { + Set questionnaireIds = new HashSet<>(); + mongoStub.stream().filter( + lunaticJsonDataDocument -> lunaticJsonDataDocument.processDate() == null + ).forEach(doc -> { + if(doc.questionnaireId() != null){ + questionnaireIds.add(doc.questionnaireId()); + } + }); + return questionnaireIds; + } + + @Override + public Set findModesByQuestionnaire(String questionnaireId) { + return new HashSet<>(mongoStub.stream() + .filter(doc -> Objects.equals(doc.questionnaireId(), questionnaireId)) + .map(LunaticJsonRawDataDocument::mode) + .distinct() + .toList()); + } + @Override public List findRawData(String campaignName, Mode mode, List interrogationIdList) { List docs = mongoStub.stream().filter(lunaticJsonRawDataDocument -> diff --git a/src/test/java/fr/insee/genesis/stubs/RawResponseDataPersistanceStub.java b/src/test/java/fr/insee/genesis/stubs/RawResponseDataPersistanceStub.java index 24265282..5490b652 100644 --- a/src/test/java/fr/insee/genesis/stubs/RawResponseDataPersistanceStub.java +++ b/src/test/java/fr/insee/genesis/stubs/RawResponseDataPersistanceStub.java @@ -50,6 +50,17 @@ public Set findUnprocessedInterrogationIdsByCollectionInstrumentId(Strin return Set.of(); } + @Override + public List findModesByCollectionInstrument(String collectionInstrumentId) { + List modes = new ArrayList<>(); + mongoStub.stream().filter( + doc -> doc.collectionInstrumentId().equals(collectionInstrumentId) + ).forEach( + rawResponseDocument -> modes.add(Mode.valueOf(rawResponseDocument.mode())) + ); + return modes; + } + @Override public Page findByCampaignIdAndDate(String campaignId, Instant startDate, Instant endDate, Pageable pageable) { List foundRaws = mongoStub.stream()