diff --git a/src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseController.java b/src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseController.java index f2ff8da6..ec4d9aa0 100644 --- a/src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseController.java +++ b/src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseController.java @@ -12,6 +12,7 @@ import fr.insee.genesis.domain.model.surveyunit.Mode; import fr.insee.genesis.domain.model.surveyunit.rawdata.DataProcessResult; import fr.insee.genesis.domain.model.surveyunit.rawdata.LunaticJsonRawDataModel; +import fr.insee.genesis.domain.model.surveyunit.rawdata.RawResponseModel; import fr.insee.genesis.domain.ports.api.LunaticJsonRawDataApiPort; import fr.insee.genesis.domain.ports.api.RawResponseApiPort; import fr.insee.genesis.exceptions.GenesisError; @@ -270,7 +271,7 @@ public ResponseEntity>> getProcessedDataIdsSinceHours( @Operation(summary = "Get lunatic JSON data from one campaign in Genesis Database, filtered by start and end dates") @GetMapping(path = "/responses/raw/lunatic-json/{campaignId}") @PreAuthorize("hasRole('USER_BATCH_GENERIC')") - public ResponseEntity> getRawResponsesFromJsonBody( + public ResponseEntity> getLunaticJsonRawDataModelFromJsonBody( @PathVariable String campaignId, @RequestParam(value = "startDate", required = false) @DateTimeFormat(iso = DateTimeFormat.ISO.DATE_TIME) Instant startDate, @RequestParam(value = "endDate", required = false) @DateTimeFormat(iso = DateTimeFormat.ISO.DATE_TIME) Instant endDate, @@ -284,5 +285,20 @@ public ResponseEntity> getRawResponsesFromJs return ResponseEntity.status(HttpStatus.OK).body(new PagedModel<>(rawResponses)); } - + @Operation(summary = "Get rawResponse JSON data from one campaign in Genesis Database, filtered by start and end dates") + @GetMapping(path = "/raw-responses/{campaignId}") + @PreAuthorize("hasRole('USER_BATCH_GENERIC')") + public ResponseEntity> getRawResponsesFromJsonBody( + @PathVariable String campaignId, + @RequestParam(value = "startDate", required = false) @DateTimeFormat(iso = DateTimeFormat.ISO.DATE_TIME) Instant startDate, + @RequestParam(value = "endDate", required = false) @DateTimeFormat(iso = DateTimeFormat.ISO.DATE_TIME) Instant endDate, + @RequestParam(value = "page", defaultValue = "0") int page, + @RequestParam(value = "size", defaultValue = "1000") int size + ) { + log.info("Try to read raw JSONs for campaign {}, with startDate={} and endDate={} - page={} - size={}", campaignId, startDate, endDate,page,size); + Pageable pageable = PageRequest.of(page, size); + Page rawResponses = rawResponseApiPort.findRawResponseDataByCampaignIdAndDate(campaignId, startDate, endDate, pageable); + log.info("rawResponses={}", rawResponses.getContent().size()); + return ResponseEntity.status(HttpStatus.OK).body(new PagedModel<>(rawResponses)); + } } diff --git a/src/main/java/fr/insee/genesis/domain/model/surveyunit/rawdata/RawResponseModel.java b/src/main/java/fr/insee/genesis/domain/model/surveyunit/rawdata/RawResponseModel.java index fdd87038..f92606af 100644 --- a/src/main/java/fr/insee/genesis/domain/model/surveyunit/rawdata/RawResponseModel.java +++ b/src/main/java/fr/insee/genesis/domain/model/surveyunit/rawdata/RawResponseModel.java @@ -1,6 +1,7 @@ package fr.insee.genesis.domain.model.surveyunit.rawdata; import fr.insee.genesis.domain.model.surveyunit.Mode; +import lombok.Builder; import org.bson.types.ObjectId; import java.time.LocalDateTime; diff --git a/src/main/java/fr/insee/genesis/domain/ports/api/RawResponseApiPort.java b/src/main/java/fr/insee/genesis/domain/ports/api/RawResponseApiPort.java index 9245c47e..2b13bd31 100644 --- a/src/main/java/fr/insee/genesis/domain/ports/api/RawResponseApiPort.java +++ b/src/main/java/fr/insee/genesis/domain/ports/api/RawResponseApiPort.java @@ -7,7 +7,10 @@ import fr.insee.genesis.domain.model.surveyunit.rawdata.RawResponseModel; import fr.insee.genesis.exceptions.GenesisError; import fr.insee.genesis.exceptions.GenesisException; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; +import java.time.Instant; import java.util.List; public interface RawResponseApiPort { @@ -19,5 +22,5 @@ public interface RawResponseApiPort { List convertRawResponse(List rawResponses, VariablesMap variablesMap); List getUnprocessedCollectionInstrumentIds(); void updateProcessDates(List surveyUnitModels); - + Page findRawResponseDataByCampaignIdAndDate(String campaignId, Instant startDate, Instant endDate, Pageable pageable); } diff --git a/src/main/java/fr/insee/genesis/domain/ports/spi/RawResponsePersistencePort.java b/src/main/java/fr/insee/genesis/domain/ports/spi/RawResponsePersistencePort.java index 512b8f46..17c25dce 100644 --- a/src/main/java/fr/insee/genesis/domain/ports/spi/RawResponsePersistencePort.java +++ b/src/main/java/fr/insee/genesis/domain/ports/spi/RawResponsePersistencePort.java @@ -2,7 +2,10 @@ import fr.insee.genesis.domain.model.surveyunit.Mode; import fr.insee.genesis.domain.model.surveyunit.rawdata.RawResponseModel; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; +import java.time.Instant; import java.util.List; import java.util.Set; @@ -13,4 +16,5 @@ public interface RawResponsePersistencePort { void updateProcessDates(String collectionInstrumentId, Set interrogationIds); List getUnprocessedCollectionIds(); Set findUnprocessedInterrogationIdsByCollectionInstrumentId(String collectionInstrumentId); - } + Page findByCampaignIdAndDate(String campaignId, Instant startDate, Instant endDate, Pageable pageable); +} diff --git a/src/main/java/fr/insee/genesis/domain/service/rawdata/RawResponseService.java b/src/main/java/fr/insee/genesis/domain/service/rawdata/RawResponseService.java index 269f0ab9..ca1fbc29 100644 --- a/src/main/java/fr/insee/genesis/domain/service/rawdata/RawResponseService.java +++ b/src/main/java/fr/insee/genesis/domain/service/rawdata/RawResponseService.java @@ -26,10 +26,13 @@ import fr.insee.modelefiliere.RawResponseDto; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; import org.springframework.http.ResponseEntity; import org.springframework.stereotype.Service; import java.io.IOException; +import java.time.Instant; import java.time.LocalDateTime; import java.time.format.DateTimeFormatter; import java.util.ArrayList; @@ -459,4 +462,9 @@ private static void convertToExternalVar(SurveyUnitModel dstSurveyUnitModel, Var } } } + + @Override + public Page findRawResponseDataByCampaignIdAndDate(String campaignId, Instant startDate, Instant endDate, Pageable pageable) { + return rawResponsePersistencePort.findByCampaignIdAndDate(campaignId,startDate, endDate,pageable); + } } diff --git a/src/main/java/fr/insee/genesis/infrastructure/adapter/RawResponseMongoAdapter.java b/src/main/java/fr/insee/genesis/infrastructure/adapter/RawResponseMongoAdapter.java index 56f5b5e3..1c95f507 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/adapter/RawResponseMongoAdapter.java +++ b/src/main/java/fr/insee/genesis/infrastructure/adapter/RawResponseMongoAdapter.java @@ -9,12 +9,16 @@ import fr.insee.genesis.infrastructure.repository.RawResponseRepository; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.PageImpl; +import org.springframework.data.domain.Pageable; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.data.mongodb.core.query.Criteria; import org.springframework.data.mongodb.core.query.Query; import org.springframework.data.mongodb.core.query.Update; import org.springframework.stereotype.Service; +import java.time.Instant; import java.time.LocalDateTime; import java.util.HashSet; import java.util.List; @@ -64,4 +68,11 @@ public Set findUnprocessedInterrogationIdsByCollectionInstrumentId(Strin // We remove duplicate ids return new HashSet<>(repository.findInterrogationIdByCollectionInstrumentIdAndProcessDateIsNull(collectionInstrumentId)); } + + @Override + public Page findByCampaignIdAndDate(String campaignId, Instant startDate, Instant endDate, Pageable pageable) { + Page rawDataDocs = repository.findByCampaignIdAndDate(campaignId, startDate, endDate, pageable); + List modelList = RawResponseDocumentMapper.INSTANCE.listDocumentToListModel(rawDataDocs.getContent()); + return new PageImpl<>(modelList, rawDataDocs.getPageable(), rawDataDocs.getTotalElements()); + } } diff --git a/src/main/java/fr/insee/genesis/infrastructure/document/rawdata/RawResponseDocument.java b/src/main/java/fr/insee/genesis/infrastructure/document/rawdata/RawResponseDocument.java index f969c86c..5b4d705a 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/document/rawdata/RawResponseDocument.java +++ b/src/main/java/fr/insee/genesis/infrastructure/document/rawdata/RawResponseDocument.java @@ -22,5 +22,7 @@ public record RawResponseDocument ( Map payload, LocalDateTime recordDate, @Indexed(direction = IndexDirection.DESCENDING) - LocalDateTime processDate + LocalDateTime processDate, + @Indexed + String campaignId ){} diff --git a/src/main/java/fr/insee/genesis/infrastructure/repository/RawResponseRepository.java b/src/main/java/fr/insee/genesis/infrastructure/repository/RawResponseRepository.java index 4dd548e4..699a9c97 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/repository/RawResponseRepository.java +++ b/src/main/java/fr/insee/genesis/infrastructure/repository/RawResponseRepository.java @@ -1,11 +1,14 @@ package fr.insee.genesis.infrastructure.repository; import fr.insee.genesis.infrastructure.document.rawdata.RawResponseDocument; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; import org.springframework.data.mongodb.repository.Aggregation; import org.springframework.data.mongodb.repository.MongoRepository; import org.springframework.data.mongodb.repository.Query; import org.springframework.stereotype.Repository; +import java.time.Instant; import java.util.List; @Repository @@ -26,6 +29,8 @@ public interface RawResponseRepository extends MongoRepository findInterrogationIdByCollectionInstrumentIdAndProcessDateIsNull(String collectionInstrumentId); + @Query(value = "{ 'payload.campaignId' : ?0, 'recordDate' : { $gte: ?1, $lte: ?2 } }") + Page findByCampaignIdAndDate(String campaignId, Instant startDate, Instant endDate, Pageable pageable); @Query(value = "{ 'interrogationId': ?0}") List findByInterrogationId(String interrogationId); } diff --git a/src/test/java/cucumber/functional_tests/RawDataDefinitions.java b/src/test/java/cucumber/functional_tests/RawDataDefinitions.java index 43922a4c..dc0587eb 100644 --- a/src/test/java/cucumber/functional_tests/RawDataDefinitions.java +++ b/src/test/java/cucumber/functional_tests/RawDataDefinitions.java @@ -39,6 +39,8 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.web.client.TestRestTemplate; import org.springframework.boot.test.web.server.LocalServerPort; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; import org.springframework.http.HttpEntity; import org.springframework.http.HttpHeaders; import org.springframework.http.HttpMethod; @@ -54,6 +56,7 @@ import java.security.KeyPairGenerator; import java.security.NoSuchAlgorithmException; import java.security.interfaces.RSAPublicKey; +import java.time.Instant; import java.util.Collections; import java.util.List; @@ -139,6 +142,11 @@ public List getUnprocessedCollectionInstrumentIds() { public void updateProcessDates(List surveyUnitModels) { // Do nothing - stub for test } + + @Override + public Page findRawResponseDataByCampaignIdAndDate(String campaignId, Instant startDate, Instant endDate, Pageable pageable) { + return null; + } }; RawResponseController rawResponseController = new RawResponseController( diff --git a/src/test/java/fr/insee/genesis/controller/rest/responses/RawResponseControllerTest.java b/src/test/java/fr/insee/genesis/controller/rest/responses/RawResponseControllerTest.java index 2697c936..19ba4eb9 100644 --- a/src/test/java/fr/insee/genesis/controller/rest/responses/RawResponseControllerTest.java +++ b/src/test/java/fr/insee/genesis/controller/rest/responses/RawResponseControllerTest.java @@ -22,6 +22,7 @@ import fr.insee.genesis.exceptions.GenesisError; import fr.insee.genesis.exceptions.GenesisException; import fr.insee.genesis.infrastructure.document.rawdata.LunaticJsonRawDataDocument; +import fr.insee.genesis.infrastructure.document.rawdata.RawResponseDocument; import fr.insee.genesis.infrastructure.mappers.DataProcessingContextMapper; import fr.insee.genesis.infrastructure.repository.RawResponseInputRepository; import fr.insee.genesis.infrastructure.utils.FileUtils; @@ -29,12 +30,15 @@ import fr.insee.genesis.stubs.DataProcessingContextPersistancePortStub; import fr.insee.genesis.stubs.LunaticJsonRawDataPersistanceStub; import fr.insee.genesis.stubs.QuestionnaireMetadataPersistencePortStub; +import fr.insee.genesis.stubs.RawResponseDataPersistanceStub; import fr.insee.genesis.stubs.SurveyUnitPersistencePortStub; import fr.insee.genesis.stubs.SurveyUnitQualityToolPerretAdapterStub; import fr.insee.modelefiliere.RawResponseDto; import lombok.extern.slf4j.Slf4j; import org.assertj.core.api.Assertions; import org.junit.jupiter.api.Test; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; import org.springframework.data.web.PagedModel; import org.springframework.http.ResponseEntity; @@ -49,6 +53,7 @@ class RawResponseControllerTest { private final FileUtils fileUtils = new FileUtils(new ConfigStub()); private final LunaticJsonRawDataPersistanceStub lunaticJsonRawDataPersistanceStub = new LunaticJsonRawDataPersistanceStub(); + private final RawResponseDataPersistanceStub rawResponseDataPersistanceStub = new RawResponseDataPersistanceStub(); private final SurveyUnitPersistencePortStub surveyUnitPersistencePortStub = new SurveyUnitPersistencePortStub(); private final SurveyUnitQualityToolPerretAdapterStub surveyUnitQualityToolPerretAdapterStub = new SurveyUnitQualityToolPerretAdapterStub(); private final DataProcessingContextPersistancePortStub dataProcessingContextPersistancePortStub = @@ -112,6 +117,10 @@ public List getUnprocessedCollectionInstrumentIds() { public void updateProcessDates(List surveyUnitModels) { } + @Override + public Page findRawResponseDataByCampaignIdAndDate(String campaignId, Instant startDate, Instant endDate, Pageable pageable) { + return rawResponseDataPersistanceStub.findByCampaignIdAndDate(campaignId, startDate, endDate, pageable); + } }; private final RawResponseController rawResponseController = new RawResponseController(lunaticJsonRawDataApiPort, rawResponseApiPortStub, rawResponseInputRepositoryStub); @@ -314,11 +323,10 @@ void processJsonRawDataV2Test(){ .contains(interrogationId); } - @Test - void getRawResponsesFromJsonBody() { + void getLunaticJsonRawDataModelFromJsonBody() { //GIVEN - String campaignId = "getRawResponsesFromJsonBody"; + String campaignId = "getLunaticJsonRawDataModelFromJsonBody"; String questionnaireId = campaignId + "_quest"; String interrogationId = "getRawResponsesFromJsonBody_id1"; String varName = "VARName1"; @@ -335,7 +343,29 @@ void getRawResponsesFromJsonBody() { int page=0, size= 10; //WHEN - ResponseEntity> response = rawResponseController.getRawResponsesFromJsonBody(campaignId, starDate, endDate, page, size); + ResponseEntity> response = rawResponseController.getLunaticJsonRawDataModelFromJsonBody(campaignId, starDate, endDate, page, size); + + //THEN + Assertions.assertThat(response.getStatusCode().is2xxSuccessful()).isTrue(); + Assertions.assertThat(response.getBody()).isNotNull(); + Assertions.assertThat(response.getBody().getContent()).hasSize(1); + } + + @Test + void getRawResponsesFromJsonBody() { + //GIVEN + String campaignId = "getRawResponsesFromJsonBody"; + String questionnaireId = campaignId + "_quest"; + String interrogationId = "getRawResponsesFromJsonBody_id1"; + Instant recordDate = Instant.parse("2025-01-01T01:00:00.000Z"); + + addJsonRawResponseDataDocumentToStub(campaignId, questionnaireId, interrogationId); + + Instant starDate= recordDate.minusSeconds(86400),endDate = recordDate.plusSeconds(86400); + int page=0, size= 10; + + //WHEN + ResponseEntity> response = rawResponseController.getRawResponsesFromJsonBody(campaignId, starDate, endDate, page, size); //THEN Assertions.assertThat(response.getStatusCode().is2xxSuccessful()).isTrue(); @@ -385,4 +415,15 @@ private void addJsonRawDataDocumentToStub(String campaignId, lunaticJsonRawDataPersistanceStub.getMongoStub().add(lunaticJsonDataDocument); } + + private void addJsonRawResponseDataDocumentToStub(String campaignId, String questionnaireId, String interrogationId) { + RawResponseDocument rawResponseDocument = RawResponseDocument.builder() + .campaignId(campaignId) + .collectionInstrumentId(questionnaireId) + .interrogationId(interrogationId) + .recordDate(LocalDateTime.now()) + .build(); + + rawResponseDataPersistanceStub.getMongoStub().add(rawResponseDocument); + } } diff --git a/src/test/java/fr/insee/genesis/stubs/RawResponseDataPersistanceStub.java b/src/test/java/fr/insee/genesis/stubs/RawResponseDataPersistanceStub.java new file mode 100644 index 00000000..24265282 --- /dev/null +++ b/src/test/java/fr/insee/genesis/stubs/RawResponseDataPersistanceStub.java @@ -0,0 +1,62 @@ +package fr.insee.genesis.stubs; + +import fr.insee.genesis.domain.model.surveyunit.Mode; +import fr.insee.genesis.domain.model.surveyunit.rawdata.RawResponseModel; +import fr.insee.genesis.domain.ports.spi.RawResponsePersistencePort; +import fr.insee.genesis.infrastructure.document.rawdata.RawResponseDocument; +import fr.insee.genesis.infrastructure.mappers.RawResponseDocumentMapper; +import lombok.Getter; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.PageImpl; +import org.springframework.data.domain.Pageable; + +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +@Getter +public class RawResponseDataPersistanceStub implements RawResponsePersistencePort { + List mongoStub = new ArrayList<>(); + + @Override + public List findRawResponses(String collectionInstrumentId, Mode mode, List interrogationIdList) { + return RawResponseDocumentMapper.INSTANCE.listDocumentToListModel(mongoStub.stream().filter( + doc -> doc.collectionInstrumentId().equals(collectionInstrumentId) + && Mode.valueOf(doc.mode()).equals(mode) + && interrogationIdList.contains(doc.interrogationId()) + ).toList()); + } + + @Override + public List findRawResponsesByInterrogationID(String interrogationId) { + return RawResponseDocumentMapper.INSTANCE.listDocumentToListModel(mongoStub.stream().filter( + doc -> doc.interrogationId().equals(interrogationId) + ).toList()); + } + + @Override + public void updateProcessDates(String collectionInstrumentId, Set interrogationIds) { + + } + + @Override + public List getUnprocessedCollectionIds() { + return List.of(); + } + + @Override + public Set findUnprocessedInterrogationIdsByCollectionInstrumentId(String collectionInstrumentId) { + return Set.of(); + } + + @Override + public Page findByCampaignIdAndDate(String campaignId, Instant startDate, Instant endDate, Pageable pageable) { + List foundRaws = mongoStub.stream() + .filter(rawData -> rawData.campaignId().equals(campaignId)) + .toList(); + return new PageImpl<>(RawResponseDocumentMapper.INSTANCE.listDocumentToListModel(foundRaws), + pageable, + foundRaws.size()); + } +}