Feat add endpoint raw responses by campaign id and between

src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseController.java

@@ -12,6 +12,7 @@
 import fr.insee.genesis.domain.model.surveyunit.Mode;
 import fr.insee.genesis.domain.model.surveyunit.rawdata.DataProcessResult;
 import fr.insee.genesis.domain.model.surveyunit.rawdata.LunaticJsonRawDataModel;
+import fr.insee.genesis.domain.model.surveyunit.rawdata.RawResponseModel;
 import fr.insee.genesis.domain.ports.api.LunaticJsonRawDataApiPort;
 import fr.insee.genesis.domain.ports.api.RawResponseApiPort;
 import fr.insee.genesis.exceptions.GenesisError;
@@ -270,7 +271,7 @@ public ResponseEntity<Map<String, List<String>>> getProcessedDataIdsSinceHours(
     @Operation(summary = "Get lunatic JSON data from one campaign in Genesis Database, filtered by start and end dates")
     @GetMapping(path = "/responses/raw/lunatic-json/{campaignId}")
     @PreAuthorize("hasRole('USER_BATCH_GENERIC')")
-    public ResponseEntity<PagedModel<LunaticJsonRawDataModel>> getRawResponsesFromJsonBody(
+    public ResponseEntity<PagedModel<LunaticJsonRawDataModel>> getLunaticJsonRawDataModelFromJsonBody(
             @PathVariable String campaignId,
             @RequestParam(value = "startDate", required = false) @DateTimeFormat(iso = DateTimeFormat.ISO.DATE_TIME) Instant startDate,
             @RequestParam(value = "endDate", required = false) @DateTimeFormat(iso = DateTimeFormat.ISO.DATE_TIME) Instant endDate,
@@ -284,5 +285,20 @@ public ResponseEntity<PagedModel<LunaticJsonRawDataModel>> getRawResponsesFromJs
         return ResponseEntity.status(HttpStatus.OK).body(new PagedModel<>(rawResponses));
     }
 
-
+    @Operation(summary = "Get rawResponse JSON data from one campaign in Genesis Database, filtered by start and end dates")
+    @GetMapping(path = "/raw-responses/{campaignId}")
+    @PreAuthorize("hasRole('USER_BATCH_GENERIC')")
+    public ResponseEntity<PagedModel<RawResponseModel>> getRawResponsesFromJsonBody(
+            @PathVariable String campaignId,
+            @RequestParam(value = "startDate", required = false) @DateTimeFormat(iso = DateTimeFormat.ISO.DATE_TIME) Instant startDate,
+            @RequestParam(value = "endDate", required = false) @DateTimeFormat(iso = DateTimeFormat.ISO.DATE_TIME) Instant endDate,
+            @RequestParam(value = "page", defaultValue = "0") int page,
+            @RequestParam(value = "size", defaultValue = "1000") int size
+    ) {
+        log.info("Try to read raw JSONs for campaign {}, with startDate={} and endDate={} - page={} - size={}", campaignId, startDate, endDate,page,size);
+        Pageable pageable = PageRequest.of(page, size);
+        Page<RawResponseModel> rawResponses = rawResponseApiPort.findRawResponseDataByCampaignIdAndDate(campaignId, startDate, endDate, pageable);
+        log.info("rawResponses={}", rawResponses.getContent().size());
+        return ResponseEntity.status(HttpStatus.OK).body(new PagedModel<>(rawResponses));
+    }
 }

src/main/java/fr/insee/genesis/domain/model/surveyunit/rawdata/RawResponseModel.java

@@ -1,6 +1,7 @@
 package fr.insee.genesis.domain.model.surveyunit.rawdata;
 
 import fr.insee.genesis.domain.model.surveyunit.Mode;
+import lombok.Builder;
 import org.bson.types.ObjectId;
 
 import java.time.LocalDateTime;

src/main/java/fr/insee/genesis/domain/ports/api/RawResponseApiPort.java

@@ -7,7 +7,10 @@
 import fr.insee.genesis.domain.model.surveyunit.rawdata.RawResponseModel;
 import fr.insee.genesis.exceptions.GenesisError;
 import fr.insee.genesis.exceptions.GenesisException;
+import org.springframework.data.domain.Page;
+import org.springframework.data.domain.Pageable;
 
+import java.time.Instant;
 import java.util.List;
 
 public interface RawResponseApiPort {
@@ -19,5 +22,5 @@ public interface RawResponseApiPort {
     List<SurveyUnitModel> convertRawResponse(List<RawResponseModel> rawResponses, VariablesMap variablesMap);
     List<String> getUnprocessedCollectionInstrumentIds();
     void updateProcessDates(List<SurveyUnitModel> surveyUnitModels);
-
+    Page<RawResponseModel> findRawResponseDataByCampaignIdAndDate(String campaignId, Instant startDate, Instant endDate, Pageable pageable);
 }

src/main/java/fr/insee/genesis/domain/ports/spi/RawResponsePersistencePort.java

@@ -2,7 +2,10 @@
 
 import fr.insee.genesis.domain.model.surveyunit.Mode;
 import fr.insee.genesis.domain.model.surveyunit.rawdata.RawResponseModel;
+import org.springframework.data.domain.Page;
+import org.springframework.data.domain.Pageable;
 
+import java.time.Instant;
 import java.util.List;
 import java.util.Set;
 
@@ -13,4 +16,5 @@ public interface RawResponsePersistencePort {
     void updateProcessDates(String collectionInstrumentId, Set<String> interrogationIds);
     List<String> getUnprocessedCollectionIds();
     Set<String> findUnprocessedInterrogationIdsByCollectionInstrumentId(String collectionInstrumentId);
- }
+    Page<RawResponseModel> findByCampaignIdAndDate(String campaignId, Instant startDate, Instant endDate, Pageable pageable);
+}

src/main/java/fr/insee/genesis/domain/service/rawdata/RawResponseService.java

@@ -26,10 +26,13 @@
 import fr.insee.modelefiliere.RawResponseDto;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.data.domain.Page;
+import org.springframework.data.domain.Pageable;
 import org.springframework.http.ResponseEntity;
 import org.springframework.stereotype.Service;
 
 import java.io.IOException;
+import java.time.Instant;
 import java.time.LocalDateTime;
 import java.time.format.DateTimeFormatter;
 import java.util.ArrayList;
@@ -459,4 +462,9 @@ private static void convertToExternalVar(SurveyUnitModel dstSurveyUnitModel, Var
             }
         }
     }
+
+    @Override
+    public Page<RawResponseModel> findRawResponseDataByCampaignIdAndDate(String campaignId, Instant startDate, Instant endDate, Pageable pageable) {
+        return rawResponsePersistencePort.findByCampaignIdAndDate(campaignId,startDate, endDate,pageable);
+    }
 }

src/main/java/fr/insee/genesis/infrastructure/adapter/RawResponseMongoAdapter.java

@@ -9,12 +9,16 @@
 import fr.insee.genesis.infrastructure.repository.RawResponseRepository;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.data.domain.Page;
+import org.springframework.data.domain.PageImpl;
+import org.springframework.data.domain.Pageable;
 import org.springframework.data.mongodb.core.MongoTemplate;
 import org.springframework.data.mongodb.core.query.Criteria;
 import org.springframework.data.mongodb.core.query.Query;
 import org.springframework.data.mongodb.core.query.Update;
 import org.springframework.stereotype.Service;
 
+import java.time.Instant;
 import java.time.LocalDateTime;
 import java.util.HashSet;
 import java.util.List;
@@ -64,4 +68,11 @@ public Set<String> findUnprocessedInterrogationIdsByCollectionInstrumentId(Strin
         // We remove duplicate ids
         return new HashSet<>(repository.findInterrogationIdByCollectionInstrumentIdAndProcessDateIsNull(collectionInstrumentId));
     }
+
+    @Override
+    public Page<RawResponseModel> findByCampaignIdAndDate(String campaignId, Instant startDate, Instant endDate, Pageable pageable) {
+        Page<RawResponseDocument> rawDataDocs = repository.findByCampaignIdAndDate(campaignId, startDate, endDate, pageable);
+        List<RawResponseModel> modelList = RawResponseDocumentMapper.INSTANCE.listDocumentToListModel(rawDataDocs.getContent());
+        return new PageImpl<>(modelList, rawDataDocs.getPageable(), rawDataDocs.getTotalElements());
+    }
 }

src/main/java/fr/insee/genesis/infrastructure/document/rawdata/RawResponseDocument.java

@@ -22,5 +22,7 @@ public record RawResponseDocument (
         Map<String,Object> payload,
         LocalDateTime recordDate,
         @Indexed(direction = IndexDirection.DESCENDING)
-        LocalDateTime processDate
+        LocalDateTime processDate,
+        @Indexed
+        String campaignId
 ){}

src/main/java/fr/insee/genesis/infrastructure/repository/RawResponseRepository.java

@@ -1,11 +1,14 @@
 package fr.insee.genesis.infrastructure.repository;
 
 import fr.insee.genesis.infrastructure.document.rawdata.RawResponseDocument;
+import org.springframework.data.domain.Page;
+import org.springframework.data.domain.Pageable;
 import org.springframework.data.mongodb.repository.Aggregation;
 import org.springframework.data.mongodb.repository.MongoRepository;
 import org.springframework.data.mongodb.repository.Query;
 import org.springframework.stereotype.Repository;
 
+import java.time.Instant;
 import java.util.List;
 
 @Repository
@@ -26,6 +29,8 @@ public interface RawResponseRepository extends MongoRepository<RawResponseDocume
     })
     List<String> findInterrogationIdByCollectionInstrumentIdAndProcessDateIsNull(String collectionInstrumentId);
 
+    @Query(value = "{ 'payload.campaignId' : ?0, 'recordDate' : { $gte: ?1, $lte: ?2 } }")
+    Page<RawResponseDocument> findByCampaignIdAndDate(String campaignId, Instant startDate, Instant endDate, Pageable pageable);
     @Query(value = "{ 'interrogationId': ?0}")
     List<RawResponseDocument> findByInterrogationId(String interrogationId);
 }

src/test/java/cucumber/functional_tests/RawDataDefinitions.java

@@ -39,6 +39,8 @@
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.boot.test.web.client.TestRestTemplate;
 import org.springframework.boot.test.web.server.LocalServerPort;
+import org.springframework.data.domain.Page;
+import org.springframework.data.domain.Pageable;
 import org.springframework.http.HttpEntity;
 import org.springframework.http.HttpHeaders;
 import org.springframework.http.HttpMethod;
@@ -54,6 +56,7 @@
 import java.security.KeyPairGenerator;
 import java.security.NoSuchAlgorithmException;
 import java.security.interfaces.RSAPublicKey;
+import java.time.Instant;
 import java.util.Collections;
 import java.util.List;
 
@@ -139,6 +142,11 @@ public List<String> getUnprocessedCollectionInstrumentIds() {
         public void updateProcessDates(List<SurveyUnitModel> surveyUnitModels) {
             // Do nothing - stub for test
         }
+
+        @Override
+        public Page<RawResponseModel> findRawResponseDataByCampaignIdAndDate(String campaignId, Instant startDate, Instant endDate, Pageable pageable) {
+            return null;
+        }
     };
 
     RawResponseController rawResponseController = new RawResponseController(

src/test/java/fr/insee/genesis/controller/rest/responses/RawResponseControllerTest.java

@@ -22,19 +22,23 @@
 import fr.insee.genesis.exceptions.GenesisError;
 import fr.insee.genesis.exceptions.GenesisException;
 import fr.insee.genesis.infrastructure.document.rawdata.LunaticJsonRawDataDocument;
+import fr.insee.genesis.infrastructure.document.rawdata.RawResponseDocument;
 import fr.insee.genesis.infrastructure.mappers.DataProcessingContextMapper;
 import fr.insee.genesis.infrastructure.repository.RawResponseInputRepository;
 import fr.insee.genesis.infrastructure.utils.FileUtils;
 import fr.insee.genesis.stubs.ConfigStub;
 import fr.insee.genesis.stubs.DataProcessingContextPersistancePortStub;
 import fr.insee.genesis.stubs.LunaticJsonRawDataPersistanceStub;
 import fr.insee.genesis.stubs.QuestionnaireMetadataPersistencePortStub;
+import fr.insee.genesis.stubs.RawResponseDataPersistanceStub;
 import fr.insee.genesis.stubs.SurveyUnitPersistencePortStub;
 import fr.insee.genesis.stubs.SurveyUnitQualityToolPerretAdapterStub;
 import fr.insee.modelefiliere.RawResponseDto;
 import lombok.extern.slf4j.Slf4j;
 import org.assertj.core.api.Assertions;
 import org.junit.jupiter.api.Test;
+import org.springframework.data.domain.Page;
+import org.springframework.data.domain.Pageable;
 import org.springframework.data.web.PagedModel;
 import org.springframework.http.ResponseEntity;
 
@@ -49,6 +53,7 @@
 class RawResponseControllerTest {
     private final FileUtils fileUtils = new FileUtils(new ConfigStub());
     private final LunaticJsonRawDataPersistanceStub lunaticJsonRawDataPersistanceStub = new LunaticJsonRawDataPersistanceStub();
+    private final RawResponseDataPersistanceStub rawResponseDataPersistanceStub = new RawResponseDataPersistanceStub();
     private final SurveyUnitPersistencePortStub surveyUnitPersistencePortStub = new SurveyUnitPersistencePortStub();
     private final SurveyUnitQualityToolPerretAdapterStub surveyUnitQualityToolPerretAdapterStub = new SurveyUnitQualityToolPerretAdapterStub();
     private final DataProcessingContextPersistancePortStub dataProcessingContextPersistancePortStub =
@@ -112,6 +117,10 @@ public List<String> getUnprocessedCollectionInstrumentIds() {
         public void updateProcessDates(List<SurveyUnitModel> surveyUnitModels) {
 
         }
+        @Override
+        public Page<RawResponseModel> findRawResponseDataByCampaignIdAndDate(String campaignId, Instant startDate, Instant endDate, Pageable pageable) {
+            return rawResponseDataPersistanceStub.findByCampaignIdAndDate(campaignId, startDate, endDate, pageable);
+        }
     };
 
     private final RawResponseController rawResponseController = new RawResponseController(lunaticJsonRawDataApiPort,  rawResponseApiPortStub, rawResponseInputRepositoryStub);
@@ -314,11 +323,10 @@ void processJsonRawDataV2Test(){
                 .contains(interrogationId);
     }
 
-
     @Test
-    void getRawResponsesFromJsonBody() {
+    void getLunaticJsonRawDataModelFromJsonBody() {
         //GIVEN
-        String campaignId = "getRawResponsesFromJsonBody";
+        String campaignId = "getLunaticJsonRawDataModelFromJsonBody";
         String questionnaireId = campaignId + "_quest";
         String interrogationId = "getRawResponsesFromJsonBody_id1";
         String varName = "VARName1";
@@ -335,7 +343,29 @@ void getRawResponsesFromJsonBody() {
         int page=0, size= 10;
 
         //WHEN
-        ResponseEntity<PagedModel<LunaticJsonRawDataModel>> response = rawResponseController.getRawResponsesFromJsonBody(campaignId, starDate, endDate, page, size);
+        ResponseEntity<PagedModel<LunaticJsonRawDataModel>> response = rawResponseController.getLunaticJsonRawDataModelFromJsonBody(campaignId, starDate, endDate, page, size);
+
+        //THEN
+        Assertions.assertThat(response.getStatusCode().is2xxSuccessful()).isTrue();
+        Assertions.assertThat(response.getBody()).isNotNull();
+        Assertions.assertThat(response.getBody().getContent()).hasSize(1);
+    }
+
+    @Test
+    void getRawResponsesFromJsonBody() {
+        //GIVEN
+        String campaignId = "getRawResponsesFromJsonBody";
+        String questionnaireId = campaignId + "_quest";
+        String interrogationId = "getRawResponsesFromJsonBody_id1";
+        Instant recordDate = Instant.parse("2025-01-01T01:00:00.000Z");
+
+        addJsonRawResponseDataDocumentToStub(campaignId, questionnaireId, interrogationId);
+
+        Instant starDate= recordDate.minusSeconds(86400),endDate = recordDate.plusSeconds(86400);
+        int page=0, size= 10;
+
+        //WHEN
+        ResponseEntity<PagedModel<RawResponseModel>> response = rawResponseController.getRawResponsesFromJsonBody(campaignId, starDate, endDate, page, size);
 
         //THEN
         Assertions.assertThat(response.getStatusCode().is2xxSuccessful()).isTrue();
@@ -385,4 +415,15 @@ private void addJsonRawDataDocumentToStub(String campaignId,
 
         lunaticJsonRawDataPersistanceStub.getMongoStub().add(lunaticJsonDataDocument);
     }
+
+    private void addJsonRawResponseDataDocumentToStub(String campaignId, String questionnaireId, String interrogationId) {
+        RawResponseDocument rawResponseDocument = RawResponseDocument.builder()
+                .campaignId(campaignId)
+                .collectionInstrumentId(questionnaireId)
+                .interrogationId(interrogationId)
+                .recordDate(LocalDateTime.now())
+                .build();
+
+        rawResponseDataPersistanceStub.getMongoStub().add(rawResponseDocument);
+    }
 }

src/test/java/fr/insee/genesis/stubs/RawResponseDataPersistanceStub.java

@@ -0,0 +1,62 @@
+package fr.insee.genesis.stubs;
+
+import fr.insee.genesis.domain.model.surveyunit.Mode;
+import fr.insee.genesis.domain.model.surveyunit.rawdata.RawResponseModel;
+import fr.insee.genesis.domain.ports.spi.RawResponsePersistencePort;
+import fr.insee.genesis.infrastructure.document.rawdata.RawResponseDocument;
+import fr.insee.genesis.infrastructure.mappers.RawResponseDocumentMapper;
+import lombok.Getter;
+import org.springframework.data.domain.Page;
+import org.springframework.data.domain.PageImpl;
+import org.springframework.data.domain.Pageable;
+
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+
+@Getter
+public class RawResponseDataPersistanceStub implements RawResponsePersistencePort {
+    List<RawResponseDocument> mongoStub = new ArrayList<>();
+
+    @Override
+    public List<RawResponseModel> findRawResponses(String collectionInstrumentId, Mode mode, List<String> interrogationIdList) {
+        return RawResponseDocumentMapper.INSTANCE.listDocumentToListModel(mongoStub.stream().filter(
+                doc -> doc.collectionInstrumentId().equals(collectionInstrumentId)
+                        && Mode.valueOf(doc.mode()).equals(mode)
+                        && interrogationIdList.contains(doc.interrogationId())
+        ).toList());
+    }
+
+    @Override
+    public List<RawResponseModel> findRawResponsesByInterrogationID(String interrogationId) {
+        return RawResponseDocumentMapper.INSTANCE.listDocumentToListModel(mongoStub.stream().filter(
+                doc -> doc.interrogationId().equals(interrogationId)
+        ).toList());
+    }
+
+    @Override
+    public void updateProcessDates(String collectionInstrumentId, Set<String> interrogationIds) {
+
+    }
+
+    @Override
+    public List<String> getUnprocessedCollectionIds() {
+        return List.of();
+    }
+
+    @Override
+    public Set<String> findUnprocessedInterrogationIdsByCollectionInstrumentId(String collectionInstrumentId) {
+        return Set.of();
+    }
+
+    @Override
+    public Page<RawResponseModel> findByCampaignIdAndDate(String campaignId, Instant startDate, Instant endDate, Pageable pageable) {
+        List<RawResponseDocument> foundRaws = mongoStub.stream()
+                .filter(rawData -> rawData.campaignId().equals(campaignId))
+                .toList();
+        return new PageImpl<>(RawResponseDocumentMapper.INSTANCE.listDocumentToListModel(foundRaws),
+                pageable,
+                foundRaws.size());
+    }
+}