Skip to content

Dev/combined rawdata #367

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Hajarel-moukh wants to merge 4 commits into
base: devFixUsualSurveyUnitId
Choose a base branch
from
Open

Dev/combined rawdata #367

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
f89d126
get RawResponseData + LunaticJsonRawData by InterrogationId
Hajarel-moukh Dec 22, 2025
8b9e84c
Merge remote-tracking branch 'origin/devFixUsualSurveyUnitId' into de…
Hajarel-moukh Dec 23, 2025
b850c95
add some modifications
Hajarel-moukh Dec 23, 2025
d1281a0
add modifications after CR
Hajarel-moukh Dec 23, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions src/main/java/fr/insee/genesis/controller/dto/rawdata/CombinedRawDataDto.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
package fr.insee.genesis.controller.dto.rawdata;

import fr.insee.genesis.domain.model.surveyunit.rawdata.LunaticJsonRawDataModel;
import fr.insee.genesis.domain.model.surveyunit.rawdata.RawResponseModel;

import java.util.List;

public record CombinedRawDataDto(
List<RawResponseModel> rawResponseModels,
List<LunaticJsonRawDataModel> lunaticRawDataModels
) {
}

41 changes: 41 additions & 0 deletions src/main/java/fr/insee/genesis/controller/rest/CombinedRawDataController.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
package fr.insee.genesis.controller.rest;

import fr.insee.genesis.controller.dto.rawdata.CombinedRawDataDto;
import fr.insee.genesis.domain.service.rawdata.CombinedRawDataService;
import io.swagger.v3.oas.annotations.Operation;
import lombok.extern.slf4j.Slf4j;
import org.springframework.http.ResponseEntity;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;

@Slf4j
@Controller
@RequestMapping(path = "/combined-raw-data" )
public class CombinedRawDataController {

private static final String INTERROGATION_ID = "interrogationId";
private final CombinedRawDataService combinedRawDataService;

public CombinedRawDataController(CombinedRawDataService combinedRawDataService) {
this.combinedRawDataService = combinedRawDataService;
}

@Operation(summary = "Retrieve combined raw responses and Lunatic raw data for a given interrogationId")
@GetMapping
@PreAuthorize("hasAnyRole('ADMIN', 'USER_PLATINE')")
public ResponseEntity<CombinedRawDataDto> getCombinetRawData(
@RequestParam(INTERROGATION_ID) String interrogationId
){
CombinedRawDataDto data = combinedRawDataService.getCombinedRawDataByInterrogationId(interrogationId);

if (data.rawResponseModels().isEmpty()) {
return ResponseEntity.notFound().build();
}

return ResponseEntity.ok(data);
}

}
2 changes: 1 addition & 1 deletion ...model/surveyunit/rawdata/RawResponse.java → .../surveyunit/rawdata/RawResponseModel.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
import java.time.LocalDateTime;
import java.util.Map;

public record RawResponse(
public record RawResponseModel(
ObjectId id,
String interrogationId,
String collectionInstrumentId,
Expand Down
7 changes: 7 additions & 0 deletions src/main/java/fr/insee/genesis/domain/model/surveyunit/rawdata/combinedRawDataModel.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
package fr.insee.genesis.domain.model.surveyunit.rawdata;

import java.util.List;

public record combinedRawDataModel(List<RawResponseModel> rawResponseModels,
List<LunaticJsonRawDataModel> lunaticRawDataModels) {
}
2 changes: 2 additions & 0 deletions src/main/java/fr/insee/genesis/domain/ports/api/LunaticJsonRawDataApiPort.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,8 @@ public interface LunaticJsonRawDataApiPort {
long countResponsesByQuestionnaireId(String campaignId);
Page<LunaticJsonRawDataModel> findRawDataByCampaignIdAndDate(String campaignId, Instant startDt, Instant endDt, Pageable pageable);

List<LunaticJsonRawDataModel> getRawDataByInterrogationId(String interrogationId);

@Deprecated(since = "1.13.0")
DataProcessResult processRawData(String campaignName, List<String> interrogationIdList, List<GenesisError> errors) throws GenesisException;

Expand Down
7 changes: 4 additions & 3 deletions src/main/java/fr/insee/genesis/domain/ports/api/RawResponseApiPort.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,18 +4,19 @@
import fr.insee.genesis.domain.model.surveyunit.Mode;
import fr.insee.genesis.domain.model.surveyunit.SurveyUnitModel;
import fr.insee.genesis.domain.model.surveyunit.rawdata.DataProcessResult;
import fr.insee.genesis.domain.model.surveyunit.rawdata.RawResponse;
import fr.insee.genesis.domain.model.surveyunit.rawdata.RawResponseModel;
import fr.insee.genesis.exceptions.GenesisError;
import fr.insee.genesis.exceptions.GenesisException;

import java.util.List;

public interface RawResponseApiPort {

List<RawResponse> getRawResponses(String collectionInstrumentId, Mode mode, List<String> interrogationIdList);
List<RawResponseModel> getRawResponses(String collectionInstrumentId, Mode mode, List<String> interrogationIdList);
List<RawResponseModel> getRawResponsesByInterrogationID(String interrogationId);
DataProcessResult processRawResponses(String collectionInstrumentId, List<String> interrogationIdList, List<GenesisError> errors) throws GenesisException;
DataProcessResult processRawResponses(String collectionInstrumentId) throws GenesisException;
List<SurveyUnitModel> convertRawResponse(List<RawResponse> rawResponses, VariablesMap variablesMap);
List<SurveyUnitModel> convertRawResponse(List<RawResponseModel> rawResponses, VariablesMap variablesMap);
List<String> getUnprocessedCollectionInstrumentIds();
void updateProcessDates(List<SurveyUnitModel> surveyUnitModels);

Expand Down
1 change: 1 addition & 0 deletions src/main/java/fr/insee/genesis/domain/ports/spi/LunaticJsonRawDataPersistencePort.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ public interface LunaticJsonRawDataPersistencePort {

void save(LunaticJsonRawDataModel rawData);
List<LunaticJsonRawDataModel> findRawData(String campaignName, Mode mode, List<String> interrogationIdList);
List<LunaticJsonRawDataModel> findRawDataByInterrogationID(String interrogationId);
List<LunaticJsonRawDataModel> getAllUnprocessedData();
void updateProcessDates(String campaignId, Set<String> interrogationIds);
Set<String> findDistinctQuestionnaireIds();
Expand Down
5 changes: 3 additions & 2 deletions src/main/java/fr/insee/genesis/domain/ports/spi/RawResponsePersistencePort.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,14 +1,15 @@
package fr.insee.genesis.domain.ports.spi;

import fr.insee.genesis.domain.model.surveyunit.Mode;
import fr.insee.genesis.domain.model.surveyunit.rawdata.RawResponse;
import fr.insee.genesis.domain.model.surveyunit.rawdata.RawResponseModel;

import java.util.List;
import java.util.Set;

public interface RawResponsePersistencePort {

List<RawResponse> findRawResponses(String collectionInstrumentId, Mode mode, List<String> interrogationIdList);
List<RawResponseModel> findRawResponses(String collectionInstrumentId, Mode mode, List<String> interrogationIdList);
List<RawResponseModel> findRawResponsesByInterrogationID(String interrogationId);
void updateProcessDates(String collectionInstrumentId, Set<String> interrogationIds);
List<String> getUnprocessedCollectionIds();
Set<String> findUnprocessedInterrogationIdsByCollectionInstrumentId(String collectionInstrumentId);
Expand Down
2 changes: 2 additions & 0 deletions src/main/java/fr/insee/genesis/domain/service/metadata/QuestionnaireMetadataService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,8 @@ public MetadataModel loadAndSaveIfNotExists(String campaignName, String collecti
saveMetadata(collectionInstrumentId.toUpperCase(), mode, metadataModel);
return metadataModel;
}


return questionnaireMetadataModels.getFirst().metadataModel();
}

Expand Down
43 changes: 43 additions & 0 deletions src/main/java/fr/insee/genesis/domain/service/rawdata/CombinedRawDataService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
package fr.insee.genesis.domain.service.rawdata;

import fr.insee.genesis.controller.dto.rawdata.CombinedRawDataDto;
import fr.insee.genesis.domain.model.surveyunit.rawdata.LunaticJsonRawDataModel;
import fr.insee.genesis.domain.model.surveyunit.rawdata.RawResponseModel;
import fr.insee.genesis.domain.ports.spi.LunaticJsonRawDataPersistencePort;
import fr.insee.genesis.domain.ports.spi.RawResponsePersistencePort;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.stereotype.Service;

import java.util.List;

@Service
@Slf4j
public class CombinedRawDataService {

@Qualifier("lunaticJsonMongoAdapter")
private final LunaticJsonRawDataPersistencePort lunaticJsonRawDataPersistencePort;
@Qualifier("rawResponseMongoAdapter")
private final RawResponsePersistencePort rawResponsePersistencePort;

public CombinedRawDataService(LunaticJsonRawDataPersistencePort lunaticJsonRawDataPersistencePort, RawResponsePersistencePort rawResponsePersistencePort) {
this.lunaticJsonRawDataPersistencePort = lunaticJsonRawDataPersistencePort;
this.rawResponsePersistencePort = rawResponsePersistencePort;
}

public CombinedRawDataDto getCombinedRawDataByInterrogationId(String interrogationId) {

List<RawResponseModel> rawResponseModels =
rawResponsePersistencePort.findRawResponsesByInterrogationID(interrogationId);

List<LunaticJsonRawDataModel> lunaticRawDataModels =
lunaticJsonRawDataPersistencePort.findRawDataByInterrogationID(interrogationId);

return new CombinedRawDataDto(
rawResponseModels,
lunaticRawDataModels
);
}


}
5 changes: 5 additions & 0 deletions src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,11 @@ public List<LunaticJsonRawDataModel> getRawData(String campaignName, Mode mode,
return lunaticJsonRawDataPersistencePort.findRawData(campaignName, mode, interrogationIdList);
}

@Override
public List<LunaticJsonRawDataModel> getRawDataByInterrogationId(String interrogationId) {
return lunaticJsonRawDataPersistencePort.findRawDataByInterrogationID(interrogationId);
}

@Override
@Deprecated(since = "1.13.0")
public DataProcessResult processRawData(String campaignName, List<String> interrogationIdList, List<GenesisError> errors) throws GenesisException {
Expand Down
69 changes: 37 additions & 32 deletions src/main/java/fr/insee/genesis/domain/service/rawdata/RawResponseService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
import fr.insee.genesis.domain.model.surveyunit.SurveyUnitModel;
import fr.insee.genesis.domain.model.surveyunit.VariableModel;
import fr.insee.genesis.domain.model.surveyunit.rawdata.DataProcessResult;
import fr.insee.genesis.domain.model.surveyunit.rawdata.RawResponse;
import fr.insee.genesis.domain.model.surveyunit.rawdata.RawResponseModel;
import fr.insee.genesis.domain.ports.api.RawResponseApiPort;
import fr.insee.genesis.domain.ports.spi.RawResponsePersistencePort;
import fr.insee.genesis.domain.ports.spi.SurveyUnitQualityToolPort;
Expand Down Expand Up @@ -67,10 +67,15 @@ public RawResponseService(ControllerUtils controllerUtils, QuestionnaireMetadata
}

@Override
public List<RawResponse> getRawResponses(String collectionInstrumentId, Mode mode, List<String> interrogationIdList) {
public List<RawResponseModel> getRawResponses(String collectionInstrumentId, Mode mode, List<String> interrogationIdList) {
return rawResponsePersistencePort.findRawResponses(collectionInstrumentId,mode,interrogationIdList);
}

@Override
public List<RawResponseModel> getRawResponsesByInterrogationID(String interrogationId) {
return rawResponsePersistencePort.findRawResponsesByInterrogationID(interrogationId);
}

@Override
public DataProcessResult processRawResponses(String collectionInstrumentId, List<String> interrogationIdList, List<GenesisError> errors) throws GenesisException {
int dataCount=0;
Expand All @@ -89,10 +94,10 @@ public DataProcessResult processRawResponses(String collectionInstrumentId, List
int maxIndex = Math.min(interrogationIdListForMode.size(), config.getRawDataProcessingBatchSize());
List<String> interrogationIdToProcess = interrogationIdListForMode.subList(0, maxIndex);

List<RawResponse> rawResponses = getRawResponses(collectionInstrumentId, mode, interrogationIdToProcess);
List<RawResponseModel> rawResponseModels = getRawResponses(collectionInstrumentId, mode, interrogationIdToProcess);

List<SurveyUnitModel> surveyUnitModels = convertRawResponse(
rawResponses,
rawResponseModels,
variablesMap
);

Expand Down Expand Up @@ -182,9 +187,9 @@ public DataProcessResult processRawResponses(String collectionInstrumentId) thro

private List<SurveyUnitModel> getConvertedSurveyUnits(String collectionInstrumentId, Mode mode, List<String> interrogationIdListForMode, int maxIndex, VariablesMap variablesMap) {
List<String> interrogationIdToProcess = interrogationIdListForMode.subList(0, maxIndex);
List<RawResponse> rawResponses = getRawResponses(collectionInstrumentId, mode, interrogationIdToProcess);
List<RawResponseModel> rawResponseModels = getRawResponses(collectionInstrumentId, mode, interrogationIdToProcess);
return convertRawResponse(
rawResponses,
rawResponseModels,
variablesMap
);
}
Expand All @@ -202,47 +207,47 @@ private VariablesMap getVariablesMap(String collectionInstrumentId, Mode mode, L
}

@Override
public List<SurveyUnitModel> convertRawResponse(List<RawResponse> rawResponses, VariablesMap variablesMap) {
public List<SurveyUnitModel> convertRawResponse(List<RawResponseModel> rawResponseModels, VariablesMap variablesMap) {
//Convert to genesis model
List<SurveyUnitModel> surveyUnitModels = new ArrayList<>();
//For each possible data state (we receive COLLECTED or EDITED)
for(DataState dataState : List.of(DataState.COLLECTED,DataState.EDITED)){
for (RawResponse rawResponse : rawResponses) {
for (RawResponseModel rawResponseModel : rawResponseModels) {
//Get optional fields
Boolean isCapturedIndirectly = getIsCapturedIndirectly(rawResponse);
LocalDateTime validationDate = getValidationDate(rawResponse);
String usualSurveyUnitId = getStringFieldInPayload(rawResponse,"usualSurveyUnitId");
String majorModelVersion = getStringFieldInPayload(rawResponse, "majorModelVersion");
Boolean isCapturedIndirectly = getIsCapturedIndirectly(rawResponseModel);
LocalDateTime validationDate = getValidationDate(rawResponseModel);
String usualSurveyUnitId = getStringFieldInPayload(rawResponseModel,"usualSurveyUnitId");
String majorModelVersion = getStringFieldInPayload(rawResponseModel, "majorModelVersion");

SurveyUnitModel surveyUnitModel = SurveyUnitModel.builder()
.collectionInstrumentId(rawResponse.collectionInstrumentId())
.collectionInstrumentId(rawResponseModel.collectionInstrumentId())
.majorModelVersion(majorModelVersion)
.mode(rawResponse.mode())
.interrogationId(rawResponse.interrogationId())
.mode(rawResponseModel.mode())
.interrogationId(rawResponseModel.interrogationId())
.usualSurveyUnitId(usualSurveyUnitId)
.validationDate(validationDate)
.isCapturedIndirectly(isCapturedIndirectly)
.state(dataState)
.fileDate(rawResponse.recordDate())
.fileDate(rawResponseModel.recordDate())
.recordDate(LocalDateTime.now())
.collectedVariables(new ArrayList<>())
.externalVariables(new ArrayList<>())
.build();

//Data collected variables conversion
convertRawDataCollectedVariables(rawResponse, surveyUnitModel, dataState, variablesMap);
convertRawDataCollectedVariables(rawResponseModel, surveyUnitModel, dataState, variablesMap);

//External variables conversion into COLLECTED document
if(dataState == DataState.COLLECTED){
convertRawDataExternalVariables(rawResponse, surveyUnitModel, variablesMap);
convertRawDataExternalVariables(rawResponseModel, surveyUnitModel, variablesMap);
}

boolean hasNoVariable = surveyUnitModel.getCollectedVariables().isEmpty()
&& surveyUnitModel.getExternalVariables().isEmpty();

if(hasNoVariable){
if(surveyUnitModel.getState() == DataState.COLLECTED){
log.warn("No collected or external variable for interrogation {}, raw data is ignored.", rawResponse.interrogationId());
log.warn("No collected or external variable for interrogation {}, raw data is ignored.", rawResponseModel.interrogationId());
}
continue;// don't add suModel
}
Expand Down Expand Up @@ -300,29 +305,29 @@ private void sendProcessedIdsToQualityTool(List<SurveyUnitModel> surveyUnitModel
}
}

private static Boolean getIsCapturedIndirectly(RawResponse rawResponse) {
private static Boolean getIsCapturedIndirectly(RawResponseModel rawResponseModel) {
try{
return rawResponse.payload().get("isCapturedIndirectly") == null ? null :
Boolean.parseBoolean(rawResponse.payload().get("isCapturedIndirectly").toString());
return rawResponseModel.payload().get("isCapturedIndirectly") == null ? null :
Boolean.parseBoolean(rawResponseModel.payload().get("isCapturedIndirectly").toString());
}catch(Exception e){
log.warn("Exception when parsing isCapturedIndirectly : {}",e.toString());
return Boolean.FALSE;
}
}

private static LocalDateTime getValidationDate(RawResponse rawResponse) {
private static LocalDateTime getValidationDate(RawResponseModel rawResponseModel) {
try{
return rawResponse.payload().get("validationDate") == null ? null :
LocalDateTime.parse(rawResponse.payload().get("validationDate").toString());
return rawResponseModel.payload().get("validationDate") == null ? null :
LocalDateTime.parse(rawResponseModel.payload().get("validationDate").toString());
}catch(Exception e){
log.warn("Exception when parsing validation date : {}",e.toString());
return null;
}
}

private static String getStringFieldInPayload(RawResponse rawResponse, String field) {
private static String getStringFieldInPayload(RawResponseModel rawResponseModel, String field) {
try{
return rawResponse.payload().get(field).toString();
return rawResponseModel.payload().get(field).toString();
}catch(Exception e){
log.warn("Exception when parsing {} : {}",field, e.toString());
return null;
Expand All @@ -331,12 +336,12 @@ private static String getStringFieldInPayload(RawResponse rawResponse, String fi

@SuppressWarnings("unchecked")
private void convertRawDataCollectedVariables(
RawResponse rawResponse,
RawResponseModel rawResponseModel,
SurveyUnitModel dstSurveyUnitModel,
DataState dataState,
VariablesMap variablesMap
) {
Map<String, Object> dataMap = rawResponse.payload();
Map<String, Object> dataMap = rawResponseModel.payload();
dataMap = (Map<String, Object>) dataMap.get("data");

dataMap = (Map<String, Object>)dataMap.get("COLLECTED");
Expand All @@ -345,7 +350,7 @@ private void convertRawDataCollectedVariables(
Map<String,Object> collectedMap = JsonUtils.asMap(dataMap);
if (collectedMap == null || collectedMap.isEmpty()){
if(dataState.equals(DataState.COLLECTED)) {
log.warn("No collected data for interrogation {}", rawResponse.interrogationId());
log.warn("No collected data for interrogation {}", rawResponseModel.interrogationId());
}
return;
}
Expand Down Expand Up @@ -415,11 +420,11 @@ private static String getIdLoop(VariablesMap variablesMap, String variableName)

@SuppressWarnings("unchecked")
private static void convertRawDataExternalVariables(
RawResponse rawResponse,
RawResponseModel rawResponseModel,
SurveyUnitModel dstSurveyUnitModel,
VariablesMap variablesMap
) {
Map<String, Object> dataMap = rawResponse.payload();
Map<String, Object> dataMap = rawResponseModel.payload();
dataMap = (Map<String, Object>) dataMap.get("data");


Expand Down
Loading
Loading