Skip to content

Fix Process raw data by questionId #351

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Hajarel-moukh wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Fix Process raw data by questionId #351

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 10 additions & 2 deletions src/main/java/fr/insee/genesis/controller/rest/responses/ModeController.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,14 +3,17 @@
import fr.insee.genesis.controller.rest.CommonApiResponse;
import fr.insee.genesis.domain.model.surveyunit.Mode;
import fr.insee.genesis.domain.ports.api.SurveyUnitApiPort;
import fr.insee.genesis.exceptions.QuestionnaireNotFoundException;
import io.swagger.v3.oas.annotations.Operation;
import lombok.extern.slf4j.Slf4j;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;

import java.util.Collections;
import java.util.List;

@RequestMapping(path = "/modes" )
Expand All @@ -29,8 +32,13 @@ public ModeController(SurveyUnitApiPort surveyUnitService) {
@Operation(summary = "List sources/modes used for a given questionnaire")
@GetMapping(path = "/by-questionnaire")
public ResponseEntity<List<Mode>> getModesByQuestionnaire(@RequestParam("questionnaireId") String questionnaireId) {
List<Mode> modes = surveyUnitService.findModesByQuestionnaireId(questionnaireId);
return ResponseEntity.ok(modes);
try {
List<Mode> modes = surveyUnitService.findModesByQuestionnaireId(questionnaireId);
return ResponseEntity.ok(modes);
} catch (QuestionnaireNotFoundException e) {
return ResponseEntity.status(HttpStatus.NOT_FOUND)
.body(Collections.emptyList());
}
}

@Operation(summary = "List sources/modes used for a given campaign")
Expand Down
1 change: 1 addition & 0 deletions src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -252,6 +252,7 @@ public ResponseEntity<SurveyUnitSimplified> getLatestByInterrogationOneObject(@R
.campaignId(responses.getFirst().getCampaignId())
.interrogationId(responses.getFirst().getInterrogationId())
.surveyUnitId(responses.getFirst().getIdUE())
.mode(mode)
.variablesUpdate(outputVariables)
.externalVariables(outputExternalVariables)
.build());
Expand Down
1 change: 1 addition & 0 deletions src/main/java/fr/insee/genesis/domain/ports/api/LunaticJsonRawDataApiPort.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ public interface LunaticJsonRawDataApiPort {

void save(LunaticJsonRawDataModel rawData);
List<LunaticJsonRawDataModel> getRawData(String campaignName, Mode mode, List<String> interrogationIdList);
List<LunaticJsonRawDataModel> getRawDataByQuestionnaireId(String questionnaireId, Mode mode, List<String> interrogationIdList);
List<SurveyUnitModel> convertRawData(List<LunaticJsonRawDataModel> rawData, VariablesMap variablesMap);
List<LunaticJsonRawDataUnprocessedDto> getUnprocessedDataIds();
void updateProcessDates(List<SurveyUnitModel> surveyUnitModels);
Expand Down
1 change: 1 addition & 0 deletions src/main/java/fr/insee/genesis/domain/ports/spi/LunaticJsonRawDataPersistencePort.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ public interface LunaticJsonRawDataPersistencePort {

void save(LunaticJsonRawDataModel rawData);
List<LunaticJsonRawDataModel> findRawData(String campaignName, Mode mode, List<String> interrogationIdList);
List<LunaticJsonRawDataModel> findRawDataByQuestionnaireId(String questionnaireId, Mode mode, List<String> interrogationIdList);
List<LunaticJsonRawDataModel> getAllUnprocessedData();
void updateProcessDates(String campaignId, Set<String> interrogationIds);
Set<String> findDistinctQuestionnaireIds();
Expand Down
7 changes: 6 additions & 1 deletion src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,11 @@ public List<LunaticJsonRawDataModel> getRawData(String campaignName, Mode mode,
return lunaticJsonRawDataPersistencePort.findRawData(campaignName, mode, interrogationIdList);
}

@Override
public List<LunaticJsonRawDataModel> getRawDataByQuestionnaireId(String questionnaireId, Mode mode, List<String> interrogationIdList) {
return lunaticJsonRawDataPersistencePort.findRawDataByQuestionnaireId(questionnaireId, mode, interrogationIdList);
}

@Override
@Deprecated(since = "1.13.0")
public DataProcessResult processRawData(String campaignName, List<String> interrogationIdList, List<GenesisError> errors) throws GenesisException {
Expand Down Expand Up @@ -224,7 +229,7 @@ private VariablesMap getVariablesMap(String questionnaireId, Mode mode, List<Gen

private List<SurveyUnitModel> getConvertedSurveyUnits(String questionnaireId, Mode mode, List<String> interrogationIdListForMode, int maxIndex, VariablesMap variablesMap) {
List<String> interrogationIdToProcess = interrogationIdListForMode.subList(0, maxIndex);
List<LunaticJsonRawDataModel> rawData = getRawData(questionnaireId, mode, interrogationIdToProcess);
List<LunaticJsonRawDataModel> rawData = getRawDataByQuestionnaireId(questionnaireId, mode, interrogationIdToProcess);
return convertRawData(
rawData,
variablesMap
Expand Down
10 changes: 7 additions & 3 deletions src/main/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import fr.insee.genesis.domain.service.metadata.QuestionnaireMetadataService;
import fr.insee.genesis.domain.utils.GroupUtils;
import fr.insee.genesis.exceptions.GenesisException;
import fr.insee.genesis.exceptions.QuestionnaireNotFoundException;
import fr.insee.genesis.infrastructure.utils.FileUtils;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
Expand Down Expand Up @@ -336,9 +337,12 @@ public List<SurveyUnitModel> findInterrogationIdsAndModesByQuestionnaireId(Strin
@Override
public List<Mode> findModesByQuestionnaireId(String questionnaireId) {
List<SurveyUnitModel> surveyUnitModels = surveyUnitPersistencePort.findInterrogationIdsByQuestionnaireId(questionnaireId);
List<Mode> sources = new ArrayList<>();
surveyUnitModels.forEach(surveyUnitModel -> sources.add(surveyUnitModel.getMode()));
return sources.stream().distinct().toList();
if (surveyUnitModels == null || surveyUnitModels.isEmpty()) {
log.warn("No questionnaire found with id: {}", questionnaireId);
throw new QuestionnaireNotFoundException(questionnaireId);
}
List<Mode> sources = surveyUnitModels.stream().map(SurveyUnitModel::getMode).distinct().toList();
return sources;
}

@Override
Expand Down
7 changes: 7 additions & 0 deletions src/main/java/fr/insee/genesis/exceptions/QuestionnaireNotFoundException.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
package fr.insee.genesis.exceptions;

public class QuestionnaireNotFoundException extends RuntimeException {
public QuestionnaireNotFoundException(String questionnaireId) {
super("No questionnaire found with id: " + questionnaireId);
}
}
9 changes: 8 additions & 1 deletion src/main/java/fr/insee/genesis/infrastructure/adapter/LunaticJsonRawDataMongoAdapter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,14 @@ public List<LunaticJsonRawDataModel> getAllUnprocessedData() {

@Override
public List<LunaticJsonRawDataModel> findRawData(String campaignName, Mode mode, List<String> interrogationIdList) {
List<LunaticJsonRawDataDocument> rawDataDocs = repository.findModesByCampaignIdAndByModeAndinterrogationIdIninterrogationIdList(campaignName, mode, interrogationIdList);return LunaticJsonRawDataDocumentMapper.INSTANCE.listDocumentToListModel(rawDataDocs);
List<LunaticJsonRawDataDocument> rawDataDocs = repository.findByCampaignModeAndInterrogations(campaignName, mode, interrogationIdList);
return LunaticJsonRawDataDocumentMapper.INSTANCE.listDocumentToListModel(rawDataDocs);
}

@Override
public List<LunaticJsonRawDataModel> findRawDataByQuestionnaireId(String questionnaireId, Mode mode, List<String> interrogationIdList) {
List<LunaticJsonRawDataDocument> rawDataDocs = repository.findByQuestionnaireModeAndInterrogations(questionnaireId, mode, interrogationIdList);
return LunaticJsonRawDataDocumentMapper.INSTANCE.listDocumentToListModel(rawDataDocs);
}

@Override
Expand Down
5 changes: 4 additions & 1 deletion src/main/java/fr/insee/genesis/infrastructure/repository/LunaticJsonMongoDBRepository.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,10 @@ public interface LunaticJsonMongoDBRepository extends MongoRepository<LunaticJso
List<Mode> findModesByCampaignId(String campaignId);

@Query(value = "{ 'campaignId' : ?0, 'mode' : ?1, 'interrogationId': {$in: ?2} }")
List<LunaticJsonRawDataDocument> findModesByCampaignIdAndByModeAndinterrogationIdIninterrogationIdList(String campaignName, Mode mode, List<String> interrogationIdList);
List<LunaticJsonRawDataDocument> findByCampaignModeAndInterrogations(String campaignName, Mode mode, List<String> interrogationIdList);

@Query(value = "{ 'questionnaireId' : ?0, 'mode' : ?1, 'interrogationId': {$in: ?2} }")
List<LunaticJsonRawDataDocument> findByQuestionnaireModeAndInterrogations(String questionnaireId, Mode mode, List<String> interrogationIdList);

Page<LunaticJsonRawDataDocument> findByCampaignIdAndRecordDateBetween(String campagneId, Instant start, Instant end, Pageable pageable);
long countByQuestionnaireId(String questionnaireId);
Expand Down
11 changes: 10 additions & 1 deletion src/test/java/fr/insee/genesis/stubs/LunaticJsonMongoDBRepositoryStub.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ public List<Mode> findModesByCampaignId(String campaignId) {
}

@Override
public List<LunaticJsonRawDataDocument> findModesByCampaignIdAndByModeAndinterrogationIdIninterrogationIdList(
public List<LunaticJsonRawDataDocument> findByCampaignModeAndInterrogations(
String campaignName, Mode mode, List<String> interrogationIdList) {
return documents.stream()
.filter(doc -> Objects.equals(doc.campaignId(), campaignName)
Expand All @@ -56,6 +56,15 @@ public List<LunaticJsonRawDataDocument> findModesByCampaignIdAndByModeAndinterro
.toList();
}

@Override
public List<LunaticJsonRawDataDocument> findByQuestionnaireModeAndInterrogations(String questionnaireId, Mode mode, List<String> interrogationIdList) {
return documents.stream()
.filter(doc -> Objects.equals(doc.questionnaireId(), questionnaireId)
&& Objects.equals(doc.mode(), mode)
&& interrogationIdList.contains(doc.interrogationId()))
.toList();
}

@Override
public Page<LunaticJsonRawDataDocument> findByCampaignIdAndRecordDateBetween(String campagneId, Instant start, Instant end, Pageable pageable){
return Page.empty(pageable);
Expand Down
9 changes: 9 additions & 0 deletions src/test/java/fr/insee/genesis/stubs/LunaticJsonRawDataPersistanceStub.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,15 @@ public List<LunaticJsonRawDataModel> findRawData(String campaignName, Mode mode,
return LunaticJsonRawDataDocumentMapper.INSTANCE.listDocumentToListModel(docs);
}

@Override
public List<LunaticJsonRawDataModel> findRawDataByQuestionnaireId(String questionnaireId, Mode mode, List<String> interrogationIdList) {
List<LunaticJsonRawDataDocument> docs = mongoStub.stream().filter(lunaticJsonRawDataDocument ->
lunaticJsonRawDataDocument.questionnaireId().equals(questionnaireId)
&& lunaticJsonRawDataDocument.mode().equals(mode)
&& interrogationIdList.contains(lunaticJsonRawDataDocument.interrogationId())
).toList();
return LunaticJsonRawDataDocumentMapper.INSTANCE.listDocumentToListModel(docs); }


@Override
public void updateProcessDates(String campaignId, Set<String> interrogationIds) {
Expand Down
Loading