Skip to content

task/AMP-31088/Update-importer #4461

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jdeanquin-dg merged 18 commits into from
Mar 26, 2026
Merged

task/AMP-31088/Update-importer #4461

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
ec6583c
Add import for programs
brianbrix Mar 25, 2026
d3291ae
Add import for programs
brianbrix Mar 25, 2026
62b2e99
Add import for programs
brianbrix Mar 25, 2026
781cd72
Add import for programs
brianbrix Mar 25, 2026
3e3ac2f
Add import for programs
brianbrix Mar 25, 2026
2357d06
Add import for programs
brianbrix Mar 25, 2026
2f26878
Implementation level for locations
brianbrix Mar 25, 2026
f6d7c1e
Implementation level for locations
brianbrix Mar 25, 2026
8171952
Location percentages
brianbrix Mar 25, 2026
b067d40
Location percentages
brianbrix Mar 25, 2026
2d4203b
Failing program creation
brianbrix Mar 25, 2026
c0e5b4b
Failing program creation
brianbrix Mar 25, 2026
eca0e34
Failing program creation
brianbrix Mar 26, 2026
1930d55
Failing program creation
brianbrix Mar 26, 2026
bba1e74
Failing date input
brianbrix Mar 26, 2026
c3a9bd3
Failing program creation
brianbrix Mar 26, 2026
e6dbba4
Failing program creation
brianbrix Mar 26, 2026
e3aaaa8
Failing program creation
brianbrix Mar 26, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
73 changes: 69 additions & 4 deletions amp/src/main/java/org/digijava/module/aim/action/dataimporter/DataImporter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,10 +55,12 @@

import org.digijava.module.aim.action.dataimporter.util.ImporterConstants;
import org.digijava.module.aim.dbentity.AmpOrgGroup;
import org.digijava.module.aim.dbentity.AmpActivityProgramSettings;
import org.digijava.module.aim.form.DataImporterForm;
import org.digijava.module.aim.util.DbUtil;
import org.digijava.module.aim.util.DynLocationManagerUtil;
import org.digijava.module.aim.util.LocationUtil;
import org.digijava.module.aim.util.ProgramUtil;
import org.digijava.module.categorymanager.dbentity.AmpCategoryValue;
import org.digijava.module.aim.dbentity.AmpCategoryValueLocations;
import org.digijava.module.categorymanager.util.CategoryConstants;
Expand Down Expand Up @@ -90,6 +92,7 @@ public ActionForward execute(ActionMapping mapping, ActionForm form, HttpServlet
List<AmpOrgGroup> orgGroups = DbUtil.getAllOrgGroups();
request.setAttribute("orgGroups", orgGroups);
request.setAttribute("activityStatuses", getActivityStatuses());
request.setAttribute("programClassifications", getProgramClassificationNames());
List<AmpCategoryValueLocations> availableLocations = getAvailableLocations();
request.setAttribute("availableLocations", availableLocations);
AmpCategoryValueLocations defaultLocation = DynLocationManagerUtil.getDefaultCountry();
Expand Down Expand Up @@ -372,9 +375,12 @@ public ActionForward execute(ActionMapping mapping, ActionForm form, HttpServlet
boolean createMissingOrgs = dataImporterForm.isCreateMissingOrgs();
boolean createMissingSectors = dataImporterForm.isCreateMissingSectors();
boolean createMissingOrgGroups = dataImporterForm.isCreateMissingOrgGroups();
boolean createMissingPrograms = dataImporterForm.isCreateMissingPrograms();
boolean replaceExistingTransactions = dataImporterForm.isReplaceExistingTransactions();
Long orgGroupId = dataImporterForm.getOrgGroupId();
Long defaultActivityStatusId = dataImporterForm.getDefaultActivityStatusId();
Long defaultLocationId = dataImporterForm.getDefaultLocationId();
String defaultProgramClassification = dataImporterForm.getDefaultProgramClassification();
logger.info("Internal: "+ isInternal);
logger.info("Skip existing: "+ skipExisting);
logger.info("Validate activities: "+ validateActivities);
Expand All @@ -383,13 +389,41 @@ public ActionForward execute(ActionMapping mapping, ActionForm form, HttpServlet
logger.info("Create missing orgs: "+ createMissingOrgs);
logger.info("Create missing sectors: {}", createMissingSectors);
logger.info("Create missing org groups: " + createMissingOrgGroups);
logger.info("Create missing programs: " + createMissingPrograms);
logger.info("Replace existing transactions: " + replaceExistingTransactions);
logger.info("Org group id: "+ orgGroupId);
logger.info("Default activity status id: {}", defaultActivityStatusId);
logger.info("Default location id: {}", defaultLocationId);
logger.info("Default program classification: {}", defaultProgramClassification);
boolean hasGenericOrgGroupMapping = columnPairsToUse.containsValue(ImporterConstants.ORG_GROUP);
boolean donorMissingGroupMapping = columnPairsToUse.containsValue(ImporterConstants.DONOR_AGENCY)
&& !hasGenericOrgGroupMapping
&& !columnPairsToUse.containsValue(ImporterConstants.DONOR_ORGANIZATION_GROUP);
boolean responsibleMissingGroupMapping = columnPairsToUse.containsValue(ImporterConstants.RESPONSIBLE_ORGANIZATION)
&& !hasGenericOrgGroupMapping
&& !columnPairsToUse.containsValue(ImporterConstants.RESPONSIBLE_ORGANIZATION_GROUP);
boolean beneficiaryMissingGroupMapping = columnPairsToUse.containsValue(ImporterConstants.BENEFICIARY_AGENCY)
&& !hasGenericOrgGroupMapping
&& !columnPairsToUse.containsValue(ImporterConstants.BENEFICIARY_AGENCY_GROUP);
boolean executingMissingGroupMapping = columnPairsToUse.containsValue(ImporterConstants.EXECUTING_AGENCY)
&& !hasGenericOrgGroupMapping
&& !columnPairsToUse.containsValue(ImporterConstants.EXECUTING_AGENCY_GROUP);
boolean implementingMissingGroupMapping = columnPairsToUse.containsValue(ImporterConstants.IMPLEMENTING_AGENCY)
&& !hasGenericOrgGroupMapping
&& !columnPairsToUse.containsValue(ImporterConstants.IMPLEMENTING_AGENCY_GROUP);
boolean contractingMissingGroupMapping = columnPairsToUse.containsValue(ImporterConstants.CONTRACTING_AGENCY)
&& !hasGenericOrgGroupMapping
&& !columnPairsToUse.containsValue(ImporterConstants.CONTRACTING_AGENCY_GROUP);
boolean anyMappedOrgMissingGroup = donorMissingGroupMapping
|| responsibleMissingGroupMapping
|| beneficiaryMissingGroupMapping
|| executingMissingGroupMapping
|| implementingMissingGroupMapping
|| contractingMissingGroupMapping;
if (createMissingOrgs && orgGroupId == null && !createMissingOrgGroups
&& !columnPairsToUse.containsValue(ImporterConstants.ORG_GROUP)) {
&& anyMappedOrgMissingGroup) {
response.setHeader("errorMessage",
"Creating missing organizations requires a fallback Organization Group, the 'Create missing org groups' option, or an 'Organization Group' column mapping.");
"Please select a fallback Organization Group (or enable organization-group creation) when any mapped organization field has no corresponding group mapping.");
response.setStatus(400);
return mapping.findForward("importData");
}
Expand All @@ -403,15 +437,22 @@ public ActionForward execute(ActionMapping mapping, ActionForm form, HttpServlet
response.setStatus(400);
return mapping.findForward("importData");
}
if (columnPairsToUse.containsValue(ImporterConstants.PROGRAM_NAME)
&& !columnPairsToUse.containsValue(ImporterConstants.PROGRAM_CLASSIFICATION)
&& (defaultProgramClassification == null || defaultProgramClassification.trim().isEmpty())) {
response.setHeader("errorMessage", "Please select a default program classification when no 'Program Classification' column is mapped.");
response.setStatus(400);
return mapping.findForward("importData");
}
logger.info("Configuration: {}", columnPairsToUse);
try {
if ((Objects.equals(request.getParameter("fileType"), "excel") || Objects.equals(request.getParameter("fileType"), "csv"))) {
String dataSheetChoice = request.getParameter("dataSheetChoice");
String dataSheetName = request.getParameter("dataSheetName");
boolean useSpecificSheet = "sheet".equals(dataSheetChoice) && dataSheetName != null && !dataSheetName.trim().isEmpty();
res = processExcelFileInBatches(importedFilesRecord, tempFile, request, columnPairsToUse, isInternal, skipExisting, useSpecificSheet ? dataSheetName : null, createMissingOrgs, createMissingSectors, orgGroupId, createMissingOrgGroups, skipRecordsWithoutTransactions, validateActivities, addDisbursementForCommitment, defaultActivityStatusId, defaultLocationId);
res = processExcelFileInBatches(importedFilesRecord, tempFile, request, columnPairsToUse, isInternal, skipExisting, useSpecificSheet ? dataSheetName : null, createMissingOrgs, createMissingSectors, orgGroupId, createMissingOrgGroups, skipRecordsWithoutTransactions, validateActivities, addDisbursementForCommitment, defaultActivityStatusId, defaultLocationId, defaultProgramClassification, createMissingPrograms, replaceExistingTransactions);
} else if ( Objects.equals(request.getParameter("fileType"), "text")) {
res = TxtDataImporter.processTxtFileInBatches(importedFilesRecord, tempFile, request, columnPairsToUse, isInternal, skipExisting, createMissingOrgs, createMissingSectors, orgGroupId, createMissingOrgGroups, skipRecordsWithoutTransactions, validateActivities, addDisbursementForCommitment, defaultActivityStatusId, defaultLocationId);
res = TxtDataImporter.processTxtFileInBatches(importedFilesRecord, tempFile, request, columnPairsToUse, isInternal, skipExisting, createMissingOrgs, createMissingSectors, orgGroupId, createMissingOrgGroups, skipRecordsWithoutTransactions, validateActivities, addDisbursementForCommitment, defaultActivityStatusId, defaultLocationId, defaultProgramClassification, createMissingPrograms, replaceExistingTransactions);
}
} catch (Exception e) {
ImportedFileUtil.updateFileStatus(importedFilesRecord, ImportStatus.FAILED);
Expand Down Expand Up @@ -615,10 +656,16 @@ private Map<String, String> getEntityFieldsInfo() {
fieldsInfos.add(ImporterConstants.EXCHANGE_RATE);
fieldsInfos.add(ImporterConstants.DONOR_AGENCY_CODE);
fieldsInfos.add(ImporterConstants.ORG_GROUP);
fieldsInfos.add(ImporterConstants.DONOR_ORGANIZATION_GROUP);
fieldsInfos.add(ImporterConstants.RESPONSIBLE_ORGANIZATION);
fieldsInfos.add(ImporterConstants.RESPONSIBLE_ORGANIZATION_GROUP);
fieldsInfos.add(ImporterConstants.RESPONSIBLE_ORGANIZATION_CODE);
fieldsInfos.add(ImporterConstants.EXECUTING_AGENCY);
fieldsInfos.add(ImporterConstants.EXECUTING_AGENCY_GROUP);
fieldsInfos.add(ImporterConstants.IMPLEMENTING_AGENCY);
fieldsInfos.add(ImporterConstants.IMPLEMENTING_AGENCY_GROUP);
fieldsInfos.add(ImporterConstants.BENEFICIARY_AGENCY_GROUP);
fieldsInfos.add(ImporterConstants.CONTRACTING_AGENCY_GROUP);
fieldsInfos.add(ImporterConstants.ACTUAL_DISBURSEMENT);
fieldsInfos.add(ImporterConstants.ACTUAL_COMMITMENT);
fieldsInfos.add(ImporterConstants.ACTUAL_EXPENDITURE);
Expand All @@ -641,6 +688,11 @@ private Map<String, String> getEntityFieldsInfo() {
// Indicator columns for M&E import
fieldsInfos.add(ImporterConstants.INDICATOR_NAME);
fieldsInfos.add(ImporterConstants.PROGRAM_NAME);
fieldsInfos.add(ImporterConstants.PROGRAM_CLASSIFICATION);
fieldsInfos.add(ImporterConstants.PRIMARY_PROGRAM);
fieldsInfos.add(ImporterConstants.SECONDARY_PROGRAM);
fieldsInfos.add(ImporterConstants.TERTIARY_PROGRAM);
fieldsInfos.add(ImporterConstants.NATIONAL_PLAN_OBJECTIVE);
fieldsInfos.add(ImporterConstants.INDICATOR_LOCATION);
fieldsInfos.add(ImporterConstants.ORIGINAL_BASE_VALUE);
fieldsInfos.add(ImporterConstants.ORIGINAL_BASE_VALUE_DATE);
Expand Down Expand Up @@ -680,4 +732,17 @@ private List<AmpCategoryValue> getActivityStatuses() {
return activityStatuses;
}

private List<String> getProgramClassificationNames() {
List<AmpActivityProgramSettings> settings = ProgramUtil.getEnabledProgramSettings();
if (settings == null || settings.isEmpty()) {
settings = ProgramUtil.getAmpActivityProgramSettingsList(true);
}
return settings.stream()
.filter(Objects::nonNull)
.map(AmpActivityProgramSettings::getName)
.filter(Objects::nonNull)
.sorted(String.CASE_INSENSITIVE_ORDER)
.collect(Collectors.toList());
}

}
Loading
Loading