diff --git a/amp/src/main/java/org/digijava/module/aim/action/dataimporter/DataImporter.java b/amp/src/main/java/org/digijava/module/aim/action/dataimporter/DataImporter.java index 28625f69a70..862601420dd 100644 --- a/amp/src/main/java/org/digijava/module/aim/action/dataimporter/DataImporter.java +++ b/amp/src/main/java/org/digijava/module/aim/action/dataimporter/DataImporter.java @@ -55,10 +55,12 @@ import org.digijava.module.aim.action.dataimporter.util.ImporterConstants; import org.digijava.module.aim.dbentity.AmpOrgGroup; +import org.digijava.module.aim.dbentity.AmpActivityProgramSettings; import org.digijava.module.aim.form.DataImporterForm; import org.digijava.module.aim.util.DbUtil; import org.digijava.module.aim.util.DynLocationManagerUtil; import org.digijava.module.aim.util.LocationUtil; +import org.digijava.module.aim.util.ProgramUtil; import org.digijava.module.categorymanager.dbentity.AmpCategoryValue; import org.digijava.module.aim.dbentity.AmpCategoryValueLocations; import org.digijava.module.categorymanager.util.CategoryConstants; @@ -90,6 +92,7 @@ public ActionForward execute(ActionMapping mapping, ActionForm form, HttpServlet List orgGroups = DbUtil.getAllOrgGroups(); request.setAttribute("orgGroups", orgGroups); request.setAttribute("activityStatuses", getActivityStatuses()); + request.setAttribute("programClassifications", getProgramClassificationNames()); List availableLocations = getAvailableLocations(); request.setAttribute("availableLocations", availableLocations); AmpCategoryValueLocations defaultLocation = DynLocationManagerUtil.getDefaultCountry(); @@ -372,9 +375,12 @@ public ActionForward execute(ActionMapping mapping, ActionForm form, HttpServlet boolean createMissingOrgs = dataImporterForm.isCreateMissingOrgs(); boolean createMissingSectors = dataImporterForm.isCreateMissingSectors(); boolean createMissingOrgGroups = dataImporterForm.isCreateMissingOrgGroups(); + boolean createMissingPrograms = dataImporterForm.isCreateMissingPrograms(); + boolean replaceExistingTransactions = dataImporterForm.isReplaceExistingTransactions(); Long orgGroupId = dataImporterForm.getOrgGroupId(); Long defaultActivityStatusId = dataImporterForm.getDefaultActivityStatusId(); Long defaultLocationId = dataImporterForm.getDefaultLocationId(); + String defaultProgramClassification = dataImporterForm.getDefaultProgramClassification(); logger.info("Internal: "+ isInternal); logger.info("Skip existing: "+ skipExisting); logger.info("Validate activities: "+ validateActivities); @@ -383,13 +389,41 @@ public ActionForward execute(ActionMapping mapping, ActionForm form, HttpServlet logger.info("Create missing orgs: "+ createMissingOrgs); logger.info("Create missing sectors: {}", createMissingSectors); logger.info("Create missing org groups: " + createMissingOrgGroups); + logger.info("Create missing programs: " + createMissingPrograms); + logger.info("Replace existing transactions: " + replaceExistingTransactions); logger.info("Org group id: "+ orgGroupId); logger.info("Default activity status id: {}", defaultActivityStatusId); logger.info("Default location id: {}", defaultLocationId); + logger.info("Default program classification: {}", defaultProgramClassification); + boolean hasGenericOrgGroupMapping = columnPairsToUse.containsValue(ImporterConstants.ORG_GROUP); + boolean donorMissingGroupMapping = columnPairsToUse.containsValue(ImporterConstants.DONOR_AGENCY) + && !hasGenericOrgGroupMapping + && !columnPairsToUse.containsValue(ImporterConstants.DONOR_ORGANIZATION_GROUP); + boolean responsibleMissingGroupMapping = columnPairsToUse.containsValue(ImporterConstants.RESPONSIBLE_ORGANIZATION) + && !hasGenericOrgGroupMapping + && !columnPairsToUse.containsValue(ImporterConstants.RESPONSIBLE_ORGANIZATION_GROUP); + boolean beneficiaryMissingGroupMapping = columnPairsToUse.containsValue(ImporterConstants.BENEFICIARY_AGENCY) + && !hasGenericOrgGroupMapping + && !columnPairsToUse.containsValue(ImporterConstants.BENEFICIARY_AGENCY_GROUP); + boolean executingMissingGroupMapping = columnPairsToUse.containsValue(ImporterConstants.EXECUTING_AGENCY) + && !hasGenericOrgGroupMapping + && !columnPairsToUse.containsValue(ImporterConstants.EXECUTING_AGENCY_GROUP); + boolean implementingMissingGroupMapping = columnPairsToUse.containsValue(ImporterConstants.IMPLEMENTING_AGENCY) + && !hasGenericOrgGroupMapping + && !columnPairsToUse.containsValue(ImporterConstants.IMPLEMENTING_AGENCY_GROUP); + boolean contractingMissingGroupMapping = columnPairsToUse.containsValue(ImporterConstants.CONTRACTING_AGENCY) + && !hasGenericOrgGroupMapping + && !columnPairsToUse.containsValue(ImporterConstants.CONTRACTING_AGENCY_GROUP); + boolean anyMappedOrgMissingGroup = donorMissingGroupMapping + || responsibleMissingGroupMapping + || beneficiaryMissingGroupMapping + || executingMissingGroupMapping + || implementingMissingGroupMapping + || contractingMissingGroupMapping; if (createMissingOrgs && orgGroupId == null && !createMissingOrgGroups - && !columnPairsToUse.containsValue(ImporterConstants.ORG_GROUP)) { + && anyMappedOrgMissingGroup) { response.setHeader("errorMessage", - "Creating missing organizations requires a fallback Organization Group, the 'Create missing org groups' option, or an 'Organization Group' column mapping."); + "Please select a fallback Organization Group (or enable organization-group creation) when any mapped organization field has no corresponding group mapping."); response.setStatus(400); return mapping.findForward("importData"); } @@ -403,15 +437,22 @@ public ActionForward execute(ActionMapping mapping, ActionForm form, HttpServlet response.setStatus(400); return mapping.findForward("importData"); } + if (columnPairsToUse.containsValue(ImporterConstants.PROGRAM_NAME) + && !columnPairsToUse.containsValue(ImporterConstants.PROGRAM_CLASSIFICATION) + && (defaultProgramClassification == null || defaultProgramClassification.trim().isEmpty())) { + response.setHeader("errorMessage", "Please select a default program classification when no 'Program Classification' column is mapped."); + response.setStatus(400); + return mapping.findForward("importData"); + } logger.info("Configuration: {}", columnPairsToUse); try { if ((Objects.equals(request.getParameter("fileType"), "excel") || Objects.equals(request.getParameter("fileType"), "csv"))) { String dataSheetChoice = request.getParameter("dataSheetChoice"); String dataSheetName = request.getParameter("dataSheetName"); boolean useSpecificSheet = "sheet".equals(dataSheetChoice) && dataSheetName != null && !dataSheetName.trim().isEmpty(); - res = processExcelFileInBatches(importedFilesRecord, tempFile, request, columnPairsToUse, isInternal, skipExisting, useSpecificSheet ? dataSheetName : null, createMissingOrgs, createMissingSectors, orgGroupId, createMissingOrgGroups, skipRecordsWithoutTransactions, validateActivities, addDisbursementForCommitment, defaultActivityStatusId, defaultLocationId); + res = processExcelFileInBatches(importedFilesRecord, tempFile, request, columnPairsToUse, isInternal, skipExisting, useSpecificSheet ? dataSheetName : null, createMissingOrgs, createMissingSectors, orgGroupId, createMissingOrgGroups, skipRecordsWithoutTransactions, validateActivities, addDisbursementForCommitment, defaultActivityStatusId, defaultLocationId, defaultProgramClassification, createMissingPrograms, replaceExistingTransactions); } else if ( Objects.equals(request.getParameter("fileType"), "text")) { - res = TxtDataImporter.processTxtFileInBatches(importedFilesRecord, tempFile, request, columnPairsToUse, isInternal, skipExisting, createMissingOrgs, createMissingSectors, orgGroupId, createMissingOrgGroups, skipRecordsWithoutTransactions, validateActivities, addDisbursementForCommitment, defaultActivityStatusId, defaultLocationId); + res = TxtDataImporter.processTxtFileInBatches(importedFilesRecord, tempFile, request, columnPairsToUse, isInternal, skipExisting, createMissingOrgs, createMissingSectors, orgGroupId, createMissingOrgGroups, skipRecordsWithoutTransactions, validateActivities, addDisbursementForCommitment, defaultActivityStatusId, defaultLocationId, defaultProgramClassification, createMissingPrograms, replaceExistingTransactions); } } catch (Exception e) { ImportedFileUtil.updateFileStatus(importedFilesRecord, ImportStatus.FAILED); @@ -615,10 +656,16 @@ private Map getEntityFieldsInfo() { fieldsInfos.add(ImporterConstants.EXCHANGE_RATE); fieldsInfos.add(ImporterConstants.DONOR_AGENCY_CODE); fieldsInfos.add(ImporterConstants.ORG_GROUP); + fieldsInfos.add(ImporterConstants.DONOR_ORGANIZATION_GROUP); fieldsInfos.add(ImporterConstants.RESPONSIBLE_ORGANIZATION); + fieldsInfos.add(ImporterConstants.RESPONSIBLE_ORGANIZATION_GROUP); fieldsInfos.add(ImporterConstants.RESPONSIBLE_ORGANIZATION_CODE); fieldsInfos.add(ImporterConstants.EXECUTING_AGENCY); + fieldsInfos.add(ImporterConstants.EXECUTING_AGENCY_GROUP); fieldsInfos.add(ImporterConstants.IMPLEMENTING_AGENCY); + fieldsInfos.add(ImporterConstants.IMPLEMENTING_AGENCY_GROUP); + fieldsInfos.add(ImporterConstants.BENEFICIARY_AGENCY_GROUP); + fieldsInfos.add(ImporterConstants.CONTRACTING_AGENCY_GROUP); fieldsInfos.add(ImporterConstants.ACTUAL_DISBURSEMENT); fieldsInfos.add(ImporterConstants.ACTUAL_COMMITMENT); fieldsInfos.add(ImporterConstants.ACTUAL_EXPENDITURE); @@ -641,6 +688,11 @@ private Map getEntityFieldsInfo() { // Indicator columns for M&E import fieldsInfos.add(ImporterConstants.INDICATOR_NAME); fieldsInfos.add(ImporterConstants.PROGRAM_NAME); + fieldsInfos.add(ImporterConstants.PROGRAM_CLASSIFICATION); + fieldsInfos.add(ImporterConstants.PRIMARY_PROGRAM); + fieldsInfos.add(ImporterConstants.SECONDARY_PROGRAM); + fieldsInfos.add(ImporterConstants.TERTIARY_PROGRAM); + fieldsInfos.add(ImporterConstants.NATIONAL_PLAN_OBJECTIVE); fieldsInfos.add(ImporterConstants.INDICATOR_LOCATION); fieldsInfos.add(ImporterConstants.ORIGINAL_BASE_VALUE); fieldsInfos.add(ImporterConstants.ORIGINAL_BASE_VALUE_DATE); @@ -680,4 +732,17 @@ private List getActivityStatuses() { return activityStatuses; } + private List getProgramClassificationNames() { + List settings = ProgramUtil.getEnabledProgramSettings(); + if (settings == null || settings.isEmpty()) { + settings = ProgramUtil.getAmpActivityProgramSettingsList(true); + } + return settings.stream() + .filter(Objects::nonNull) + .map(AmpActivityProgramSettings::getName) + .filter(Objects::nonNull) + .sorted(String.CASE_INSENSITIVE_ORDER) + .collect(Collectors.toList()); + } + } diff --git a/amp/src/main/java/org/digijava/module/aim/action/dataimporter/ExcelImporter.java b/amp/src/main/java/org/digijava/module/aim/action/dataimporter/ExcelImporter.java index 3a7b8797d4e..e65667ec953 100644 --- a/amp/src/main/java/org/digijava/module/aim/action/dataimporter/ExcelImporter.java +++ b/amp/src/main/java/org/digijava/module/aim/action/dataimporter/ExcelImporter.java @@ -46,10 +46,10 @@ public class ExcelImporter { private static final int BATCH_SIZE = 1000; public static int processExcelFileInBatches(ImportedFilesRecord importedFilesRecord, File file, HttpServletRequest request, Map config, boolean isInternal) { - return processExcelFileInBatches(importedFilesRecord, file, request, config, isInternal, false, null, false, false, null, false, false, false, false, null, null); + return processExcelFileInBatches(importedFilesRecord, file, request, config, isInternal, false, null, false, false, null, false, false, false, false, null, null, null, false, false); } - public static int processExcelFileInBatches(ImportedFilesRecord importedFilesRecord, File file, HttpServletRequest request, Map config, boolean isInternal, boolean skipExisting, String sheetNameToProcess, boolean createMissingOrgs, boolean createMissingSectors, Long orgGroupId, boolean createMissingOrgGroups, boolean skipRecordsWithoutTransactions, boolean validateActivities, boolean addDisbursementForCommitment, Long defaultActivityStatusId, Long defaultLocationId) { + public static int processExcelFileInBatches(ImportedFilesRecord importedFilesRecord, File file, HttpServletRequest request, Map config, boolean isInternal, boolean skipExisting, String sheetNameToProcess, boolean createMissingOrgs, boolean createMissingSectors, Long orgGroupId, boolean createMissingOrgGroups, boolean skipRecordsWithoutTransactions, boolean validateActivities, boolean addDisbursementForCommitment, Long defaultActivityStatusId, Long defaultLocationId, String defaultProgramClassification, boolean createMissingPrograms, boolean replaceExistingTransactions) { int res = 0; ImportedFileUtil.updateFileStatus(importedFilesRecord, ImportStatus.IN_PROGRESS); try (Workbook workbook = new XSSFWorkbook(file)) { @@ -66,7 +66,7 @@ public static int processExcelFileInBatches(ImportedFilesRecord importedFilesRec if (isInternal) { addDonorAgencyColumn(sheet, FeaturesUtil.getGlobalSettingValue("Internal Ecowas Donor")); } - processSheetInBatches(sheet, request, config, importedFilesRecord, skipExisting, createMissingOrgs, createMissingSectors, orgGroupId, createMissingOrgGroups, skipRecordsWithoutTransactions, validateActivities, addDisbursementForCommitment, defaultActivityStatusId, defaultLocationId); + processSheetInBatches(sheet, request, config, importedFilesRecord, skipExisting, createMissingOrgs, createMissingSectors, orgGroupId, createMissingOrgGroups, skipRecordsWithoutTransactions, validateActivities, addDisbursementForCommitment, defaultActivityStatusId, defaultLocationId, defaultProgramClassification, createMissingPrograms, replaceExistingTransactions); } else { // Process each sheet in the workbook for (int i = 0; i < numberOfSheets; i++) { @@ -75,7 +75,7 @@ public static int processExcelFileInBatches(ImportedFilesRecord importedFilesRec if (isInternal) { addDonorAgencyColumn(sheet, FeaturesUtil.getGlobalSettingValue("Internal Ecowas Donor")); } - processSheetInBatches(sheet, request, config, importedFilesRecord, skipExisting, createMissingOrgs, createMissingSectors, orgGroupId, createMissingOrgGroups, skipRecordsWithoutTransactions, validateActivities, addDisbursementForCommitment, defaultActivityStatusId, defaultLocationId); + processSheetInBatches(sheet, request, config, importedFilesRecord, skipExisting, createMissingOrgs, createMissingSectors, orgGroupId, createMissingOrgGroups, skipRecordsWithoutTransactions, validateActivities, addDisbursementForCommitment, defaultActivityStatusId, defaultLocationId, defaultProgramClassification, createMissingPrograms, replaceExistingTransactions); } } @@ -119,7 +119,7 @@ private static void addDonorAgencyColumn(Sheet sheet, String donorAgencyValue) { } - public static void processSheetInBatches(Sheet sheet, HttpServletRequest request,Map config, ImportedFilesRecord importedFilesRecord, boolean skipExisting, boolean createMissingOrgs, boolean createMissingSectors, Long orgGroupId, boolean createMissingOrgGroups, boolean skipRecordsWithoutTransactions, boolean validateActivities, boolean addDisbursementForCommitment, Long defaultActivityStatusId, Long defaultLocationId) throws JsonProcessingException { + public static void processSheetInBatches(Sheet sheet, HttpServletRequest request,Map config, ImportedFilesRecord importedFilesRecord, boolean skipExisting, boolean createMissingOrgs, boolean createMissingSectors, Long orgGroupId, boolean createMissingOrgGroups, boolean skipRecordsWithoutTransactions, boolean validateActivities, boolean addDisbursementForCommitment, Long defaultActivityStatusId, Long defaultLocationId, String defaultProgramClassification, boolean createMissingPrograms, boolean replaceExistingTransactions) throws JsonProcessingException { // Get the number of rows in the sheet int rowCount = sheet.getPhysicalNumberOfRows(); logger.info("There are {} rows in sheet {} " , rowCount, sheet.getSheetName()); @@ -141,12 +141,12 @@ public static void processSheetInBatches(Sheet sheet, HttpServletRequest request } // Process the batch - processBatch(batch, sheet, request,config, importedFilesRecord, skipExisting, createMissingOrgs, createMissingSectors, orgGroupId, createMissingOrgGroups, skipRecordsWithoutTransactions, validateActivities, addDisbursementForCommitment, defaultActivityStatusId, defaultLocationId); + processBatch(batch, sheet, request,config, importedFilesRecord, skipExisting, createMissingOrgs, createMissingSectors, orgGroupId, createMissingOrgGroups, skipRecordsWithoutTransactions, validateActivities, addDisbursementForCommitment, defaultActivityStatusId, defaultLocationId, defaultProgramClassification, createMissingPrograms, replaceExistingTransactions); } } - public static void processBatch(List batch,Sheet sheet, HttpServletRequest request, Map config, ImportedFilesRecord importedFilesRecord, boolean skipExisting, boolean createMissingOrgs, boolean createMissingSectors, Long orgGroupId, boolean createMissingOrgGroups, boolean skipRecordsWithoutTransactions, boolean validateActivities, boolean addDisbursementForCommitment, Long defaultActivityStatusId, Long defaultLocationId) throws JsonProcessingException { + public static void processBatch(List batch,Sheet sheet, HttpServletRequest request, Map config, ImportedFilesRecord importedFilesRecord, boolean skipExisting, boolean createMissingOrgs, boolean createMissingSectors, Long orgGroupId, boolean createMissingOrgGroups, boolean skipRecordsWithoutTransactions, boolean validateActivities, boolean addDisbursementForCommitment, Long defaultActivityStatusId, Long defaultLocationId, String defaultProgramClassification, boolean createMissingPrograms, boolean replaceExistingTransactions) throws JsonProcessingException { // Process the batch of rows SessionUtil.extendSessionIfNeeded(request); DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSXXX"); @@ -209,10 +209,19 @@ public static void processBatch(List batch,Sheet sheet, HttpServletRequest } else { importedOrgGroupName = null; } + String donorOrgGroupNames = ImporterUtil.getCellValueByConfig(rowRef, sheet, config, ImporterConstants.DONOR_ORGANIZATION_GROUP); + String responsibleOrgGroupNames = ImporterUtil.getCellValueByConfig(rowRef, sheet, config, ImporterConstants.RESPONSIBLE_ORGANIZATION_GROUP); + String beneficiaryOrgGroupNames = ImporterUtil.getCellValueByConfig(rowRef, sheet, config, ImporterConstants.BENEFICIARY_AGENCY_GROUP); + String executingOrgGroupNames = ImporterUtil.getCellValueByConfig(rowRef, sheet, config, ImporterConstants.EXECUTING_AGENCY_GROUP); + String implementingOrgGroupNames = ImporterUtil.getCellValueByConfig(rowRef, sheet, config, ImporterConstants.IMPLEMENTING_AGENCY_GROUP); + String contractingOrgGroupNames = ImporterUtil.getCellValueByConfig(rowRef, sheet, config, ImporterConstants.CONTRACTING_AGENCY_GROUP); // Use holder arrays to capture values from lambda (for effectively final requirement) final Long[] existingActivityIdHolder = new Long[1]; // Store only the ID, not the entity final Long[] responsibleOrgIdHolder = new Long[1]; + final String[] programNamesHolder = new String[1]; + final String[] programClassificationHolder = new String[1]; + final Map specificProgramValuesHolder = new java.util.LinkedHashMap<>(); // Phase 1: Data preparation - use transaction for reading/preparing data ONLY try { @@ -255,6 +264,20 @@ public static void processBatch(List batch,Sheet sheet, HttpServletRequest if (columnIndex >= 0) { Cell cell = rowRef.getCell(columnIndex); switch (entry.getValue()) { + case ImporterConstants.PROJECT_START_DATE: { + String formatted = org.digijava.module.aim.action.dataimporter.util.ImporterUtil.extractDateFromStringCell(cell); + if (formatted != null) { + importDataModel.setActual_start_date(formatted); + } + break; + } + case ImporterConstants.PROJECT_END_DATE: { + String formatted = org.digijava.module.aim.action.dataimporter.util.ImporterUtil.extractDateFromStringCell(cell); + if (formatted != null) { + importDataModel.setActual_completion_date(formatted); + } + break; + } case ImporterConstants.PROJECT_LOCATION: updateLocations(importDataModel,Objects.requireNonNull(getStringValueFromCell(cell, false)).trim(),session); break; @@ -270,22 +293,22 @@ public static void processBatch(List batch,Sheet sheet, HttpServletRequest break; case ImporterConstants.DONOR_AGENCY: logger.info("Getting donor"); - updateOrgs(importDataModel, Objects.requireNonNull(getStringValueFromCell(cell, false)).trim(), donorAgencyCode, session, ImporterConstants.ORG_TYPE_DONOR, createMissingOrgs, orgGroupId, importedOrgGroupName, createMissingOrgGroups); + updateOrgs(importDataModel, Objects.requireNonNull(getStringValueFromCell(cell, false)).trim(), donorAgencyCode, session, ImporterConstants.ORG_TYPE_DONOR, createMissingOrgs, orgGroupId, resolveOrgGroups(donorOrgGroupNames, importedOrgGroupName), createMissingOrgGroups); break; case ImporterConstants.RESPONSIBLE_ORGANIZATION: - responsibleOrgIdHolder[0] = updateOrgs(importDataModel, Objects.requireNonNull(getStringValueFromCell(cell, false)).trim(), responsibleOrgCode, session, ImporterConstants.ORG_TYPE_RESPONSIBLE_ORG, createMissingOrgs, orgGroupId, importedOrgGroupName, createMissingOrgGroups); + responsibleOrgIdHolder[0] = updateOrgs(importDataModel, Objects.requireNonNull(getStringValueFromCell(cell, false)).trim(), responsibleOrgCode, session, ImporterConstants.ORG_TYPE_RESPONSIBLE_ORG, createMissingOrgs, orgGroupId, resolveOrgGroups(responsibleOrgGroupNames, importedOrgGroupName), createMissingOrgGroups); break; case ImporterConstants.BENEFICIARY_AGENCY: - responsibleOrgIdHolder[0] = updateOrgs(importDataModel, Objects.requireNonNull(getStringValueFromCell(cell, false)).trim(), responsibleOrgCode, session, ImporterConstants.ORG_TYPE_BENEFICIARY_AGENCY, createMissingOrgs, orgGroupId, importedOrgGroupName, createMissingOrgGroups); + responsibleOrgIdHolder[0] = updateOrgs(importDataModel, Objects.requireNonNull(getStringValueFromCell(cell, false)).trim(), responsibleOrgCode, session, ImporterConstants.ORG_TYPE_BENEFICIARY_AGENCY, createMissingOrgs, orgGroupId, resolveOrgGroups(beneficiaryOrgGroupNames, importedOrgGroupName), createMissingOrgGroups); break; case ImporterConstants.EXECUTING_AGENCY: - updateOrgs(importDataModel, Objects.requireNonNull(getStringValueFromCell(cell, false)).trim(), null, session, ImporterConstants.ORG_TYPE_EXECUTING_AGENCY, createMissingOrgs, orgGroupId, importedOrgGroupName, createMissingOrgGroups); + updateOrgs(importDataModel, Objects.requireNonNull(getStringValueFromCell(cell, false)).trim(), null, session, ImporterConstants.ORG_TYPE_EXECUTING_AGENCY, createMissingOrgs, orgGroupId, resolveOrgGroups(executingOrgGroupNames, importedOrgGroupName), createMissingOrgGroups); break; case ImporterConstants.IMPLEMENTING_AGENCY: - updateOrgs(importDataModel, Objects.requireNonNull(getStringValueFromCell(cell, false)).trim(), null, session, ImporterConstants.ORG_TYPE_IMPLEMENTING_AGENCY, createMissingOrgs, orgGroupId, importedOrgGroupName, createMissingOrgGroups); + updateOrgs(importDataModel, Objects.requireNonNull(getStringValueFromCell(cell, false)).trim(), null, session, ImporterConstants.ORG_TYPE_IMPLEMENTING_AGENCY, createMissingOrgs, orgGroupId, resolveOrgGroups(implementingOrgGroupNames, importedOrgGroupName), createMissingOrgGroups); break; case ImporterConstants.CONTRACTING_AGENCY: - updateOrgs(importDataModel, Objects.requireNonNull(getStringValueFromCell(cell, false)).trim(), null, session, ImporterConstants.ORG_TYPE_CONTRACTING_AGENCY, createMissingOrgs, orgGroupId, importedOrgGroupName, createMissingOrgGroups); + updateOrgs(importDataModel, Objects.requireNonNull(getStringValueFromCell(cell, false)).trim(), null, session, ImporterConstants.ORG_TYPE_CONTRACTING_AGENCY, createMissingOrgs, orgGroupId, resolveOrgGroups(contractingOrgGroupNames, importedOrgGroupName), createMissingOrgGroups); break; case ImporterConstants.TRANSACTION_AMOUNT: { boolean commitment = true, disbursement = true, expenditure = false; @@ -324,11 +347,35 @@ public static void processBatch(List batch,Sheet sheet, HttpServletRequest case ImporterConstants.MEASURE_TYPE: break; case ImporterConstants.ORG_GROUP: + case ImporterConstants.DONOR_ORGANIZATION_GROUP: + case ImporterConstants.RESPONSIBLE_ORGANIZATION_GROUP: + case ImporterConstants.BENEFICIARY_AGENCY_GROUP: + case ImporterConstants.EXECUTING_AGENCY_GROUP: + case ImporterConstants.IMPLEMENTING_AGENCY_GROUP: + case ImporterConstants.CONTRACTING_AGENCY_GROUP: break; case ImporterConstants.PROJECT_STATUS: break; case ImporterConstants.PROCUREMENT_SYSTEM: break; + case ImporterConstants.PROGRAM_NAME: + programNamesHolder[0] = getStringValueFromCell(cell, false); + break; + case ImporterConstants.PROGRAM_CLASSIFICATION: + programClassificationHolder[0] = getStringValueFromCell(cell, false); + break; + case ImporterConstants.PRIMARY_PROGRAM: + specificProgramValuesHolder.put(ImporterConstants.PRIMARY_PROGRAM, getStringValueFromCell(cell, false)); + break; + case ImporterConstants.SECONDARY_PROGRAM: + specificProgramValuesHolder.put(ImporterConstants.SECONDARY_PROGRAM, getStringValueFromCell(cell, false)); + break; + case ImporterConstants.TERTIARY_PROGRAM: + specificProgramValuesHolder.put(ImporterConstants.TERTIARY_PROGRAM, getStringValueFromCell(cell, false)); + break; + case ImporterConstants.NATIONAL_PLAN_OBJECTIVE: + specificProgramValuesHolder.put(ImporterConstants.NATIONAL_PLAN_OBJECTIVE, getStringValueFromCell(cell, false)); + break; case ImporterConstants.REPORTING_DATE: default: logger.error("Unexpected value: " + entry.getValue()); @@ -376,7 +423,7 @@ public static void processBatch(List batch,Sheet sheet, HttpServletRequest if (importedProject.getImportStatus() != ImportStatus.SKIPPED) { try { // Pass only the ID, not the entity - importTheData will re-fetch in its own transaction context - activityId = importTheData(importDataModel, null, importedProject, componentName, componentCode, responsibleOrgIdHolder[0], fundings, existingActivityIdHolder[0], validateActivities); + activityId = importTheData(importDataModel, null, importedProject, componentName, componentCode, responsibleOrgIdHolder[0], fundings, existingActivityIdHolder[0], validateActivities, replaceExistingTransactions); } catch (JsonProcessingException e) { throw e; } @@ -396,7 +443,46 @@ public static void processBatch(List batch,Sheet sheet, HttpServletRequest logger.error("Failed to add indicator data for activity " + activityId, e); } } + + if (activityId != null && (programNamesHolder[0] != null && !programNamesHolder[0].trim().isEmpty())) { + try { + final Long activityIdFinal = activityId; + PersistenceManager.inTransaction(() -> { + Session s = PersistenceManager.getRequestDBSession(); + addProgramsToActivity(activityIdFinal, programNamesHolder[0], programClassificationHolder[0], + defaultProgramClassification, createMissingPrograms, s); + }); + } catch (Exception e) { + logger.error("Failed to add programs for activity " + activityId, e); + } + } + + if (activityId != null && !specificProgramValuesHolder.isEmpty()) { + try { + final Long activityIdFinal = activityId; + PersistenceManager.inTransaction(() -> { + Session s = PersistenceManager.getRequestDBSession(); + for (Map.Entry specificProgramEntry : specificProgramValuesHolder.entrySet()) { + String rawProgramNames = specificProgramEntry.getValue(); + if (rawProgramNames == null || rawProgramNames.trim().isEmpty()) { + continue; + } + addProgramsToActivity(activityIdFinal, rawProgramNames, + specificProgramEntry.getKey(), null, createMissingPrograms, s); + } + }); + } catch (Exception e) { + logger.error("Failed to add specific programs for activity " + activityId, e); + } + } } } } + + private static String resolveOrgGroups(String roleSpecificGroupNames, String fallbackGroupNames) { + if (roleSpecificGroupNames != null && !roleSpecificGroupNames.trim().isEmpty()) { + return roleSpecificGroupNames.trim(); + } + return fallbackGroupNames; + } } diff --git a/amp/src/main/java/org/digijava/module/aim/action/dataimporter/TxtDataImporter.java b/amp/src/main/java/org/digijava/module/aim/action/dataimporter/TxtDataImporter.java index 18f2b32aee3..7bcfeb52db6 100644 --- a/amp/src/main/java/org/digijava/module/aim/action/dataimporter/TxtDataImporter.java +++ b/amp/src/main/java/org/digijava/module/aim/action/dataimporter/TxtDataImporter.java @@ -40,7 +40,7 @@ public class TxtDataImporter { private static final Logger logger = LoggerFactory.getLogger(TxtDataImporter.class); - public static int processTxtFileInBatches(ImportedFilesRecord importedFilesRecord, File file, HttpServletRequest request, Map config, boolean isInternal, boolean skipExisting, boolean createMissingOrgs, boolean createMissingSectors, Long orgGroupId, boolean createMissingOrgGroups, boolean skipRecordsWithoutTransactions, boolean validateActivities, boolean addDisbursementForCommitment, Long defaultActivityStatusId, Long defaultLocationId) + public static int processTxtFileInBatches(ImportedFilesRecord importedFilesRecord, File file, HttpServletRequest request, Map config, boolean isInternal, boolean skipExisting, boolean createMissingOrgs, boolean createMissingSectors, Long orgGroupId, boolean createMissingOrgGroups, boolean skipRecordsWithoutTransactions, boolean validateActivities, boolean addDisbursementForCommitment, Long defaultActivityStatusId, Long defaultLocationId, String defaultProgramClassification, boolean createMissingPrograms, boolean replaceExistingTransactions) { logger.info("Processing txt file: " + file.getName()); CSVParser parser = new CSVParserBuilder().withSeparator(request.getParameter("dataSeparator").charAt(0)).build(); @@ -59,7 +59,7 @@ public static int processTxtFileInBatches(ImportedFilesRecord importedFilesRecor logger.info("Batch number here: {}",batchNumber); // Process the batch - processBatch(batch, request, config, importedFilesRecord, skipExisting, createMissingOrgs, createMissingSectors, orgGroupId, createMissingOrgGroups, skipRecordsWithoutTransactions, validateActivities, addDisbursementForCommitment, defaultActivityStatusId, defaultLocationId); + processBatch(batch, request, config, importedFilesRecord, skipExisting, createMissingOrgs, createMissingSectors, orgGroupId, createMissingOrgGroups, skipRecordsWithoutTransactions, validateActivities, addDisbursementForCommitment, defaultActivityStatusId, defaultLocationId, defaultProgramClassification, createMissingPrograms, replaceExistingTransactions); // Clear the batch for the next set of rows batch.clear(); batchNumber+=1; @@ -69,7 +69,7 @@ public static int processTxtFileInBatches(ImportedFilesRecord importedFilesRecor // Process any remaining rows in the batch if (!batch.isEmpty()) { logger.info("Processing last batch of size {}", batch.size()); - processBatch(batch, request, config, importedFilesRecord, skipExisting, createMissingOrgs, createMissingSectors, orgGroupId, createMissingOrgGroups, skipRecordsWithoutTransactions, validateActivities, addDisbursementForCommitment, defaultActivityStatusId, defaultLocationId); + processBatch(batch, request, config, importedFilesRecord, skipExisting, createMissingOrgs, createMissingSectors, orgGroupId, createMissingOrgGroups, skipRecordsWithoutTransactions, validateActivities, addDisbursementForCommitment, defaultActivityStatusId, defaultLocationId, defaultProgramClassification, createMissingPrograms, replaceExistingTransactions); } } catch (IOException | CsvValidationException e) { logger.error("Error processing txt file "+e.getMessage(),e); @@ -79,7 +79,7 @@ public static int processTxtFileInBatches(ImportedFilesRecord importedFilesRecor } - private static void processBatch(List> batch, HttpServletRequest request, Map config, ImportedFilesRecord importedFilesRecord, boolean skipExisting, boolean createMissingOrgs, boolean createMissingSectors, Long orgGroupId, boolean createMissingOrgGroups, boolean skipRecordsWithoutTransactions, boolean validateActivities, boolean addDisbursementForCommitment, Long defaultActivityStatusId, Long defaultLocationId) throws JsonProcessingException { + private static void processBatch(List> batch, HttpServletRequest request, Map config, ImportedFilesRecord importedFilesRecord, boolean skipExisting, boolean createMissingOrgs, boolean createMissingSectors, Long orgGroupId, boolean createMissingOrgGroups, boolean skipRecordsWithoutTransactions, boolean validateActivities, boolean addDisbursementForCommitment, Long defaultActivityStatusId, Long defaultLocationId, String defaultProgramClassification, boolean createMissingPrograms, boolean replaceExistingTransactions) throws JsonProcessingException { logger.info("Processing txt batch"); SessionUtil.extendSessionIfNeeded(request); DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSXXX"); @@ -116,10 +116,19 @@ private static void processBatch(List> batch, HttpServletReq } else { importedOrgGroupName = null; } + String donorOrgGroupNames = rowRef.get(getKey(config, ImporterConstants.DONOR_ORGANIZATION_GROUP)); + String responsibleOrgGroupNames = rowRef.get(getKey(config, ImporterConstants.RESPONSIBLE_ORGANIZATION_GROUP)); + String beneficiaryOrgGroupNames = rowRef.get(getKey(config, ImporterConstants.BENEFICIARY_AGENCY_GROUP)); + String executingOrgGroupNames = rowRef.get(getKey(config, ImporterConstants.EXECUTING_AGENCY_GROUP)); + String implementingOrgGroupNames = rowRef.get(getKey(config, ImporterConstants.IMPLEMENTING_AGENCY_GROUP)); + String contractingOrgGroupNames = rowRef.get(getKey(config, ImporterConstants.CONTRACTING_AGENCY_GROUP)); // Use holder arrays to capture values from lambda (for effectively final requirement) final Long[] existingActivityIdHolder = new Long[1]; // Store only the ID, not the entity final Long[] responsibleOrgIdHolder = new Long[1]; + final String[] programNamesHolder = new String[1]; + final String[] programClassificationHolder = new String[1]; + final Map specificProgramValuesHolder = new java.util.LinkedHashMap<>(); // Phase 1: Data preparation - use transaction for reading/preparing data ONLY try { @@ -153,6 +162,26 @@ private static void processBatch(List> batch, HttpServletReq logger.info("Configuration: " + config); for (Map.Entry entry : config.entrySet()) { switch (entry.getValue()) { + case ImporterConstants.PROJECT_START_DATE: { + String dateStr = rowRef.get(entry.getKey().trim()); + if (dateStr != null && !dateStr.trim().isEmpty()) { + String formatted = org.digijava.module.aim.action.dataimporter.util.ImporterUtil.formatDateFromDateObject(dateStr.trim()); + if (formatted != null) { + importDataModel.setActual_start_date(formatted); + } + } + break; + } + case ImporterConstants.PROJECT_END_DATE: { + String dateStr = rowRef.get(entry.getKey().trim()); + if (dateStr != null && !dateStr.trim().isEmpty()) { + String formatted = org.digijava.module.aim.action.dataimporter.util.ImporterUtil.formatDateFromDateObject(dateStr.trim()); + if (formatted != null) { + importDataModel.setActual_completion_date(formatted); + } + } + break; + } case ImporterConstants.PROJECT_LOCATION: updateLocations(importDataModel, rowRef.get(entry.getKey().trim()), session); break; @@ -167,22 +196,22 @@ private static void processBatch(List> batch, HttpServletReq ImporterConstants.SECONDARY_SECTOR); break; case ImporterConstants.DONOR_AGENCY: - updateOrgs(importDataModel, rowRef.get(entry.getKey().trim()), donorAgencyCode, session, ImporterConstants.ORG_TYPE_DONOR, createMissingOrgs, orgGroupId, importedOrgGroupName, createMissingOrgGroups); + updateOrgs(importDataModel, rowRef.get(entry.getKey().trim()), donorAgencyCode, session, ImporterConstants.ORG_TYPE_DONOR, createMissingOrgs, orgGroupId, resolveOrgGroups(donorOrgGroupNames, importedOrgGroupName), createMissingOrgGroups); break; case ImporterConstants.RESPONSIBLE_ORGANIZATION: - responsibleOrgIdHolder[0] = updateOrgs(importDataModel, rowRef.get(entry.getKey().trim()), responsibleOrgCode, session, ImporterConstants.ORG_TYPE_RESPONSIBLE_ORG, createMissingOrgs, orgGroupId, importedOrgGroupName, createMissingOrgGroups); + responsibleOrgIdHolder[0] = updateOrgs(importDataModel, rowRef.get(entry.getKey().trim()), responsibleOrgCode, session, ImporterConstants.ORG_TYPE_RESPONSIBLE_ORG, createMissingOrgs, orgGroupId, resolveOrgGroups(responsibleOrgGroupNames, importedOrgGroupName), createMissingOrgGroups); break; case ImporterConstants.BENEFICIARY_AGENCY: - responsibleOrgIdHolder[0] = updateOrgs(importDataModel, rowRef.get(entry.getKey().trim()), responsibleOrgCode, session, ImporterConstants.ORG_TYPE_BENEFICIARY_AGENCY, createMissingOrgs, orgGroupId, importedOrgGroupName, createMissingOrgGroups); + responsibleOrgIdHolder[0] = updateOrgs(importDataModel, rowRef.get(entry.getKey().trim()), responsibleOrgCode, session, ImporterConstants.ORG_TYPE_BENEFICIARY_AGENCY, createMissingOrgs, orgGroupId, resolveOrgGroups(beneficiaryOrgGroupNames, importedOrgGroupName), createMissingOrgGroups); break; case ImporterConstants.EXECUTING_AGENCY: - updateOrgs(importDataModel, rowRef.get(entry.getKey().trim()), null, session, ImporterConstants.ORG_TYPE_EXECUTING_AGENCY, createMissingOrgs, orgGroupId, importedOrgGroupName, createMissingOrgGroups); + updateOrgs(importDataModel, rowRef.get(entry.getKey().trim()), null, session, ImporterConstants.ORG_TYPE_EXECUTING_AGENCY, createMissingOrgs, orgGroupId, resolveOrgGroups(executingOrgGroupNames, importedOrgGroupName), createMissingOrgGroups); break; case ImporterConstants.IMPLEMENTING_AGENCY: - updateOrgs(importDataModel, rowRef.get(entry.getKey().trim()), null, session, ImporterConstants.ORG_TYPE_IMPLEMENTING_AGENCY, createMissingOrgs, orgGroupId, importedOrgGroupName, createMissingOrgGroups); + updateOrgs(importDataModel, rowRef.get(entry.getKey().trim()), null, session, ImporterConstants.ORG_TYPE_IMPLEMENTING_AGENCY, createMissingOrgs, orgGroupId, resolveOrgGroups(implementingOrgGroupNames, importedOrgGroupName), createMissingOrgGroups); break; case ImporterConstants.CONTRACTING_AGENCY: - updateOrgs(importDataModel, rowRef.get(entry.getKey().trim()), null, session, ImporterConstants.ORG_TYPE_CONTRACTING_AGENCY, createMissingOrgs, orgGroupId, importedOrgGroupName, createMissingOrgGroups); + updateOrgs(importDataModel, rowRef.get(entry.getKey().trim()), null, session, ImporterConstants.ORG_TYPE_CONTRACTING_AGENCY, createMissingOrgs, orgGroupId, resolveOrgGroups(contractingOrgGroupNames, importedOrgGroupName), createMissingOrgGroups); break; case ImporterConstants.TRANSACTION_AMOUNT: { boolean commitment = true, disbursement = true, expenditure = false; @@ -221,9 +250,33 @@ private static void processBatch(List> batch, HttpServletReq case ImporterConstants.MEASURE_TYPE: break; case ImporterConstants.ORG_GROUP: + case ImporterConstants.DONOR_ORGANIZATION_GROUP: + case ImporterConstants.RESPONSIBLE_ORGANIZATION_GROUP: + case ImporterConstants.BENEFICIARY_AGENCY_GROUP: + case ImporterConstants.EXECUTING_AGENCY_GROUP: + case ImporterConstants.IMPLEMENTING_AGENCY_GROUP: + case ImporterConstants.CONTRACTING_AGENCY_GROUP: break; case ImporterConstants.PROJECT_STATUS: break; + case ImporterConstants.PROGRAM_NAME: + programNamesHolder[0] = rowRef.get(entry.getKey().trim()); + break; + case ImporterConstants.PROGRAM_CLASSIFICATION: + programClassificationHolder[0] = rowRef.get(entry.getKey().trim()); + break; + case ImporterConstants.PRIMARY_PROGRAM: + specificProgramValuesHolder.put(ImporterConstants.PRIMARY_PROGRAM, rowRef.get(entry.getKey().trim())); + break; + case ImporterConstants.SECONDARY_PROGRAM: + specificProgramValuesHolder.put(ImporterConstants.SECONDARY_PROGRAM, rowRef.get(entry.getKey().trim())); + break; + case ImporterConstants.TERTIARY_PROGRAM: + specificProgramValuesHolder.put(ImporterConstants.TERTIARY_PROGRAM, rowRef.get(entry.getKey().trim())); + break; + case ImporterConstants.NATIONAL_PLAN_OBJECTIVE: + specificProgramValuesHolder.put(ImporterConstants.NATIONAL_PLAN_OBJECTIVE, rowRef.get(entry.getKey().trim())); + break; default: logger.error("Unexpected value: " + entry.getValue()); break; @@ -268,11 +321,40 @@ private static void processBatch(List> batch, HttpServletReq // This avoids nested transaction issues when ActivityGatekeeper.doWithLock creates its own transaction try { // Pass only the ID, not the entity - importTheData will re-fetch in its own transaction context - importTheData(importDataModel, null, importedProject, componentName, componentCode, responsibleOrgIdHolder[0], fundings, existingActivityIdHolder[0], validateActivities); + Long activityId = importTheData(importDataModel, null, importedProject, componentName, componentCode, responsibleOrgIdHolder[0], fundings, existingActivityIdHolder[0], validateActivities, replaceExistingTransactions); + if (activityId != null && programNamesHolder[0] != null && !programNamesHolder[0].trim().isEmpty()) { + final Long activityIdFinal = activityId; + PersistenceManager.inTransaction(() -> { + Session s = PersistenceManager.getRequestDBSession(); + addProgramsToActivity(activityIdFinal, programNamesHolder[0], programClassificationHolder[0], + defaultProgramClassification, createMissingPrograms, s); + }); + } + if (activityId != null && !specificProgramValuesHolder.isEmpty()) { + final Long activityIdFinal = activityId; + PersistenceManager.inTransaction(() -> { + Session s = PersistenceManager.getRequestDBSession(); + for (Map.Entry specificProgramEntry : specificProgramValuesHolder.entrySet()) { + String rawProgramNames = specificProgramEntry.getValue(); + if (rawProgramNames == null || rawProgramNames.trim().isEmpty()) { + continue; + } + addProgramsToActivity(activityIdFinal, rawProgramNames, + specificProgramEntry.getKey(), null, createMissingPrograms, s); + } + }); + } } catch (JsonProcessingException e) { throw e; } } } + + private static String resolveOrgGroups(String roleSpecificGroupNames, String fallbackGroupNames) { + if (roleSpecificGroupNames != null && !roleSpecificGroupNames.trim().isEmpty()) { + return roleSpecificGroupNames.trim(); + } + return fallbackGroupNames; + } } diff --git a/amp/src/main/java/org/digijava/module/aim/action/dataimporter/util/ImporterConstants.java b/amp/src/main/java/org/digijava/module/aim/action/dataimporter/util/ImporterConstants.java index 8637a6fbcf5..298d069c6e9 100644 --- a/amp/src/main/java/org/digijava/module/aim/action/dataimporter/util/ImporterConstants.java +++ b/amp/src/main/java/org/digijava/module/aim/action/dataimporter/util/ImporterConstants.java @@ -35,12 +35,18 @@ private ImporterConstants() { public static final String DONOR_AGENCY_CODE = "Donor Agency Code"; public static final String EXCHANGE_RATE = "Exchange Rate"; public static final String ORG_GROUP = "Organization Group"; + public static final String DONOR_ORGANIZATION_GROUP = "Donor Organization Group"; public static final String RESPONSIBLE_ORGANIZATION = "Responsible Organization"; + public static final String RESPONSIBLE_ORGANIZATION_GROUP = "Responsible Organization Group"; public static final String RESPONSIBLE_ORGANIZATION_CODE = "Responsible Organization Code"; public static final String EXECUTING_AGENCY = "Executing Agency"; + public static final String EXECUTING_AGENCY_GROUP = "Executing Agency Group"; public static final String IMPLEMENTING_AGENCY = "Implementing Agency"; + public static final String IMPLEMENTING_AGENCY_GROUP = "Implementing Agency Group"; public static final String CONTRACTING_AGENCY = "Contracting Agency"; + public static final String CONTRACTING_AGENCY_GROUP = "Contracting Agency Group"; public static final String BENEFICIARY_AGENCY = "Beneficiary Agency"; + public static final String BENEFICIARY_AGENCY_GROUP = "Beneficiary Agency Group"; public static final String ACTUAL_DISBURSEMENT = "Actual Disbursement"; public static final String ACTUAL_COMMITMENT = "Actual Commitment"; @@ -66,6 +72,11 @@ private ImporterConstants() { // ----- Indicator (M&E) columns ----- public static final String INDICATOR_NAME = "Indicator Name"; public static final String PROGRAM_NAME = "Program Name"; + public static final String PROGRAM_CLASSIFICATION = "Program Classification"; + public static final String PRIMARY_PROGRAM = "Primary Program"; + public static final String SECONDARY_PROGRAM = "Secondary Program"; + public static final String TERTIARY_PROGRAM = "Tertiary Program"; + public static final String NATIONAL_PLAN_OBJECTIVE = "National Plan Objective"; /** Used for project-level location (e.g. Project Location). */ public static final String LOCATION = "Location"; /** Used for matching indicator value to activity location; distinct from project Location. */ diff --git a/amp/src/main/java/org/digijava/module/aim/action/dataimporter/util/ImporterUtil.java b/amp/src/main/java/org/digijava/module/aim/action/dataimporter/util/ImporterUtil.java index f96aebe3ec3..60443eb52c8 100644 --- a/amp/src/main/java/org/digijava/module/aim/action/dataimporter/util/ImporterUtil.java +++ b/amp/src/main/java/org/digijava/module/aim/action/dataimporter/util/ImporterUtil.java @@ -10,6 +10,7 @@ import org.apache.poi.ss.usermodel.Row; import org.apache.poi.ss.usermodel.Sheet; import org.dgfoundation.amp.ar.ArConstants; +import org.dgfoundation.amp.ar.ColumnConstants; import org.dgfoundation.amp.ar.ARUtil; import org.digijava.kernel.ampapi.endpoints.activity.ActivityImportRules; import org.digijava.kernel.ampapi.endpoints.activity.ActivityInterchangeUtils; @@ -113,8 +114,11 @@ public static List setFundingItemsForExcel(Sheet sheet, Map= 0 ? getStringValueFromCell(row.getCell(donorAgencyCodeColumn), true) : null; - updateOrgs(importDataModel, columnIndex1 >= 0 ? Objects.requireNonNull(getStringValueFromCell(row.getCell(columnIndex1), false)).trim() : "no org", donorAgencyCode, session, "donor", createMissingOrgs, orgGroupId, importedOrgGroupName, createMissingOrgGroups); + String donorOrgGroupNames = donorOrgGroupColumn >= 0 ? getStringValueFromCell(row.getCell(donorOrgGroupColumn), false) : null; + String resolvedDonorOrgGroups = StringUtils.isNotBlank(donorOrgGroupNames) ? donorOrgGroupNames.trim() : importedOrgGroupName; + updateOrgs(importDataModel, columnIndex1 >= 0 ? Objects.requireNonNull(getStringValueFromCell(row.getCell(columnIndex1), false)).trim() : "no org", donorAgencyCode, session, "donor", createMissingOrgs, orgGroupId, resolvedDonorOrgGroups, createMissingOrgGroups); List donors = new ArrayList<>(importDataModel.getDonor_organization()); List splits = splitAmounts(getNumericValueFromCell(cell).doubleValue(), donors.size()); for (int i = 0; i < donors.size(); i++) { @@ -181,8 +185,10 @@ public static List setFundingItemsForTxt(Map row, Map donors = new ArrayList<>(importDataModel.getDonor_organization()); List splits = splitAmounts(value != null ? value.doubleValue() : 0.0, donors.size()); for (int i = 0; i < donors.size(); i++) { @@ -218,7 +224,7 @@ public static String getStringValueFromCell(Cell cell, boolean nullable) { } return cell.getStringCellValue(); } catch (Exception e) { - logger.error("Error getting cell {} value: ", cell, e); + logger.error("Error getting cell {} value: ", cell); return nullable ? null : ""; } } @@ -234,7 +240,7 @@ public static Number getNumericValueFromCell(Cell cell) { } return cell.getNumericCellValue(); } catch (Exception e) { - logger.error("Error getting cell {} value: ", cell, e); + logger.error("Error getting cell {} value: ", cell); return 0; } } @@ -245,7 +251,7 @@ private static String getDateFromExcel(Row row, int columnIndex) { } - private static String extractDateFromStringCell(Cell cell) { + public static String extractDateFromStringCell(Cell cell) { if (cell == null) { return null; } @@ -339,6 +345,7 @@ private static String getFundingDate(String dateString) { List formatters = Arrays.asList( DateTimeFormatter.ofPattern("yyyy-MM-dd"), DateTimeFormatter.ofPattern("dd/MM/yyyy"), + DateTimeFormatter.ofPattern("d/M/yyyy"), DateTimeFormatter.ofPattern("MM/dd/yyyy"), DateTimeFormatter.ofPattern("MM-dd-yyyy"), DateTimeFormatter.ofPattern("yyyy/MM/dd"), @@ -371,9 +378,10 @@ private static String getFundingDate(String dateString) { } - private static String formatDateFromDateObject(String date) { + public static String formatDateFromDateObject(String date) { List formatters = Arrays.asList( new SimpleDateFormat("yyyy-MM-dd"), + new SimpleDateFormat("d/M/yyyy"), new SimpleDateFormat("dd/MM/yyyy"), new SimpleDateFormat("MM/dd/yyyy"), new SimpleDateFormat("MM-dd-yyyy"), @@ -416,6 +424,7 @@ private static String formatDateFromDateObject(String date) { public static boolean isCommonDateFormat(String dateString) { List dateFormats = Arrays.asList( "yyyy-MM-dd", + "d/M/yyyy", "dd-MM-yyyy", "MM-dd-yyyy", "MM/dd/yyyy", @@ -940,7 +949,7 @@ private static void ensureCreatedBySet(Map map, AmpActivityVersi } /** @return activity ID on success, null on skip or failure */ - public static Long importTheData(ImportDataModel importDataModel, Session session, ImportedProject importedProject, String componentName, String componentCode, Long responsibleOrgId, List fundings, Long existingActivityId, boolean validateActivities) throws JsonProcessingException { + public static Long importTheData(ImportDataModel importDataModel, Session session, ImportedProject importedProject, String componentName, String componentCode, Long responsibleOrgId, List fundings, Long existingActivityId, boolean validateActivities, boolean replaceExistingTransactions) throws JsonProcessingException { if (session == null || !session.isOpen()) { session = PersistenceManager.getRequestDBSession(); } @@ -956,6 +965,7 @@ public static Long importTheData(ImportDataModel importDataModel, Session sessio objectMapper.configure(ESCAPE_NON_ASCII, false); // Disable escaping of non-ASCII characters during serialization objectMapper.configure(MapperFeature.ACCEPT_CASE_INSENSITIVE_PROPERTIES, true); + pruneParentLocationsWhenChildPresent(importDataModel, session); normalizeLocationPercentages(importDataModel); Map map = objectMapper .convertValue(importDataModel, new TypeReference>() { @@ -996,10 +1006,14 @@ public static Long importTheData(ImportDataModel importDataModel, Session sessio } importDataModel.setProject_title(existing.getName() != null ? existing.getName() : ""); importDataModel.setProject_code(!Objects.equals(importDataModel.getProject_code(), "") ? importDataModel.getProject_code() : (existing.getProjectCode() != null ? existing.getProjectCode() : "")); - updateFundingOrgsAndSectorsWithAlreadyExisting(existing, importDataModel); + updateFundingOrgsAndSectorsWithAlreadyExisting(existing, importDataModel, replaceExistingTransactions); // Merge existing activity locations into payload so we only add (row + existing), never remove mergeExistingActivityLocationsIntoImport(existing, importDataModel); + // Preserve existing programs with their DB IDs so the API updates in-place; avoids both + // StaleStateException (delete+insert) and SizeValidator failures on re-validation + preserveExistingPrograms(existing, importDataModel); ensureImplementationLevelWhenHasLocations(importDataModel, session); + pruneParentLocationsWhenChildPresent(importDataModel, session); normalizeLocationPercentages(importDataModel); map = objectMapper .convertValue(importDataModel, new TypeReference>() { @@ -1008,11 +1022,6 @@ public static Long importTheData(ImportDataModel importDataModel, Session sessio map.entrySet().removeIf(entry -> entry.getValue() == null || "null".equals(String.valueOf(entry.getValue()))); map.remove("indicators"); // preserve existing indicators; we append in addIndicatorDataToActivity - // Do not replace programs; avoids StaleStateException when deleting AMP_ACTIVITY_PROGRAM rows - map.remove("national_plan_objective"); - map.remove("primary_programs"); - map.remove("secondary_programs"); - map.remove("tertiary_programs"); // Avoid triggering merge of contacts/documents that may reference deleted rows (ObjectNotFoundException) map.remove("activity_contacts"); map.remove("activityContacts"); @@ -1081,9 +1090,9 @@ public static Long importTheData(ImportDataModel importDataModel, Session sessio return activityId; } - private static void updateFundingOrgsAndSectorsWithAlreadyExisting(AmpActivityVersion ampActivityVersion, ImportDataModel importDataModel) { + private static void updateFundingOrgsAndSectorsWithAlreadyExisting(AmpActivityVersion ampActivityVersion, ImportDataModel importDataModel, boolean replaceExistingTransactions) { - if (ampActivityVersion.getFunding() != null) { + if (!replaceExistingTransactions && ampActivityVersion.getFunding() != null) { Hibernate.initialize(ampActivityVersion.getFunding()); Long adjType = getCategoryValue("adjustmentType", CategoryConstants.ADJUSTMENT_TYPE_KEY, ""); Long assType = getCategoryValue("assistanceType", CategoryConstants.TYPE_OF_ASSISTENCE_KEY, ""); @@ -1237,6 +1246,29 @@ private static void updateFundingOrgsAndSectorsWithAlreadyExisting(AmpActivityVe * (row locations + existing), never remove. Any existing activity location not already in importDataModel * is added. This avoids activity/update deleting locations (e.g. those referenced by indicator connections). */ + private static void preserveExistingPrograms(AmpActivityVersion existing, ImportDataModel importDataModel) { + if (existing == null || importDataModel == null) return; + Set actPrograms = existing.getActPrograms(); + if (actPrograms == null || actPrograms.isEmpty()) return; + Hibernate.initialize(actPrograms); + for (AmpActivityProgram ap : actPrograms) { + if (ap.getProgram() == null || ap.getProgram().getAmpThemeId() == null) continue; + Program p = new Program(); + p.setId(ap.getAmpActivityProgramId()); + p.setProgram(ap.getProgram().getAmpThemeId()); + String settingName = ap.getProgramSetting() != null ? ap.getProgramSetting().getName() : null; + if (ProgramUtil.NATIONAL_PLAN_OBJECTIVE.equals(settingName)) { + importDataModel.getNational_plan_objective().add(p); + } else if (ProgramUtil.PRIMARY_PROGRAM.equals(settingName)) { + importDataModel.getPrimary_programs().add(p); + } else if (ProgramUtil.SECONDARY_PROGRAM.equals(settingName)) { + importDataModel.getSecondary_programs().add(p); + } else if (ProgramUtil.TERTIARY_PROGRAM.equals(settingName)) { + importDataModel.getTertiary_programs().add(p); + } + } + } + private static void mergeExistingActivityLocationsIntoImport(AmpActivityVersion existing, ImportDataModel importDataModel) { if (existing == null || importDataModel == null) return; if (existing.getLocations() == null) return; @@ -1269,28 +1301,67 @@ private static void normalizeLocationPercentages(ImportDataModel importDataModel if (importDataModel == null || importDataModel.getLocations() == null || importDataModel.getLocations().isEmpty()) return; Set locs = importDataModel.getLocations(); - double sum = 0; + // Only consider locations that were found (location != null) + List foundLocations = new ArrayList<>(); for (Location loc : locs) { - Double pct = loc.getLocation_percentage(); - sum += (pct != null ? pct : 0); - } - if (sum <= 0) return; - if (Math.abs(sum - 100.0) < 0.001) return; // already 100 - List list = new ArrayList<>(locs); - double scale = 100.0 / sum; - double running = 0; - for (int i = 0; i < list.size(); i++) { - Location loc = list.get(i); - double v; - if (i == list.size() - 1) { - v = 100.0 - running; // last one gets remainder so total is exactly 100 - } else { - Double pct = loc.getLocation_percentage(); - v = (pct != null ? pct : 0) * scale; - running += v; + if (loc != null && loc.getLocation() != null) { + foundLocations.add(loc); } - loc.setLocation_percentage(v); } + if (foundLocations.isEmpty()) { + return; + } + + Map percentages = divide100(foundLocations.size()); + for (int i = 0; i < foundLocations.size(); i++) { + foundLocations.get(i).setLocation_percentage((double)percentages.get(i)); + } + importDataModel.setLocations(new HashSet<>(foundLocations)); + } + + /** + * Removes ancestor locations when both ancestor and descendant are present in payload. + * AMP validation rejects payloads that contain parent+child in the same locations collection. + */ + private static void pruneParentLocationsWhenChildPresent(ImportDataModel importDataModel, Session session) { + if (importDataModel == null || importDataModel.getLocations() == null || importDataModel.getLocations().size() < 2) { + return; + } + if (session == null || !session.isOpen()) { + session = PersistenceManager.getRequestDBSession(); + } + + Set selectedLocationIds = new HashSet<>(); + for (Location loc : importDataModel.getLocations()) { + if (loc != null && loc.getLocation() != null) { + selectedLocationIds.add(loc.getLocation()); + } + } + if (selectedLocationIds.size() < 2) { + return; + } + + Set ancestorsToRemove = new HashSet<>(); + for (Long locId : selectedLocationIds) { + AmpCategoryValueLocations current = session.get(AmpCategoryValueLocations.class, locId); + while (current != null && current.getParentLocation() != null) { + current = current.getParentLocation(); + if (current != null && current.getId() != null && selectedLocationIds.contains(current.getId())) { + ancestorsToRemove.add(current.getId()); + } + } + } + if (ancestorsToRemove.isEmpty()) { + return; + } + + Set filteredLocations = new HashSet<>(); + for (Location loc : importDataModel.getLocations()) { + if (loc == null || loc.getLocation() == null || !ancestorsToRemove.contains(loc.getLocation())) { + filteredLocations.add(loc); + } + } + importDataModel.setLocations(filteredLocations); } /** @@ -1299,7 +1370,6 @@ private static void normalizeLocationPercentages(ImportDataModel importDataModel private static void ensureImplementationLevelWhenHasLocations(ImportDataModel importDataModel, Session session) { if (importDataModel == null || importDataModel.getLocations() == null || importDataModel.getLocations().isEmpty()) return; - if (importDataModel.getImplementation_level() != null) return; updateImpLevels(importDataModel, session); } @@ -1546,7 +1616,7 @@ public static void updateSectors(ImportDataModel importDataModel, String name, S { name = subSector; } - for (String sectorName : splitMultiValues(name)) { + for (String sectorName : splitMultipleValues(name)) { updateSingleSector(importDataModel, sectorName, session, primary, createMissingSectors, importerSectorField); } } @@ -1670,11 +1740,15 @@ private static String getClassificationNameForImporterField(String importerSecto : AmpClassificationConfiguration.SECONDARY_CLASSIFICATION_CONFIGURATION_NAME; } - private static List splitMultiValues(String value) { + private static List splitMultipleValues(String value) { + return splitMultipleValues(value, "[;\\u061B\\uFF1B]"); + } + + private static List splitMultipleValues(String value, String separatorRegex) { if (value == null || value.trim().isEmpty()) return Collections.emptyList(); List result = new ArrayList<>(); // Support standard and locale-specific semicolons used in spreadsheet exports. - for (String part : value.split("[;\\u061B\\uFF1B]")) { + for (String part : value.split(separatorRegex)) { String trimmed = part.trim(); if (trimmed.length() >= 2 && trimmed.startsWith("\"") && trimmed.endsWith("\"")) { trimmed = trimmed.substring(1, trimmed.length() - 1).trim(); @@ -1684,24 +1758,21 @@ private static List splitMultiValues(String value) { return result; } - - private static List splitLocationNames(String locationNames) { - if (locationNames == null || locationNames.trim().isEmpty()) return Collections.emptyList(); - List result = new ArrayList<>(); - for (String part : locationNames.split("[,;]")) { - String trimmed = part.trim(); - if (!trimmed.isEmpty()) result.add(trimmed); - } - return result; - } - public static void updateLocations(ImportDataModel importDataModel, String locationNames, Session session) { logger.info("Updating locations"); if (locationNames == null || locationNames.trim().isEmpty()) return; - for (String locationName : splitLocationNames(locationNames)) { - if (ConstantsMap.containsKey("location_" + locationName)) { - Long location = ConstantsMap.get("location_" + locationName); - logger.info("In cache... location " + "location_" + locationName + ":" + location); + for (String locationName : splitMultipleValues(locationNames, "[,;\\u061B\\uFF1B]")) { + String normalizedLocationName = normalizeLocationNameForLookup(locationName); + if (normalizedLocationName.isEmpty()) { + continue; + } + Long cachedLocationId = ConstantsMap.get("location_" + normalizedLocationName); + if (cachedLocationId == null) { + cachedLocationId = ConstantsMap.get("location_" + locationName); + } + if (cachedLocationId != null) { + Long location = cachedLocationId; + logger.info("In cache... location " + "location_" + normalizedLocationName + ":" + location); importDataModel.getLocations().add(new Location(location, 100.00)); } else { @@ -1709,29 +1780,100 @@ public static void updateLocations(ImportDataModel importDataModel, String locat session = PersistenceManager.getRequestDBSession(); } - final String locationNameFinal = locationName; - session.doWork(connection -> { - String query = "SELECT acvl.id AS location_id FROM amp_category_value_location acvl WHERE LOWER(acvl.location_name) = LOWER(?)"; - try (PreparedStatement statement = connection.prepareStatement(query)) { - statement.setString(1, locationNameFinal); - - try (ResultSet resultSet = statement.executeQuery()) { - while (resultSet.next()) { - Long location = resultSet.getLong("location_id"); - logger.info("Location:" + location); - importDataModel.getLocations().add(new Location(location, 100.00)); - ConstantsMap.put("location_" + locationNameFinal, location); - } - } + Set resolvedLocationIds = resolveLocationIdsByName(session, normalizedLocationName); + if (resolvedLocationIds.isEmpty()) { + logger.warn("Location not found for importer value '{}'", normalizedLocationName); + continue; + } + for (Long location : resolvedLocationIds) { + logger.info("Location:" + location); + importDataModel.getLocations().add(new Location(location, 100.00)); + ConstantsMap.put("location_" + locationName, location); + ConstantsMap.put("location_" + normalizedLocationName, location); + } + } + } + updateImpLevels(importDataModel, session); + } - } catch (SQLException e) { - logger.error("Error getting locations", e); + private static String normalizeLocationNameForLookup(String rawLocationName) { + if (rawLocationName == null) { + return ""; + } + String normalized = rawLocationName + .replace('\u00A0', ' ') + .replace('\u202F', ' ') + .trim(); + normalized = normalized.replaceAll("\\.{2,}$", "").trim(); + return normalized; + } + + private static Set resolveLocationIdsByName(Session session, String locationName) { + Set result = new LinkedHashSet<>(); + if (locationName == null || locationName.trim().isEmpty()) { + return result; + } + + if (session == null || !session.isOpen()) { + session = PersistenceManager.getRequestDBSession(); + } + + final String exactName = locationName.trim(); + final String likeName = "%" + exactName + "%"; + + session.doWork(connection -> { + // 1) Exact match (fast path) + String exactQuery = "SELECT acvl.id AS location_id FROM amp_category_value_location acvl WHERE LOWER(acvl.location_name) = LOWER(?)"; + try (PreparedStatement statement = connection.prepareStatement(exactQuery)) { + statement.setString(1, exactName); + try (ResultSet rs = statement.executeQuery()) { + while (rs.next()) { + result.add(rs.getLong("location_id")); } + } + } catch (SQLException e) { + logger.error("Error resolving location by exact name", e); + } - }); + if (!result.isEmpty()) { + return; } - } - updateImpLevels(importDataModel, session); + + // 2) Normalized match (ignore punctuation/spacing differences, e.g. "Saint Louis" vs "Saint-Louis") + String normalizedQuery = "SELECT acvl.id AS location_id " + + "FROM amp_category_value_location acvl " + + "WHERE LOWER(regexp_replace(acvl.location_name, '[^[:alnum:]]', '', 'g')) " + + "= LOWER(regexp_replace(?, '[^[:alnum:]]', '', 'g'))"; + try (PreparedStatement statement = connection.prepareStatement(normalizedQuery)) { + statement.setString(1, exactName); + try (ResultSet rs = statement.executeQuery()) { + while (rs.next()) { + result.add(rs.getLong("location_id")); + } + } + } catch (SQLException e) { + logger.error("Error resolving location by normalized name", e); + } + + if (!result.isEmpty()) { + return; + } + + // 3) Loose contains match as last fallback. + String containsQuery = "SELECT acvl.id AS location_id FROM amp_category_value_location acvl WHERE LOWER(acvl.location_name) LIKE LOWER(?)"; + try (PreparedStatement statement = connection.prepareStatement(containsQuery)) { + statement.setString(1, likeName); + try (ResultSet rs = statement.executeQuery()) { + while (rs.next()) { + result.add(rs.getLong("location_id")); + } + } + } catch (SQLException e) { + logger.error("Error resolving location by contains search", e); + } + }); + + return result; } public static void applyDefaultLocation(ImportDataModel importDataModel, Long locationId, Session session) { @@ -1806,34 +1948,109 @@ private static AmpActivityLocation getOrAddActivityLocationForName(AmpActivityVe public static void updateImpLevels(ImportDataModel importDataModel, Session session) { + if (importDataModel == null) { + return; + } + if (session == null || !session.isOpen()) { + session = PersistenceManager.getRequestDBSession(); + } + + Long resolved = resolveImplementationLevelForLocations(importDataModel, session); + if (resolved != null) { + importDataModel.setImplementation_level(resolved); + return; + } + + // Fallback when no locations are present: keep previous default behavior (National). if (ConstantsMap.containsKey("implementation_level_")) { Long implementationLevel = ConstantsMap.get("implementation_level_"); - logger.info("In cache... imp level "+"implementation_level:"+implementationLevel); + logger.info("In cache... imp level " + "implementation_level:" + implementationLevel); importDataModel.setImplementation_level(implementationLevel); - }else { - if (!session.isOpen()) { - session = PersistenceManager.getRequestDBSession(); - } - + } else { session.doWork(connection -> { - String query2 = "SELECT acv.id as implementation_level FROM amp_category_value acv JOIN amp_category_class acc ON acv.amp_category_class_id=acc.id WHERE LOWER(acv.category_value)=? AND LOWER(acc.keyname)=?"; - try (PreparedStatement statement = connection.prepareStatement(query2)) { - statement.setString(1, "national"); - statement.setString(2, "implementation_level"); + String query2 = "SELECT acv.id as implementation_level FROM amp_category_value acv JOIN amp_category_class acc ON acv.amp_category_class_id=acc.id WHERE LOWER(acv.category_value)=? AND LOWER(acc.keyname)=?"; + try (PreparedStatement statement = connection.prepareStatement(query2)) { + statement.setString(1, "national"); + statement.setString(2, "implementation_level"); - try (ResultSet resultSet = statement.executeQuery()) { - while (resultSet.next()) { - Long implementationLevel = resultSet.getLong("implementation_level"); - logger.info("Imp level:" + implementationLevel); - importDataModel.setImplementation_level(implementationLevel); - ConstantsMap.put("implementation_level_", implementationLevel); + try (ResultSet resultSet = statement.executeQuery()) { + while (resultSet.next()) { + Long implementationLevel = resultSet.getLong("implementation_level"); + logger.info("Imp level:" + implementationLevel); + importDataModel.setImplementation_level(implementationLevel); + ConstantsMap.put("implementation_level_", implementationLevel); + } } + + } catch (SQLException e) { + logger.error("Error getting imp levels", e); } + }); + } + } - } catch (SQLException e) { - logger.error("Error getting imp levels", e); - }}); + private static Long resolveImplementationLevelForLocations(ImportDataModel importDataModel, Session session) { + if (importDataModel.getLocations() == null || importDataModel.getLocations().isEmpty()) { + return null; } + + Set commonAllowedLevels = null; + for (Location loc : importDataModel.getLocations()) { + if (loc == null || loc.getLocation() == null) { + continue; + } + Set allowedForLocation = getAllowedImplementationLevelsForLocation(loc.getLocation(), session); + if (allowedForLocation.isEmpty()) { + continue; + } + if (commonAllowedLevels == null) { + commonAllowedLevels = new HashSet<>(allowedForLocation); + } else { + commonAllowedLevels.retainAll(allowedForLocation); + } + } + + if (commonAllowedLevels == null || commonAllowedLevels.isEmpty()) { + return null; + } + + Long existingImplementationLevel = importDataModel.getImplementation_level(); + if (existingImplementationLevel != null && commonAllowedLevels.contains(existingImplementationLevel)) { + return existingImplementationLevel; + } + + // Prefer common hard-coded levels if available to keep behavior predictable. + Long national = CategoryConstants.IMPLEMENTATION_LEVEL_NATIONAL.getIdInDatabase(); + if (national != null && commonAllowedLevels.contains(national)) { + return national; + } + Long regional = CategoryConstants.IMPLEMENTATION_LEVEL_REGIONAL.getIdInDatabase(); + if (regional != null && commonAllowedLevels.contains(regional)) { + return regional; + } + Long international = CategoryConstants.IMPLEMENTATION_LEVEL_INTERNATIONAL.getIdInDatabase(); + if (international != null && commonAllowedLevels.contains(international)) { + return international; + } + + return commonAllowedLevels.iterator().next(); + } + + private static Set getAllowedImplementationLevelsForLocation(Long locationId, Session session) { + Set allowed = new HashSet<>(); + if (locationId == null) { + return allowed; + } + AmpCategoryValueLocations location = session.get(AmpCategoryValueLocations.class, locationId); + if (location == null || location.getParentCategoryValue() == null || location.getParentCategoryValue().getUsedValues() == null) { + return allowed; + } + for (AmpCategoryValue level : location.getParentCategoryValue().getUsedValues()) { + if (level != null && level.getId() != null) { + allowed.add(level.getId()); + } + } + return allowed; } private static void createSector(ImportDataModel importDataModel, boolean primary, Long ampSectorId) { @@ -1904,13 +2121,18 @@ public static Long updateOrgs(ImportDataModel importDataModel, String name, Stri public static Long updateOrgs(ImportDataModel importDataModel, String name, String code, Session session, String type, boolean createMissingOrgs, Long orgGroupId, String importedOrgGroupName, boolean createMissingOrgGroups) { - List names = splitMultiValues(name); - List codes = splitMultiValues(code); + List names = splitMultipleValues(name); + List codes = splitMultipleValues(code); + List importedOrgGroups = splitMultipleValues(importedOrgGroupName); + String fallbackOrgGroup = importedOrgGroups.isEmpty() ? null : importedOrgGroups.get(importedOrgGroups.size() - 1); Long lastOrgId = null; for (int i = 0; i < names.size(); i++) { String singleName = names.get(i); String singleCode = i < codes.size() ? codes.get(i) : null; - lastOrgId = updateSingleOrg(importDataModel, singleName, singleCode, session, type, createMissingOrgs, orgGroupId, importedOrgGroupName, createMissingOrgGroups); + String singleOrgGroupName = importedOrgGroups.isEmpty() + ? null + : (i < importedOrgGroups.size() ? importedOrgGroups.get(i) : fallbackOrgGroup); + lastOrgId = updateSingleOrg(importDataModel, singleName, singleCode, session, type, createMissingOrgs, orgGroupId, singleOrgGroupName, createMissingOrgGroups); } return lastOrgId; } @@ -2284,7 +2506,7 @@ public static void addIndicatorDataToActivity(Long activityId, Row row, Sheet sh return; } indicatorName = indicatorName.trim(); - List locationNames = splitLocationNames(locationNamesStr); + List locationNames = splitMultipleValues(locationNamesStr, "[,;\\u061B\\uFF1B]"); logger.info("addIndicatorDataToActivity: indicator='{}', locations(count={}): {}", indicatorName, locationNames.size(), locationNames); if (locationNames.isEmpty()) { logger.debug("addIndicatorDataToActivity: no location names after split"); @@ -2701,6 +2923,11 @@ private static List getValuesByType(Set va * divided evenly among all activity programs (including the one just added). */ public static void addProgramToActivityIfMissing(AmpActivityVersion activity, AmpTheme program, Session session) { + addProgramToActivityIfMissing(activity, program, session, null); + } + + public static void addProgramToActivityIfMissing(AmpActivityVersion activity, AmpTheme program, Session session, + AmpActivityProgramSettings programSetting) { if (activity == null || program == null) return; Set actPrograms = activity.getActPrograms(); if (actPrograms == null) { @@ -2710,34 +2937,154 @@ public static void addProgramToActivityIfMissing(AmpActivityVersion activity, Am for (AmpActivityProgram ap : actPrograms) { if (ap.getProgram() != null && program.getAmpThemeId() != null && program.getAmpThemeId().equals(ap.getProgram().getAmpThemeId())) { + if (ap.getProgramSetting() == null && programSetting != null) { + ap.setProgramSetting(programSetting); + session.saveOrUpdate(ap); + } return; } } AmpActivityProgram activityProgram = new AmpActivityProgram(); activityProgram.setActivity(activity); activityProgram.setProgram(program); + activityProgram.setProgramSetting(programSetting); activityProgram.setProgramPercentage(100f); actPrograms.add(activityProgram); - session.save(activityProgram); + } - // If Program Percentage field is enabled, distribute 100% evenly among all programs - boolean percentageEnabled = false; + public static void addProgramsToActivity(Long activityId, String rawProgramNames, String rowProgramClassification, + String fallbackProgramClassification, boolean createMissingPrograms, + Session session) { + if (activityId == null || rawProgramNames == null || rawProgramNames.trim().isEmpty()) { + return; + } + String resolvedClassification = (rowProgramClassification != null && !rowProgramClassification.trim().isEmpty()) + ? rowProgramClassification.trim() + : (fallbackProgramClassification != null ? fallbackProgramClassification.trim() : null); + if (resolvedClassification == null || resolvedClassification.isEmpty()) { + logger.info("Skipping programs because program classification could not be resolved"); + return; + } + AmpActivityVersion activity = session.get(AmpActivityVersion.class, activityId); + if (activity == null) { + logger.info("Skipping programs because activity {} was not found", activityId); + return; + } + + AmpActivityProgramSettings programSetting = resolveProgramSettingByName(resolvedClassification); + if (programSetting == null) { + logger.warn("Program classification '{}' not found; skipping programs for activity {}", + resolvedClassification, activityId); + return; + } + + for (String programName : splitMultipleValues(rawProgramNames)) { + AmpTheme program = createMissingPrograms + ? getOrCreateProgramByName(programName, resolvedClassification, session) + : getProgramByNameAndClassification(programName, resolvedClassification, session); + if (program == null) { + logger.info("Program '{}' not found and createMissingPrograms is disabled; skipping", programName); + continue; + } + addProgramToActivityIfMissing(activity, program, session, programSetting); + } + + logger.info("Adding percentages to programs"); + if (activity.getActPrograms() != null && !activity.getActPrograms().isEmpty()) { + // Group by programSetting (classification) + List group = new ArrayList<>(); + for (AmpActivityProgram ap : activity.getActPrograms()) { + if (ap.getProgramSetting() != null && ap.getProgramSetting().equals(programSetting)) { + group.add(ap); + } + } + if (!group.isEmpty()) { + Map percentages = divide100(group.size()); + for (int i = 0; i < group.size(); i++) { + group.get(i).setProgramPercentage(percentages.get(i)); + session.saveOrUpdate(group.get(i)); + } + } + } + session.flush(); + } + + private static AmpTheme getProgramByNameAndClassification(String programName, String classification, + Session session) { + if (programName == null || programName.trim().isEmpty()) { + return null; + } + AmpActivityProgramSettings setting = resolveProgramSettingByName(classification); + if (setting == null) { + return ProgramUtil.getTheme(programName.trim()); + } + String hql = "SELECT t FROM " + AmpTheme.class.getName() + " t WHERE LOWER(t.name) = LOWER(:name)"; + Query query; + if (setting.getDefaultHierarchy() != null && setting.getDefaultHierarchy().getAmpThemeId() != null) { + hql += " AND t.parentThemeId.ampThemeId = :parentId"; + query = session.createQuery(hql); + query.setParameter("parentId", setting.getDefaultHierarchy().getAmpThemeId()); + } else { + query = session.createQuery(hql); + } + query.setParameter("name", programName.trim(), StringType.INSTANCE); + query.setMaxResults(1); + AmpTheme theme = (AmpTheme) query.uniqueResult(); + return theme != null ? theme : ProgramUtil.getTheme(programName.trim()); + } + + public static AmpTheme getOrCreateProgramByName(String programName, String classification, Session session) { + AmpTheme existing = getProgramByNameAndClassification(programName, classification, session); + if (existing != null) { + return existing; + } + if (programName == null || programName.trim().isEmpty()) { + return null; + } try { - percentageEnabled = FeaturesUtil.isVisibleField(ArConstants.PROGRAM_PERCENTAGE); + AmpTheme newTheme = new AmpTheme(); + String trimmedName = programName.trim(); + newTheme.setName(trimmedName); + String code = trimmedName.replaceAll("[^a-zA-Z0-9_-]", "_").replaceAll("_+", "_").trim(); + if (code.length() > 45) code = code.substring(0, 45); + newTheme.setThemeCode("IMP_" + code + "_" + System.currentTimeMillis()); + newTheme.setIndlevel(0); + AmpActivityProgramSettings setting = resolveProgramSettingByName(classification); + if (setting != null && setting.getDefaultHierarchy() != null) { + newTheme.setParentThemeId(setting.getDefaultHierarchy()); + Integer rootLevel = setting.getDefaultHierarchy().getIndlevel(); + newTheme.setIndlevel(rootLevel != null ? rootLevel + 1 : 1); + } else { + newTheme.setParentThemeId(null); + } + // Set typeCategoryValue from classification's defaultHierarchy's category value + if (setting != null && setting.getDefaultHierarchy() != null && setting.getDefaultHierarchy().getTypeCategoryValue() != null) { + newTheme.setTypeCategoryValue(setting.getDefaultHierarchy().getTypeCategoryValue()); + } else { + throw new IllegalStateException("Cannot create program: classification '" + classification + "' does not resolve to a valid typeCategoryValue (required for DB constraint)"); + } + session.save(newTheme); + session.flush(); + return newTheme; } catch (Exception e) { - // No request/session (e.g. batch) – skip percentage redistribution - logger.error("Could not determine if Program Percentage field is enabled; skipping percentage redistribution", e); + logger.warn("Failed to create program '{}' for classification '{}'", programName, classification, e); + return null; } - if (percentageEnabled && !actPrograms.isEmpty()) { - List list = new ArrayList<>(actPrograms); - int n = list.size(); - Map percentages = divide100(n); - for (int i = 0; i < n; i++) { - list.get(i).setProgramPercentage(percentages.get(i)); - } + } + + private static AmpActivityProgramSettings resolveProgramSettingByName(String classification) { + if (classification == null || classification.trim().isEmpty()) { + return null; + } + try { + return ProgramUtil.getAmpActivityProgramSettings(classification); + } catch (Exception e) { + logger.warn("Could not resolve program setting '{}'", classification, e); + return null; } } + /** * Returns the program (theme) by name, or creates a new root-level program if it does not exist. * @param programName program name (must be non-empty) diff --git a/amp/src/main/java/org/digijava/module/aim/form/DataImporterForm.java b/amp/src/main/java/org/digijava/module/aim/form/DataImporterForm.java index c1b46a0869f..6958f300b0d 100644 --- a/amp/src/main/java/org/digijava/module/aim/form/DataImporterForm.java +++ b/amp/src/main/java/org/digijava/module/aim/form/DataImporterForm.java @@ -17,9 +17,12 @@ public class DataImporterForm extends ActionForm { private boolean createMissingOrgs; private boolean createMissingSectors; private boolean createMissingOrgGroups; + private boolean createMissingPrograms; + private boolean replaceExistingTransactions; private Long orgGroupId; private Long defaultActivityStatusId; private Long defaultLocationId; + private String defaultProgramClassification; private boolean validateActivities; private boolean addDisbursementForCommitment; @@ -79,6 +82,22 @@ public void setCreateMissingOrgGroups(boolean createMissingOrgGroups) { this.createMissingOrgGroups = createMissingOrgGroups; } + public boolean isCreateMissingPrograms() { + return createMissingPrograms; + } + + public void setCreateMissingPrograms(boolean createMissingPrograms) { + this.createMissingPrograms = createMissingPrograms; + } + + public boolean isReplaceExistingTransactions() { + return replaceExistingTransactions; + } + + public void setReplaceExistingTransactions(boolean replaceExistingTransactions) { + this.replaceExistingTransactions = replaceExistingTransactions; + } + public Long getOrgGroupId() { return orgGroupId; } @@ -103,6 +122,14 @@ public void setDefaultLocationId(Long defaultLocationId) { this.defaultLocationId = defaultLocationId; } + public String getDefaultProgramClassification() { + return defaultProgramClassification; + } + + public void setDefaultProgramClassification(String defaultProgramClassification) { + this.defaultProgramClassification = defaultProgramClassification; + } + public Set getFileHeaders() { return fileHeaders; } diff --git a/amp/src/main/webapp/WEB-INF/jsp/aim/view/dataImporter.jsp b/amp/src/main/webapp/WEB-INF/jsp/aim/view/dataImporter.jsp index 0e5d1be6acb..16ade5b9122 100644 --- a/amp/src/main/webapp/WEB-INF/jsp/aim/view/dataImporter.jsp +++ b/amp/src/main/webapp/WEB-INF/jsp/aim/view/dataImporter.jsp @@ -77,6 +77,13 @@ }); } + if ($('#defaultProgramClassification').length) { + applySelect2('#defaultProgramClassification', { + width: '100%', + placeholder: 'Select fallback program classification' + }); + } + if ($('#data-sheet').length) { applySelect2('#data-sheet', { width: '100%', @@ -122,6 +129,28 @@ $('#defaultLocationId').val(defaultLocationId).trigger('change.select2'); } } + toggleProgramClassificationFallback(); + } + + function hasMappedProgramField() { + return $('#selected-pairs-table-body tr').filter(function() { + return $(this).attr('data-selected-field') === 'Program Name'; + }).length > 0; + } + + function hasMappedProgramClassificationField() { + return $('#selected-pairs-table-body tr').filter(function() { + return $(this).attr('data-selected-field') === 'Program Classification'; + }).length > 0; + } + + function toggleProgramClassificationFallback() { + var hasMappings = $('#selected-pairs-table-body tr').length > 0; + var showFallback = hasMappings && hasMappedProgramField() && !hasMappedProgramClassificationField(); + $('#program-classification-fallback').toggle(showFallback); + if (!showFallback) { + $('#defaultProgramClassification').val('').trigger('change.select2'); + } } function repopulateSelectedPairs(updatedMap) { @@ -199,20 +228,20 @@ fileType = 'excel'; } if (fileType === "csv") { - $('#select-file-label').html("Select csv file"); + $('#select-file-label').html("Select CSV File"); $('#data-file').attr("accept", ".csv"); $('#template-file').attr("accept", ".csv"); $('#separator-div').hide(); $('#data-sheet-choice-div').hide(); } else if(fileType==="text") { - $('#select-file-label').html("Select text file"); + $('#select-file-label').html("Select Text File"); $('#data-file').attr("accept", ".txt"); $('#template-file').attr("accept", ".txt"); $('#separator-div').show(); $('#data-sheet-choice-div').hide(); } else if(fileType==="excel") { - $('#select-file-label').html("Select excel file"); + $('#select-file-label').html("Select Excel File"); $('#data-file').attr("accept", ".xls,.xlsx"); $('#template-file').attr("accept", ".xls,.xlsx"); $('#separator-div').hide(); @@ -240,11 +269,11 @@ formData.append('action', 'getDataFileSheets'); formData.append('fileType', $('#file-type').val()); var $select = $('#data-sheet'); - $select.prop('disabled', true).empty().append(''); + $select.prop('disabled', true).empty().append(''); fetch("${pageContext.request.contextPath}/aim/dataImporter.do", { method: "POST", body: formData }) .then(function(r) { return r.json(); }) .then(function(names) { - $select.empty().append(''); + $select.empty().append(''); if (Array.isArray(names)) { names.forEach(function(name) { $select.append($('').attr('value', name).text(name)); @@ -253,7 +282,7 @@ } }) .catch(function() { - $select.empty().append('').prop('disabled', false); + $select.empty().append('').prop('disabled', false); alert("Could not load sheets from file."); }); }); @@ -366,12 +395,34 @@ tbody.appendChild(row); } - function hasMappedOrgGroupField() { + function hasMappedField(fieldName) { return $('#selected-pairs-table-body tr').filter(function() { - return $(this).attr('data-selected-field') === 'Organization Group'; + return $(this).attr('data-selected-field') === fieldName; }).length > 0; } + function hasAnyMappedOrgMissingGroupMapping() { + if (hasMappedField('Organization Group')) { + return false; + } + var roleMappings = [ + ['Donor Agency', 'Donor Organization Group'], + ['Responsible Organization', 'Responsible Organization Group'], + ['Beneficiary Agency', 'Beneficiary Agency Group'], + ['Executing Agency', 'Executing Agency Group'], + ['Implementing Agency', 'Implementing Agency Group'], + ['Contracting Agency', 'Contracting Agency Group'] + ]; + for (var i = 0; i < roleMappings.length; i++) { + var orgField = roleMappings[i][0]; + var groupField = roleMappings[i][1]; + if (hasMappedField(orgField) && !hasMappedField(groupField)) { + return true; + } + } + return false; + } + function uploadTemplateFile() { $('#existing-config').val('0'); var formData = new FormData(); @@ -475,11 +526,16 @@ var createMissingOrgs = $('#createMissingOrgs').prop('checked'); var createMissingSectors = $('#createMissingSectors').prop('checked'); var createMissingOrgGroups = $('#createMissingOrgGroups').prop('checked'); + var createMissingPrograms = $('#createMissingPrograms').prop('checked'); + var replaceExistingTransactions = $('#replaceExistingTransactions').prop('checked'); var orgGroupId = $('#orgGroupId').val(); var defaultActivityStatusId = $('#defaultActivityStatusId').val(); var defaultLocationId = $('#defaultLocationId').val() || $('#defaultLocationId').attr('data-default-location-id') || ''; - var hasOrgGroupMapping = hasMappedOrgGroupField(); + var defaultProgramClassification = $('#defaultProgramClassification').val(); + var anyMappedOrgMissingGroupMapping = hasAnyMappedOrgMissingGroupMapping(); var hasProjectLocationMapping = hasMappedProjectLocationField(); + var hasProgramMapping = hasMappedProgramField(); + var hasProgramClassificationMapping = hasMappedProgramClassificationField(); console.log("Internal", internal); console.log("Skip existing", skipExisting); console.log("Skip records without transactions", skipRecordsWithoutTransactions); @@ -488,17 +544,24 @@ console.log("Create missing orgs", createMissingOrgs); console.log("Create missing sectors", createMissingSectors); console.log("Create missing org groups", createMissingOrgGroups); + console.log("Create missing programs", createMissingPrograms); + console.log("Replace existing transactions", replaceExistingTransactions); console.log("Org group id", orgGroupId); console.log("Default activity status id", defaultActivityStatusId); console.log("Default location id", defaultLocationId); - if (createMissingOrgs && !orgGroupId && !hasOrgGroupMapping && !createMissingOrgGroups) { - alert("Please select an Organization Group, map the Organization Group column, or enable organization group creation for newly created organizations."); + console.log("Default program classification", defaultProgramClassification); + if (createMissingOrgs && !orgGroupId && !createMissingOrgGroups && anyMappedOrgMissingGroupMapping) { + alert("Please select a fallback Organization Group (or enable organization-group creation) when any mapped organization field has no corresponding group mapping."); return; } if (!hasProjectLocationMapping && !defaultLocationId) { alert("Please select a fallback location when no Project Location column is mapped."); return; } + if (hasProgramMapping && !hasProgramClassificationMapping && !defaultProgramClassification) { + alert("Please select a default program classification when no Program Classification column is mapped."); + return; + } var dataSeparator = $('#data-separator').val(); var currentConfigName = $('#current-config-name').val(); var existingConfig = (currentConfigName && currentConfigName.trim() !== '') ? currentConfigName.trim() : $('#existing-config').val(); @@ -524,6 +587,8 @@ formData.append('createMissingOrgs', createMissingOrgs); formData.append('createMissingSectors', createMissingSectors); formData.append('createMissingOrgGroups', createMissingOrgGroups); + formData.append('createMissingPrograms', createMissingPrograms); + formData.append('replaceExistingTransactions', replaceExistingTransactions); if (createMissingOrgs && orgGroupId) { formData.append('orgGroupId', orgGroupId); } @@ -533,6 +598,9 @@ if (defaultLocationId) { formData.append('defaultLocationId', defaultLocationId); } + if (defaultProgramClassification) { + formData.append('defaultProgramClassification', defaultProgramClassification); + } formData.append('action',"uploadDataFile"); formData.append('fileType', fileType); formData.append('dataSeparator', dataSeparator); @@ -588,10 +656,10 @@ --accent-warm: #8a6f56; --surface-muted: #f0f2f4; --row-alt: #f8f9fa; - --shadow: 0 8px 20px rgba(25, 39, 52, 0.06); - --radius-lg: 20px; - --radius-md: 14px; - --radius-sm: 10px; + --shadow: 0 4px 12px rgba(25, 39, 52, 0.05); + --radius-lg: 14px; + --radius-md: 10px; + --radius-sm: 6px; } html { @@ -601,8 +669,8 @@ body { margin: 0; font-family: Arial, Helvetica, sans-serif; - font-size: 14px; - line-height: 1.5; + font-size: 12px; + line-height: 1.4; color: var(--text-strong); background: var(--page-bg); } @@ -617,7 +685,7 @@ .importer-page { max-width: 1180px; margin: 0 auto; - padding: 40px 20px 56px; + padding: 22px 16px 34px; } .hero-card, @@ -630,52 +698,53 @@ } .hero-card { - border-radius: 24px; - padding: 36px; - margin-bottom: 24px; + border-radius: 16px; + padding: 20px; + margin-bottom: 14px; background: var(--panel-bg); } .hero-card h1, .workspace-card h2, .upload-stage h3 { - margin: 0 0 10px; - font-size: 1.5rem; + margin: 0 0 8px; + font-size: 18px; font-weight: 700; - letter-spacing: 0.02em; + letter-spacing: 0; } .hero-card p, .section-copy, .helper-note { color: var(--text-soft); - line-height: 1.6; + line-height: 1.45; margin: 0; + font-size: 12px; } .panel-grid { display: grid; grid-template-columns: repeat(auto-fit, minmax(280px, 1fr)); - gap: 20px; - margin-bottom: 24px; + gap: 14px; + margin-bottom: 14px; } .panel-card, .workspace-card, .upload-stage { border-radius: var(--radius-lg); - padding: 24px; + padding: 16px; } .workspace-card { - margin-top: 24px; + margin-top: 14px; } .section-label { display: inline-block; - margin-bottom: 8px; - font-size: 13px; - letter-spacing: 0.18em; + margin-bottom: 6px; + font-size: 11px; + letter-spacing: 0.12em; text-transform: uppercase; color: var(--accent); font-weight: 700; @@ -683,8 +752,8 @@ label { display: inline-block; - margin-bottom: 6px; - font-size: 14px; + margin-bottom: 4px; + font-size: 12px; font-weight: 700; color: var(--text-strong); } @@ -695,16 +764,16 @@ width: 100%; max-width: 100%; box-sizing: border-box; - padding: 12px 14px; + padding: 8px 10px; border-radius: var(--radius-sm); border: 1px solid rgba(22, 53, 67, 0.18); background: #fff; color: var(--text-strong); - font-size: 14px; + font-size: 12px; } input[type="file"] { - padding: 10px 12px; + padding: 7px 9px; background: var(--surface-muted); } @@ -712,12 +781,12 @@ button { border: 1px solid #506673; border-radius: 999px; - padding: 12px 18px; + padding: 8px 14px; font-weight: 700; cursor: pointer; color: #fff; background: var(--accent); - font-size: 14px; + font-size: 12px; box-shadow: none; transition: background-color 0.18s ease, border-color 0.18s ease, color 0.18s ease; } @@ -736,14 +805,14 @@ .inline-field, .toggle-grid, .sheet-choice-card { - margin-top: 16px; + margin-top: 12px; } .toggle-grid { display: grid; - gap: 10px; - margin-top: 18px; - padding: 18px; + gap: 8px; + margin-top: 14px; + padding: 12px; border-radius: var(--radius-md); background: var(--surface-muted); border: 1px solid var(--panel-border); @@ -776,16 +845,16 @@ table th, table td { text-align: left; - padding: 14px 16px; - font-size: 14px; + padding: 10px 12px; + font-size: 12px; border-bottom: 1px solid rgba(22, 53, 67, 0.08); } table th { background: #eef1f3; color: var(--text-strong); - font-size: 13px; - letter-spacing: 0.12em; + font-size: 11px; + letter-spacing: 0.08em; text-transform: uppercase; } @@ -800,25 +869,25 @@ .mapping-toolbar { display: grid; grid-template-columns: repeat(auto-fit, minmax(240px, 1fr)); - gap: 16px; + gap: 12px; align-items: end; - margin-bottom: 18px; + margin-bottom: 12px; } .mapping-actions { display: flex; - gap: 12px; + gap: 8px; flex-wrap: wrap; - margin-top: 16px; + margin-top: 10px; } .upload-stage { - margin-top: 22px; + margin-top: 14px; } #config-empty-note { - margin-top: 18px; - padding: 16px 18px; + margin-top: 12px; + padding: 10px 12px; border-radius: var(--radius-md); background: #f3efe9; color: #6a5a48; @@ -826,10 +895,10 @@ } .select2-container--default .select2-selection--single { - height: 46px; + height: 34px; border-radius: var(--radius-sm); border: 1px solid rgba(22, 53, 67, 0.18); - padding: 8px 12px; + padding: 3px 8px; display: flex; align-items: center; background: #fff; @@ -837,20 +906,20 @@ .select2-container--default .select2-selection--single .select2-selection__rendered { color: var(--text-strong); - font-size: 14px; - line-height: 28px; + font-size: 12px; + line-height: 24px; padding-left: 0; } .select2-dropdown { - border-radius: 14px; + border-radius: 8px; border-color: rgba(22, 53, 67, 0.18); overflow: hidden; } .select2-search__field { - border-radius: 10px; - padding: 8px 10px; + border-radius: 6px; + padding: 6px 8px; } @media (max-width: 768px) { @@ -875,14 +944,14 @@
- Import Workspace + Import Workspace

Import Data

Prepare a template, map source columns to AMP entity fields, and upload your data only after the configuration table is ready.

- Step 1 + Step 1

Choose File Settings

Pick the source format and optionally load an existing configuration before uploading a template.

@@ -909,7 +978,7 @@
- Step 2 + Step 2

Load Configuration

Resume from a saved mapping or start with a fresh template upload.

@@ -939,7 +1008,7 @@ - Step 3 + Step 3

Build Your Mapping

Search entity fields, add mappings, and review the configuration table before uploading the actual data file.

@@ -984,7 +1053,7 @@
+ +
diff --git a/amp/src/main/webapp/WEB-INF/jsp/aim/view/viewImportProgress.jsp b/amp/src/main/webapp/WEB-INF/jsp/aim/view/viewImportProgress.jsp index 128fce476c8..bf33a88b7f7 100644 --- a/amp/src/main/webapp/WEB-INF/jsp/aim/view/viewImportProgress.jsp +++ b/amp/src/main/webapp/WEB-INF/jsp/aim/view/viewImportProgress.jsp @@ -19,7 +19,7 @@ --danger: #7a5555; --warning: #7d6a53; --row-alt: #f8f9fa; - --shadow: 0 8px 20px rgba(25, 39, 52, 0.06); + --shadow: 0 4px 12px rgba(25, 39, 52, 0.05); } html { @@ -29,8 +29,8 @@ body { margin: 0; font-family: Arial, Helvetica, sans-serif; - font-size: 14px; - line-height: 1.5; + font-size: 12px; + line-height: 1.4; color: var(--text-strong); background: var(--page-bg); } @@ -45,7 +45,7 @@ .progress-page { max-width: 1220px; margin: 0 auto; - padding: 40px 20px 56px; + padding: 22px 16px 34px; } .hero-card, @@ -54,27 +54,27 @@ background: var(--panel-bg); border: 1px solid var(--panel-border); box-shadow: var(--shadow); - border-radius: 28px; + border-radius: 14px; } .hero-card { - padding: 34px; - margin-bottom: 22px; + padding: 20px; + margin-bottom: 14px; background: var(--panel-bg); } .hero-card h1, .panel-card h2, .records-card h2 { - margin: 0 0 10px; - font-size: 1.5rem; + margin: 0 0 8px; + font-size: 18px; } .section-label { display: inline-block; - margin-bottom: 8px; - font-size: 13px; - letter-spacing: 0.18em; + margin-bottom: 6px; + font-size: 11px; + letter-spacing: 0.12em; text-transform: uppercase; color: var(--accent); font-weight: 700; @@ -83,13 +83,14 @@ .section-copy { margin: 0; color: var(--text-soft); - line-height: 1.6; + line-height: 1.45; + font-size: 12px; } .panel-card, .records-card { - padding: 24px; - margin-bottom: 22px; + padding: 16px; + margin-bottom: 14px; } table { @@ -104,15 +105,15 @@ td, th { text-align: left; - padding: 14px 16px; - font-size: 14px; + padding: 10px 12px; + font-size: 12px; border-bottom: 1px solid rgba(22, 53, 67, 0.08); } th { background: #eef1f3; - font-size: 13px; - letter-spacing: 0.12em; + font-size: 11px; + letter-spacing: 0.08em; text-transform: uppercase; } @@ -129,8 +130,8 @@ .nav-action-btn { border: 1px solid #506673; border-radius: 999px; - padding: 10px 16px; - font-size: 14px; + padding: 8px 14px; + font-size: 12px; font-weight: 700; cursor: pointer; color: #fff; @@ -139,13 +140,13 @@ } .view-more-btn { - margin-top: 8px; - padding: 8px 14px; - font-size: 13px; + margin-top: 6px; + padding: 6px 12px; + font-size: 11px; } .hero-actions { - margin-top: 18px; + margin-top: 12px; display: flex; justify-content: flex-end; } @@ -153,16 +154,17 @@ .status-summary { display: grid; grid-template-columns: repeat(auto-fit, minmax(180px, 1fr)); - gap: 14px; - margin: 18px 0; + gap: 10px; + margin: 12px 0; } .status-pill { - padding: 14px 16px; - border-radius: 18px; + padding: 10px 12px; + border-radius: 10px; font-weight: 700; background: #f1f3f5; border: 1px solid var(--panel-border); + font-size: 12px; } .status-pill.all { @@ -184,17 +186,17 @@ .filter-div { display: flex; flex-wrap: wrap; - gap: 12px; + gap: 8px; align-items: center; - margin-bottom: 18px; - padding: 14px 16px; - border-radius: 18px; + margin-bottom: 12px; + padding: 10px 12px; + border-radius: 10px; background: #f1f3f5; border: 1px solid var(--panel-border); } .filter-div label { - font-size: 14px; + font-size: 12px; font-weight: 700; } @@ -209,20 +211,20 @@ .dataTables_wrapper .dataTables_filter input, .dataTables_wrapper .dataTables_length select { border: 1px solid rgba(22, 53, 67, 0.18); - border-radius: 10px; - padding: 6px 10px; + border-radius: 6px; + padding: 5px 8px; background: #fff; - font-size: 14px; + font-size: 12px; } .uploads-filter-bar { display: flex; flex-wrap: wrap; - gap: 12px; + gap: 8px; align-items: end; - margin: 16px 0 18px; - padding: 14px 16px; - border-radius: 18px; + margin: 10px 0 12px; + padding: 10px 12px; + border-radius: 10px; background: #f1f3f5; border: 1px solid var(--panel-border); } @@ -230,21 +232,21 @@ .uploads-filter-field { display: flex; flex-direction: column; - gap: 6px; - min-width: 180px; + gap: 4px; + min-width: 160px; } .uploads-filter-field label { - font-size: 14px; + font-size: 12px; font-weight: 700; } .uploads-filter-field input { border: 1px solid rgba(22, 53, 67, 0.18); - border-radius: 10px; - padding: 8px 10px; + border-radius: 6px; + padding: 6px 8px; background: #fff; - font-size: 14px; + font-size: 12px; } .truncated-response { @@ -263,8 +265,8 @@ .hero-card, .panel-card, .records-card { - padding: 18px; - border-radius: 20px; + padding: 14px; + border-radius: 12px; } .filter-div { @@ -279,13 +281,30 @@