FINERACT-2421: Allow model recalculation (in case of version mismatch)

fineract-e2e-tests-core/src/test/java/org/apache/fineract/test/stepdef/loan/LoanStepDef.java

@@ -5926,6 +5926,13 @@ public void inLoanTransactionsTheThTransactionOfOnHasRelationshipWithTypeREPLAYE
         assertEquals(Integer.valueOf(numberOfRelations), relationshipOptional.size(), "Missed relationship for transaction");
     }
 
+    @When("Call Internal API to remove progressive loan model by loan Id")
+    public void callInternalAPIToRemoveProgressiveLoanModelByLoanId() {
+        final PostLoansResponse loanCreateResponse = testContext().get(TestContextKey.LOAN_CREATE_RESPONSE);
+        final long loanId = loanCreateResponse.getLoanId();
+        ok(() -> fineractClient.progressiveLoan().deleteModel(loanId));
+    }
+
     public static AdvancedPaymentData editPaymentAllocationFutureInstallment(String transactionType, String futureInstallmentAllocationRule,
             List<PaymentAllocationOrder> paymentAllocationOrder) {
         AdvancedPaymentData advancedPaymentData = new AdvancedPaymentData();

fineract-e2e-tests-runner/src/test/resources/features/LoanAccrualTransaction.feature

@@ -1619,6 +1619,7 @@ Feature: LoanAccrualTransaction
       | 01 January 2024  | Disbursement     | 100.0  | 0.0       | 0.0      | 0.0  | 0.0       | 100.0        | false    | false    |
 #    --- Early repayment with 17.01 EUR on 15 Jan ---
     When Admin sets the business date to "15 January 2024"
+    When Call Internal API to remove progressive loan model by loan Id
     When Admin makes "MERCHANT_ISSUED_REFUND" transaction with "AUTOPAY" payment type on "15 January 2024" with 17.01 EUR transaction amount
     Then Loan Repayment schedule has 6 periods, with the following data for periods:
       | Nr | Days | Date             | Paid date       | Balance of loan | Principal due | Interest | Fees | Penalties | Due   | Paid  | In advance | Late | Outstanding |
@@ -1660,6 +1661,7 @@ Feature: LoanAccrualTransaction
       | 01 May 2024      | Accrual Activity          | 0.48   |  0.0      | 0.48     | 0.0  | 0.0       |  0.0         | false    | false    |
       | 31 May 2024      | Accrual                   | 1.87   |  0.0      | 1.87     | 0.0  | 0.0       |  0.0         | false    | false    |
     When Admin sets the business date to "02 June 2024"
+    When Call Internal API to remove progressive loan model by loan Id
     And Admin runs inline COB job for Loan
     Then Loan Repayment schedule has 6 periods, with the following data for periods:
       | Nr | Days | Date             | Paid date       | Balance of loan | Principal due | Interest | Fees | Penalties | Due   | Paid  | In advance | Late | Outstanding |

fineract-loan/src/main/java/org/apache/fineract/portfolio/loanaccount/domain/LoanRepository.java

@@ -279,4 +279,10 @@ List<Loan> findLoansForAddAccrual(@Param("accountingType") AccountingRuleType ac
     @Query(FIND_ALL_LOANS_BEHIND_ON_DISBURSEMENT_DATE)
     List<COBIdAndLastClosedBusinessDate> findAllLoansBehindOnDisbursementDate(@Param("cobBusinessDate") LocalDate cobBusinessDate,
             @Param("loanIds") List<Long> loanIds, @Param("loanStatuses") Collection<LoanStatus> loanStatuses);
+
+    @Query("SELECT CASE WHEN COUNT(l) > 0 THEN TRUE ELSE FALSE END FROM Loan l WHERE l.id = :loanId and l.loanRepaymentScheduleDetail.loanScheduleType = org.apache.fineract.portfolio.loanaccount.loanschedule.domain.LoanScheduleType.PROGRESSIVE")
+    Boolean isProgressiveLoan(@Param("loanId") Long loanId);
+
+    @Query("SELECT CASE WHEN COUNT(l) > 0 THEN TRUE ELSE FALSE END FROM Loan l WHERE l.id = :loanId and l.loanStatus in :allowedLoanStatuses")
+    Boolean isLoanInAllowedStatus(@Param("loanId") Long loanId, @Param("allowedLoanStatuses") List<LoanStatus> allowedLoanStatuses);
 }

fineract-loan/src/main/java/org/apache/fineract/portfolio/loanaccount/domain/LoanRepositoryWrapper.java

@@ -313,4 +313,8 @@ public boolean existsByLoanId(Long loanId) {
     public boolean isEnabledCapitalizedIncome(Long loanId) {
         return repository.isEnabledCapitalizedIncome(loanId);
     }
+
+    public boolean isLoanInAllowedStatus(Long loanId, List<LoanStatus> allowedLoanStatuses) {
+        return repository.isLoanInAllowedStatus(loanId, allowedLoanStatuses);
+    }
 }

fineract-progressive-loan-embeddable-schedule-generator/src/main/java/org/apache/fineract/portfolio/loanaccount/loanschedule/domain/EmbeddableProgressiveLoanScheduleGenerator.java

@@ -83,5 +83,10 @@ public boolean hasValidModelForDate(Long loanId, LocalDate targetDate) {
         public Optional<ProgressiveLoanInterestScheduleModel> getSavedModel(Loan loan, LocalDate businessDate) {
             return Optional.empty();
         }
+
+        @Override
+        public Long removeByLoanId(Long loanId) {
+            return 0L;
+        }
     }
 }

fineract-progressive-loan/src/main/java/org/apache/fineract/portfolio/loanaccount/domain/ProgressiveLoanModel.java

@@ -52,4 +52,7 @@ public class ProgressiveLoanModel extends AbstractPersistableCustom<Long> {
     @Column(name = "last_modified_on_utc", nullable = false)
     private OffsetDateTime lastModifiedDate;
 
+    @Column(name = "json_model_version", nullable = false)
+    private String jsonModelVersion;
+
 }

fineract-progressive-loan/src/main/java/org/apache/fineract/portfolio/loanaccount/repository/ProgressiveLoanModelRepository.java

@@ -23,11 +23,18 @@
 import org.apache.fineract.portfolio.loanaccount.domain.ProgressiveLoanModel;
 import org.springframework.data.jpa.repository.JpaRepository;
 import org.springframework.data.jpa.repository.JpaSpecificationExecutor;
+import org.springframework.data.jpa.repository.Query;
+import org.springframework.data.repository.query.Param;
 
 public interface ProgressiveLoanModelRepository
         extends JpaSpecificationExecutor<ProgressiveLoanModel>, JpaRepository<ProgressiveLoanModel, Long> {
 
     Optional<ProgressiveLoanModel> findOneByLoanId(Long loanId);
 
     Optional<ProgressiveLoanModel> findOneByLoan(Loan loan);
+
+    Long removeByLoanId(Long loanId);
+
+    @Query("SELECT CASE WHEN COUNT(plm) > 0 THEN TRUE ELSE FALSE END FROM ProgressiveLoanModel plm WHERE plm.loan.id = :loanId AND plm.jsonModelVersion = :modelVersion")
+    Boolean hasValidModel(@Param("loanId") Long loanId, @Param("modelVersion") String modelVersion);
 }

fineract-progressive-loan/src/main/java/org/apache/fineract/portfolio/loanaccount/service/InterestScheduleModelRepositoryWrapper.java

@@ -41,4 +41,6 @@ Optional<ProgressiveLoanInterestScheduleModel> readProgressiveLoanInterestSchedu
     boolean hasValidModelForDate(Long loanId, LocalDate targetDate);
 
     Optional<ProgressiveLoanInterestScheduleModel> getSavedModel(Loan loan, LocalDate businessDate);
+
+    Long removeByLoanId(Long loanId);
 }

fineract-progressive-loan/src/main/java/org/apache/fineract/portfolio/loanaccount/service/InterestScheduleModelRepositoryWrapperImpl.java

@@ -61,11 +61,13 @@ public ProgressiveLoanInterestScheduleModel writeInterestScheduleModel(Loan loan
             ProgressiveLoanModel progressiveLoanModel = loanModelRepository.findOneByLoanId(loan.getId()).orElseGet(() -> {
                 ProgressiveLoanModel plm = new ProgressiveLoanModel();
                 plm.setLoan(loan);
+                plm.setJsonModelVersion(ProgressiveLoanInterestScheduleModel.getModelVersion());
                 return plm;
             });
             progressiveLoanModel.setBusinessDate(ThreadLocalContextUtil.getBusinessDate());
             progressiveLoanModel.setLastModifiedDate(DateUtils.getAuditOffsetDateTime());
             progressiveLoanModel.setJsonModel(jsonModel);
+            progressiveLoanModel.setJsonModelVersion(ProgressiveLoanInterestScheduleModel.getModelVersion());
             loanModelRepository.save(progressiveLoanModel);
         });
         return model;
@@ -133,4 +135,9 @@ public Optional<ProgressiveLoanInterestScheduleModel> readProgressiveLoanInteres
                 .map(jsonModel -> progressiveLoanInterestScheduleModelParserService.fromJson(jsonModel, detail,
                         MoneyHelper.getMathContext(), installmentAmountInMultipliesOf)); //
     }
+
+    @Override
+    public Long removeByLoanId(Long loanId) {
+        return loanModelRepository.removeByLoanId(loanId);
+    }
 }

fineract-progressive-loan/src/main/java/org/apache/fineract/portfolio/loanaccount/service/InternalProgressiveLoanApiResource.java

@@ -22,6 +22,7 @@
 import io.swagger.v3.oas.annotations.Operation;
 import io.swagger.v3.oas.annotations.Parameter;
 import io.swagger.v3.oas.annotations.tags.Tag;
+import jakarta.ws.rs.DELETE;
 import jakarta.ws.rs.GET;
 import jakarta.ws.rs.POST;
 import jakarta.ws.rs.Path;
@@ -99,4 +100,13 @@ public ProgressiveLoanInterestScheduleModel updateModel(@PathParam("loanId") @Pa
 
         return writePlatformService.writeInterestScheduleModel(loan, model);
     }
+
+    @DELETE
+    @Path("{loanId}/model")
+    @Produces({ MediaType.APPLICATION_JSON })
+    @Operation(summary = "Delete ProgressiveLoanInterestScheduleModel By Loan ID", description = "DO NOT USE THIS IN PRODUCTION!")
+    @Transactional
+    public Long deleteModel(@PathParam("loanId") @Parameter(description = "loanId") long loanId) {
+        return writePlatformService.removeByLoanId(loanId);
+    }
 }

fineract-progressive-loan/src/main/java/org/apache/fineract/portfolio/loanaccount/service/ProgressiveLoanModelProcessingService.java

@@ -0,0 +1,61 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.fineract.portfolio.loanaccount.service;
+
+import java.util.List;
+import lombok.RequiredArgsConstructor;
+import org.apache.fineract.infrastructure.core.service.ThreadLocalContextUtil;
+import org.apache.fineract.portfolio.loanaccount.domain.Loan;
+import org.apache.fineract.portfolio.loanaccount.domain.LoanRepositoryWrapper;
+import org.apache.fineract.portfolio.loanaccount.domain.LoanStatus;
+import org.apache.fineract.portfolio.loanaccount.repository.ProgressiveLoanModelRepository;
+import org.apache.fineract.portfolio.loanproduct.calc.data.ProgressiveLoanInterestScheduleModel;
+import org.springframework.stereotype.Component;
+import org.springframework.transaction.annotation.Propagation;
+import org.springframework.transaction.annotation.Transactional;
+
+@Component
+@RequiredArgsConstructor
+public class ProgressiveLoanModelProcessingService {
+
+    private static final List<LoanStatus> allowedLoanStatuses = List.of(LoanStatus.ACTIVE, LoanStatus.CLOSED_OBLIGATIONS_MET,
+            LoanStatus.CLOSED_WRITTEN_OFF, LoanStatus.OVERPAID);
+    private final LoanRepositoryWrapper loanRepositoryWrapper;
+    private final ProgressiveLoanModelRecalculationService modelProcessingService;
+    private final InterestScheduleModelRepositoryWrapper modelRepositoryWrapper;
+    private final ProgressiveLoanModelRepository progressiveLoanModelRepository;
+
+    public boolean hasValidModel(Long loanId, String modelVersion) {
+        return progressiveLoanModelRepository.hasValidModel(loanId, modelVersion);
+    }
+
+    @Transactional(propagation = Propagation.REQUIRES_NEW)
+    public void recalculateModelAndSave(Long loanId) {
+        Loan loan = loanRepositoryWrapper.findOneWithNotFoundDetection(loanId);
+        ProgressiveLoanInterestScheduleModel recalculatedModel = modelProcessingService.getRecalculatedModel(loan.getId(),
+                ThreadLocalContextUtil.getBusinessDate());
+        if (recalculatedModel != null) {
+            modelRepositoryWrapper.writeInterestScheduleModel(loan, recalculatedModel);
+        }
+    }
+
+    public boolean allowedLoanStatuses(Long loanId) {
+        return loanRepositoryWrapper.isLoanInAllowedStatus(loanId, allowedLoanStatuses);
+    }
+}

fineract-progressive-loan/src/main/java/org/apache/fineract/portfolio/loanaccount/service/ProgressiveLoanModelRecalculationService.java

@@ -0,0 +1,61 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.fineract.portfolio.loanaccount.service;
+
+import java.time.LocalDate;
+import java.util.List;
+import lombok.RequiredArgsConstructor;
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.fineract.portfolio.loanaccount.domain.ChangedTransactionDetail;
+import org.apache.fineract.portfolio.loanaccount.domain.Loan;
+import org.apache.fineract.portfolio.loanaccount.domain.LoanRepaymentScheduleTransactionProcessorFactory;
+import org.apache.fineract.portfolio.loanaccount.domain.LoanRepositoryWrapper;
+import org.apache.fineract.portfolio.loanaccount.domain.LoanTransaction;
+import org.apache.fineract.portfolio.loanaccount.domain.LoanTransactionRepository;
+import org.apache.fineract.portfolio.loanaccount.domain.transactionprocessor.LoanRepaymentScheduleTransactionProcessor;
+import org.apache.fineract.portfolio.loanaccount.domain.transactionprocessor.impl.AdvancedPaymentScheduleTransactionProcessor;
+import org.apache.fineract.portfolio.loanproduct.calc.data.ProgressiveLoanInterestScheduleModel;
+import org.springframework.stereotype.Component;
+import org.springframework.transaction.annotation.Propagation;
+import org.springframework.transaction.annotation.Transactional;
+
+@Component
+@RequiredArgsConstructor
+public class ProgressiveLoanModelRecalculationService {
+
+    private final LoanRepaymentScheduleTransactionProcessorFactory transactionProcessorFactory;
+    private final LoanTransactionRepository loanTransactionRepository;
+    private final LoanRepositoryWrapper loanRepositoryWrapper;
+
+    @Transactional(readOnly = true, propagation = Propagation.REQUIRES_NEW)
+    public ProgressiveLoanInterestScheduleModel getRecalculatedModel(Long loanId, LocalDate tillDate) {
+        Loan loan = loanRepositoryWrapper.findOneWithNotFoundDetection(loanId);
+        LoanRepaymentScheduleTransactionProcessor transactionProcessor = transactionProcessorFactory
+                .determineProcessor(loan.getTransactionProcessingStrategyCode());
+        if (transactionProcessor instanceof AdvancedPaymentScheduleTransactionProcessor advancedPaymentScheduleTransactionProcessor) {
+            List<LoanTransaction> loanTransactions = loanTransactionRepository.findNonReversedTransactionsForReprocessingByLoan(loan);
+            Pair<ChangedTransactionDetail, ProgressiveLoanInterestScheduleModel> result = advancedPaymentScheduleTransactionProcessor
+                    .reprocessProgressiveLoanTransactions(loan.getDisbursementDate(), tillDate, loanTransactions, loan.getCurrency(),
+                            loan.getRepaymentScheduleInstallments(), loan.getActiveCharges());
+            return result.getRight();
+        }
+        return null;
+    }
+
+}

fineract-progressive-loan/src/main/java/org/apache/fineract/portfolio/loanproduct/calc/data/ProgressiveLoanInterestScheduleModel.java

@@ -54,6 +54,7 @@
 @AllArgsConstructor
 public class ProgressiveLoanInterestScheduleModel {
 
+    private static final String modelVersion = "2";
     private final List<RepaymentPeriod> repaymentPeriods;
     private final TreeSet<InterestRate> interestRates;
     @JsonExclude
@@ -419,4 +420,8 @@ public Function<Long, LocalDate> resolveRepaymentPeriodLengthGeneratorFunction(f
         };
     }
 
+    public static String getModelVersion() {
+        return modelVersion;
+    }
+
 }

fineract-progressive-loan/src/main/resources/db/changelog/tenant/module/progressiveloan/module-changelog-master.xml

@@ -25,4 +25,5 @@
     <!-- Sequence is starting from 5000 to make it easier to move existing liquibase changesets here -->
     <include file="parts/5001_create_progressive_loan_model.xml" relativeToChangelogFile="true"/>
     <include file="parts/5002_add_contract_termination_transaction.xml" relativeToChangelogFile="true"/>
+    <include file="parts/5003_add_model_version.xml" relativeToChangelogFile="true"/>
 </databaseChangeLog>

fineract-progressive-loan/src/main/resources/db/changelog/tenant/module/progressiveloan/parts/5003_add_model_version.xml

@@ -0,0 +1,35 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+    Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements. See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership. The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License. You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied. See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+-->
+<databaseChangeLog
+        xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
+        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+        xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog
+                      http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-4.31.xsd"
+        objectQuotingStrategy="QUOTE_ONLY_RESERVED_WORDS">
+    <changeSet author="fineract" id="1">
+        <addColumn tableName="m_loan_progressive_model">
+            <column name="json_model_version" type="VARCHAR(100)" defaultValue="1">
+                <constraints nullable="false"/>
+            </column>
+        </addColumn>
+    </changeSet>
+</databaseChangeLog>

fineract-provider/src/main/java/org/apache/fineract/cob/loan/AbstractLoanItemProcessor.java

@@ -32,6 +32,8 @@
 import org.apache.fineract.cob.COBBusinessStepService;
 import org.apache.fineract.cob.data.BusinessStepNameAndOrder;
 import org.apache.fineract.portfolio.loanaccount.domain.Loan;
+import org.apache.fineract.portfolio.loanaccount.service.ProgressiveLoanModelProcessingService;
+import org.apache.fineract.portfolio.loanproduct.calc.data.ProgressiveLoanInterestScheduleModel;
 import org.springframework.batch.core.ExitStatus;
 import org.springframework.batch.core.StepExecution;
 import org.springframework.batch.core.annotation.AfterStep;
@@ -44,25 +46,34 @@
 public abstract class AbstractLoanItemProcessor implements ItemProcessor<Loan, Loan> {
 
     private final COBBusinessStepService cobBusinessStepService;
+    private final ProgressiveLoanModelProcessingService progressiveLoanModelProcessingService;
 
     @Setter(AccessLevel.PROTECTED)
     private ExecutionContext executionContext;
     private LocalDate businessDate;
 
     @SuppressWarnings({ "unchecked" })
     @Override
-    public Loan process(@NonNull Loan item) throws Exception {
+    public Loan process(@NonNull Loan loan) throws Exception {
+        if (needToRebuildModel(loan)) {
+            progressiveLoanModelProcessingService.recalculateModelAndSave(loan.getId());
+        }
         Set<BusinessStepNameAndOrder> businessSteps = (Set<BusinessStepNameAndOrder>) executionContext.get(LoanCOBConstant.BUSINESS_STEPS);
         if (businessSteps == null) {
             throw new IllegalStateException("No business steps found in the execution context");
         }
         TreeMap<Long, String> businessStepMap = getBusinessStepMap(businessSteps);
 
-        Loan alreadyProcessedLoan = cobBusinessStepService.run(businessStepMap, item);
+        Loan alreadyProcessedLoan = cobBusinessStepService.run(businessStepMap, loan);
         alreadyProcessedLoan.setLastClosedBusinessDate(businessDate);
         return alreadyProcessedLoan;
     }
 
+    private boolean needToRebuildModel(Loan loan) {
+        return loan.isProgressiveSchedule() && !progressiveLoanModelProcessingService.hasValidModel(loan.getId(),
+                ProgressiveLoanInterestScheduleModel.getModelVersion());
+    }
+
     private TreeMap<Long, String> getBusinessStepMap(Set<BusinessStepNameAndOrder> businessSteps) {
         Map<Long, String> businessStepMap = businessSteps.stream()
                 .collect(Collectors.toMap(BusinessStepNameAndOrder::getStepOrder, BusinessStepNameAndOrder::getStepName));