feat: add statement meta info

janz93 · janz93 · commit d23e25918d2e · 2025-01-22T01:10:37.000+01:00
enrich the statement with the info if the statement in not yet confirmed by the bank (expected)
or if a statement is reversal of a previous transaction
diff --git a/Gemfile b/Gemfile
@@ -6,7 +6,7 @@ ruby "3.3.2"
 
 gem "activesupport"
 gem "camt_parser", git: "https://github.com/railslove/camt_parser.git"
-gem "cmxl", git: "https://github.com/railslove/cmxl"
+gem "cmxl"
 gem "epics"
 gem "faraday"
 gem "grape"
diff --git a/Gemfile.lock b/Gemfile.lock
@@ -5,13 +5,6 @@ GIT
     camt_parser (1.0.2)
       nokogiri
 
-GIT
-  remote: https://github.com/railslove/cmxl
-  revision: b92aea3de958426119d76d043a886233a313f43f
-  specs:
-    cmxl (1.5.0)
-      rchardet
-
 GIT
   remote: https://github.com/railslove/king_dtaus_ruby_3.git
   revision: 28b077aa6b2cf5590e322d0d207236adccf2d543
@@ -45,6 +38,8 @@ GEM
     base64 (0.2.0)
     bigdecimal (3.1.8)
     builder (3.3.0)
+    cmxl (2.0)
+      rchardet
     coderay (1.1.3)
     concurrent-ruby (1.3.4)
     connection_pool (2.4.1)
@@ -160,7 +155,7 @@ GEM
       rack (>= 1.0, < 3)
     rainbow (3.1.1)
     rake (13.2.1)
-    rchardet (1.8.0)
+    rchardet (1.9.0)
     redis (4.8.1)
     regexp_parser (2.9.2)
     rest-client (2.1.0)
@@ -261,7 +256,7 @@ DEPENDENCIES
   airborne
   barnes
   camt_parser!
-  cmxl!
+  cmxl
   database_cleaner-sequel
   dotenv
   epics
diff --git a/box/apis/v2/documentation.yml b/box/apis/v2/documentation.yml
@@ -84,7 +84,7 @@ Statement Created:
   account_id,
   statement: {
     id, account (iban), name, bic, iban, type, amount (cents), date,
-    remittance_information
+    remittance_information, expected, reversal
   }
 }
 ```
diff --git a/box/business_processes/import_statements.rb b/box/business_processes/import_statements.rb
@@ -112,7 +112,9 @@ def self.statement_attributes_from_bank_transaction(transaction, bank_statement)
           mref: transaction.respond_to?(:mref) ? transaction.mref : transaction.sepa["MREF"],
           svwz: transaction.respond_to?(:svwz) ? transaction.svwz : transaction.sepa["SVWZ"],
           tx_id: transaction.try(:transaction_id),
-          creditor_identifier: transaction.respond_to?(:creditor_identifier) ? transaction.creditor_identifier : transaction.sepa["CRED"]
+          creditor_identifier: transaction.respond_to?(:creditor_identifier) ? transaction.creditor_identifier : transaction.sepa["CRED"],
+          expected: transaction.expected?,
+          reversal: transaction.reversal?
         }
       end
     end
diff --git a/box/entities/statement.rb b/box/entities/statement.rb
@@ -11,7 +11,8 @@ class Statement < Grape::Entity
       expose :name
       expose :bic
       expose :iban
-      expose :type
+      expose :expected, documentation: {type: "Boolean", desc: "Expected statement are not yet confirmed"}
+      expose :reversal, documentation: {type: "Boolean", desc: "Reversal of a previous transaction"}
       expose :amount, documentation: {type: "Integer", desc: "Amount in cents"}
       expose :date
       expose(:remittance_information, documentation: {type: "String", desc: "Wire transfer reference"}) { |statement| statement[:svwz] || statement[:information] }
diff --git a/box/entities/v2/transaction.rb b/box/entities/v2/transaction.rb
@@ -12,6 +12,8 @@ class Transaction < Grape::Entity
         expose :iban
         expose :bic
         expose :type
+        expose :expected
+        expose :reversal
         expose :amount, as: "amount_in_cents"
         expose :date, as: "executed_on"
         expose(:settled_at) { |trx| trx.settled ? trx.date : nil }
diff --git a/box/models/statement.rb b/box/models/statement.rb
@@ -81,7 +81,14 @@ def credit?
     end
 
     def debit?
-      debit
+    end
+
+    def reversal?
+      reversal
+    end
+
+    def expected?
+      !!expected
     end
 
     def type
diff --git a/db/migrations/20250121163300_add_meta_data_to_statement.rb b/db/migrations/20250121163300_add_meta_data_to_statement.rb
@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+Sequel.migration do
+  change do
+    add_column :statements, :expected, :boolean
+    add_column :statements, :reversal, :boolean
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
@@ -117,6 +117,8 @@
       column :settled, "boolean", :default=>true
       column :sha, "text"
       column :tx_id, "text"
+      column :expected, "boolean"
+      column :reversal, "boolean"
       
       index [:sha], :name=>:statements_sha2_index, :unique=>true
       index [:sha_bak], :name=>:statements_sha_key, :unique=>true
diff --git a/spec/apis/v2/transactions_spec.rb b/spec/apis/v2/transactions_spec.rb
@@ -16,6 +16,8 @@ module Box
         amount_in_cents: :integer,
         executed_on: :date,
         type: :string,
+        expected: :boolean,
+        reversal: :boolean,
         reference: :string,
         end_to_end_reference: :string,
         settled_at: :string
diff --git a/spec/business_processes/import_statement_spec.rb b/spec/business_processes/import_statement_spec.rb
@@ -265,6 +265,19 @@ def exec_link_action
             expect(described_class.create_statement(mt940_bank_statement, transaction)).to be_truthy
           end
         end
+
+        context "importing vmk transaction that is expected" do
+          it "marks statements as expected" do
+            vmk_data = File.read("spec/fixtures/master_diesel_vmk_data.txt")
+            bank_statement = ImportBankStatement.from_mt940(vmk_data, account)
+            bank_transactions = described_class.parse_bank_statement(bank_statement)
+            transaction = bank_transactions.first
+
+            described_class.create_statement(bank_statement, transaction)
+
+            expect(Statement.first).to be_expected
+          end
+        end
       end
     end
   end
diff --git a/spec/fabricators/statement_fabricator.rb b/spec/fabricators/statement_fabricator.rb
@@ -44,4 +44,6 @@
   bic { STATEMENT_BICS.sample }
   eref { "eref-#{Fabricate.sequence(:eref)}" }
   svwz { Faker::Lorem.sentence }
+  expected { [true, false].sample }
+  reversal { [true, false].sample }
 end
diff --git a/spec/fixtures/abnamro.mt940 b/spec/fixtures/abnamro.mt940
@@ -0,0 +1,41 @@
+ABNANL2A
+940
+ABNANL2A
+:20:ABN AMRO BANK NV
+:25:517852257
+:28:19321/1
+:60F:C110522EUR3236,28
+:61:1105240524D9,N192NONREF
+:86:GIRO   428428 KPN - DIGITENNE    BETALINGSKENM.  000000042188659
+5314606715                       BETREFT FACTUUR D.D. 20-05-2011
+INCL. 1,44 BTW
+:61:1105210523D11,59N426NONREF
+:86:BEA   NR:XXX1234   21.05.11/12.54 DIRCKIII FIL2500 KATWIJK,PAS999
+:61:1105230523D11,63N426NONREF
+:86:BEA   NR:XXX1234   23.05.11/09.08 DIGROS FIL1015 KATWIJK Z,PAS999
+:61:1105220523D11,8N426NONREF
+:86:BEA   NR:XXX1234   22.05.11/14.25 MC DONALDS A44 LEIDEN,PAS999
+:61:1105210523D13,45N426NONREF
+:86:BEA   NR:XXX1234   21.05.11/12.09 PRINCE FIL. 55 KATWIJK Z,PAS999
+:61:1105210523D15,49N426NONREF
+:86:BEA   NR:XXX1234   21.05.11/12.55 DIRX FIL6017 KATWIJK ZH ,PAS999
+
+:61:1105210523D107,N426NONREF
+:86:BEA   NR:XXX1234   21.05.11/12.04 HANS ANDERS OPT./056 KAT,PAS999
+:61:1105220523D141,48N426NONREF
+:86:BEA   NR:XXX1234   22.05.11/13.45 MYCOM DEN HAAG  S-GRAVEN,PAS999
+:62F:C110523EUR876,84
+-
+ABNANL2A
+940
+ABNANL2A
+:20:ABN AMRO BANK NV
+:25:517852257
+:28:19322/1
+:60F:C110523EUR2876,84
+:61:1105240524D9,49N426NONREF
+:86:BEA   NR:XXX1234   24.05.11/09.18 PETS PLACE KATWIJK KATWI,PAS999
+:61:1105240524D15,N426NONREF
+:86:52.89.39.882 MYCOM DEN HAAG  S-GRAVEN,PAS999
+:62F:C110524EUR1849,75
+-
diff --git a/spec/fixtures/handelbank.mt940 b/spec/fixtures/handelbank.mt940
@@ -0,0 +1,16 @@
+ä4:
+:20:5566778899100112
+:25:10020030/1234567
+:28C:188/1
+:60F:C130928SEK0,
+:62F:C130930SEK0,
+:64:C130930SEK0,
+-å
+ä4:
+:20:5566778899100169
+:25:10020030/1234567
+:28C:188/1
+:60F:C130928SEK0,
+:62F:C130930SEK0,
+:64:C130930SEK0,
+-å
diff --git a/spec/fixtures/master_diesel_vmk_data.txt b/spec/fixtures/master_diesel_vmk_data.txt
@@ -0,0 +1,58 @@
+RLNWATWWAMS  00001
+942 01
+ELBA-INT    
+:20:20241212080001
+:25://10020030/1234567
+:28C:24241/001
+:34F:EUR0,
+:13D:2412120800+0100
+:61:2412121212ED15162,57NDDTNONREF//950
+:86:999Kreuzmayr GmbH
+Siehe Avis vom 12.12.24
+:61:2412121212ED6136,95NDDTNONREF//950
+:86:999Moser Wurst GmbH
+1161235,1161236
+:61:2412121212ED2079,12NDDT0094324106//950
+:86:999JULIUS MEINL Austria GmbH
+0094324106
+:61:2412121212ED1218,84NDDTNONREF//950
+:86:999unik GmbH
+2200243181, EUR 1 218,84
+:61:2412121212ED223,NSUEEUIEV1IV//950
+:86:999ELBA-AUFTRAG EUIEV1IV
+:61:2412121212ED190,08NSUEEUIEV1IM//950
+:86:999ELBA-AUFTRAG EUIEV1IM
+:61:2412121212EC739,5NTRF186454//950
+:86:999G�ls�m Cakmak
+186454
+:61:2412121212EC1000,NTRFNONREF//950
+:86:999IKO-DACH GmbH
+:61:2412121212EC1836,25NTRFNONREF//950
+:86:999Int. Transporte A. Fasching Anton F
+176471
+:61:2412121212EC2000,NTRF80235//950
+:86:999Valentin Klizan
+80235
+:61:2412121212EC2233,5NTRF176417//950
+:86:999Miroslav Beganovic
+176417
+:61:2412121212EC14409,79NTRFNONREF//950
+:86:999GLOBAL PAYMENTS S.R.O.
+KOMB /REF 4850003118 111224 /B 14673,47 /D 219,73 /U 43,95 /G 269
+master DIESEL Tankstelle
+:90D:6EUR25010,56
+:90C:6EUR22219,04
+-
+RLNWATWWAMS  00001
+942 01
+ELBA-INT    
+:20:20241212100006
+:25://10020030/1234567
+:28C:24241/001
+:34F:EUR0,
+:13D:2412121000+0100
+:61:2412121212EC43374,55NTRFNONREF//950
+:86:999Staack Pooltankstellen GmbH
+RNR 176080 23781,44 0,00 RNR 175584 19593,11 0,00
+:90C:1EUR43374,55
+  
diff --git a/spec/fixtures/single_with_headers.mt940 b/spec/fixtures/single_with_headers.mt940
@@ -0,0 +1,16 @@
+{1:D02AASDISLNETAXXXXXXXXXXXXX}
+{2:E623XXXXXXXXAXXXN}
+{4:
+:20:1234567
+:21:9876543210
+:25:10020030/1234567
+:28C:5/1
+:60F:C160314EUR2187,95
+:61:0211011102DR800,NSTONONREF//55555
+:86:008?00DAUERAUFTRAG?100599?20Miete November?3010020030?31234567?32MUELLER?34339
+:61:0211021102CR3000,NTRFNONREF//55555
+:86:051?00UEBERWEISUNG?100599?20Gehalt Oktober?21Firma
+Mustermann GmbH?3050060400?310847564700?32MUELLER?34339
+:62F:C160315EUR4387,95
+:86:Some random data
+-}
diff --git a/spec/jobs/fetch_upcoming_statements_spec.rb b/spec/jobs/fetch_upcoming_statements_spec.rb
@@ -41,10 +41,11 @@ module Jobs
 
       describe ".fetch_for_account" do
         let(:client) { double("Epics Client") }
+        let(:import_file) { File.read("spec/fixtures/multiple-transactions.mt942") }
 
         before do
           allow_any_instance_of(EbicsUser).to receive(:client) { client }
-          allow(client).to receive(:VMK).and_return(File.read("spec/fixtures/multiple-transactions.mt942"))
+          allow(client).to receive(:VMK).and_return(import_file)
           allow(Account).to(
             receive(:[]).and_return(double("account", organization: double("orga", webhook_token: "token")))
           )
@@ -53,14 +54,63 @@ module Jobs
           allow(BusinessProcesses::ImportStatements).to receive(:from_bank_statement).and_call_original
         end
 
-        it "imports all bank statements" do
-          included_vmk = 3
+        context "with handelsbank format" do
+          before { Cmxl.config[:strip_headers] = true }
+          after { Cmxl.config[:strip_headers] = false }
 
-          job.fetch_for_account(account)
+          it "imports all bank statements" do
+            included_vmk = 3
 
-          expect(BusinessProcesses::ImportBankStatement).to(
-            have_received(:from_cmxl).exactly(included_vmk).times
-          )
+            job.fetch_for_account(account)
+
+            expect(BusinessProcesses::ImportBankStatement).to(
+              have_received(:from_cmxl).exactly(included_vmk).times
+            )
+          end
+
+          xit "updates the account balance" do
+            job.fetch_for_account(account)
+
+            expect(account.reload.balance_in_cents).to eql(0.0)
+          end
+        end
+
+        context "with unstructured headers" do
+          let(:import_file) { File.read("spec/fixtures/master_diesel_vmk_data.txt") }
+          before { Cmxl.config[:strip_headers] = true }
+          after { Cmxl.config[:strip_headers] = false }
+
+          it "imports all bank statements" do
+            included_vmk = 2
+
+            job.fetch_for_account(account)
+
+            expect(BusinessProcesses::ImportBankStatement).to(
+              have_received(:from_cmxl).exactly(included_vmk).times
+            )
+          end
+
+          it "flags the statements as expected" do
+            job.fetch_for_account(account)
+
+            expect(Statement.first).to be_expected
+          end
+        end
+
+        context "with structured header" do
+          let(:import_file) { File.read("spec/fixtures/single_with_headers.mt940") }
+          before { Cmxl.config[:strip_headers] = true }
+          after { Cmxl.config[:strip_headers] = false }
+
+          it "imports all bank statements" do
+            included_vmk = 1
+
+            job.fetch_for_account(account)
+
+            expect(BusinessProcesses::ImportBankStatement).to(
+              have_received(:from_cmxl).exactly(included_vmk).times
+            )
+          end
         end
 
         it "imports all statements for all bank statements" do
@@ -82,6 +132,17 @@ module Jobs
           expect(BusinessProcesses::ImportStatements).not_to have_received(:from_bank_statement)
         end
 
+        it "logs error when ebics client raises business error" do
+          allow(client).to receive(:VMK).and_raise(Epics::Error::BusinessError, "foo")
+          allow(Box.logger).to receive(:error)
+
+          job.fetch_for_account(account)
+
+          expect(Box.logger).to(
+            have_received(:error).with("[Jobs::FetchUpcomingStatements] EBICS error. id=#{account.id} reason='EPICS_UNKNOWN - unknown'")
+          )
+        end
+
         context "with timeframe" do
           before { job.send(:options=, from: Date.new(2019, 6, 1), to: Date.new(2019, 10, 31)) }
 

Original file line number	Diff line number	Diff line change
`@@ -84,7 +84,7 @@ Statement Created:`
`84`	`84`	`account_id,`
`85`	`85`	`statement: {`
`86`	`86`	`id, account (iban), name, bic, iban, type, amount (cents), date,`
`87`		`- remittance_information`
	`87`	`+ remittance_information, expected, reversal`
`88`	`88`	`}`
`89`	`89`	`}`
`90`	`90`	```