diff --git a/Gemfile b/Gemfile index 7277a8a6..0cced981 100644 --- a/Gemfile +++ b/Gemfile @@ -5,8 +5,8 @@ source "https://rubygems.org" ruby "3.3.2" gem "activesupport" -gem "camt_parser", git: "https://github.com/railslove/camt_parser.git" -gem "cmxl", git: "https://github.com/railslove/cmxl" +gem "sepa_file_parser" +gem "cmxl" gem "epics" gem "faraday" gem "grape" diff --git a/Gemfile.lock b/Gemfile.lock index 0b4a9055..f11a4832 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,17 +1,3 @@ -GIT - remote: https://github.com/railslove/camt_parser.git - revision: 3996d9dce6c4dbdc950c6d607102e3c4ce52ca52 - specs: - camt_parser (1.0.2) - nokogiri - -GIT - remote: https://github.com/railslove/cmxl - revision: b92aea3de958426119d76d043a886233a313f43f - specs: - cmxl (1.5.0) - rchardet - GIT remote: https://github.com/railslove/king_dtaus_ruby_3.git revision: 28b077aa6b2cf5590e322d0d207236adccf2d543 @@ -45,6 +31,8 @@ GEM base64 (0.2.0) bigdecimal (3.1.8) builder (3.3.0) + cmxl (2.2) + rchardet coderay (1.1.3) concurrent-ruby (1.3.4) connection_pool (2.4.1) @@ -55,6 +43,7 @@ GEM database_cleaner-sequel (2.0.2) database_cleaner-core (~> 2.0.0) sequel + date (3.4.1) diff-lcs (1.5.1) domain_name (0.5.20190701) unf (>= 0.0.5, < 1.0.0) @@ -123,7 +112,7 @@ GEM mime-types (3.5.2) mime-types-data (~> 3.2015) mime-types-data (3.2024.0604) - mini_portile2 (2.8.7) + mini_portile2 (2.8.8) minitest (5.15.0) multi_json (1.15.0) mustermann (2.0.2) @@ -134,7 +123,7 @@ GEM uri netrc (0.11.0) nio4r (2.7.4) - nokogiri (1.16.6) + nokogiri (1.18.2) mini_portile2 (~> 2.8.2) racc (~> 1.4) parallel (1.25.1) @@ -149,7 +138,7 @@ GEM puma (6.4.3) nio4r (~> 2.0) raabro (1.4.0) - racc (1.8.0) + racc (1.8.1) rack (2.2.9) rack-accept (0.4.5) rack (>= 0.4) @@ -160,7 +149,7 @@ GEM rack (>= 1.0, < 3) rainbow (3.1.1) rake (13.2.1) - rchardet (1.8.0) + rchardet (1.9.0) redis (4.8.1) regexp_parser (2.9.2) rest-client (2.1.0) @@ -210,6 +199,10 @@ GEM sentry-sidekiq (5.18.0) sentry-ruby (~> 5.18.0) sidekiq (>= 3.0) + sepa_file_parser (0.4.0) + bigdecimal + nokogiri + time sepa_king (0.12.0) activemodel (>= 3.1) iban-tools @@ -239,6 +232,8 @@ GEM rubocop-performance (~> 1.21.0) statsd-ruby (1.5.0) tilt (2.4.0) + time (0.4.1) + date timecop (0.9.10) tzinfo (2.0.6) concurrent-ruby (~> 1.0) @@ -260,8 +255,7 @@ DEPENDENCIES activesupport airborne barnes - camt_parser! - cmxl! + cmxl database_cleaner-sequel dotenv epics @@ -287,6 +281,7 @@ DEPENDENCIES rubocop sentry-ruby sentry-sidekiq + sepa_file_parser sepa_king sequel sidekiq diff --git a/box/apis/v2/documentation.yml b/box/apis/v2/documentation.yml index f3f879ee..50d9e2e3 100644 --- a/box/apis/v2/documentation.yml +++ b/box/apis/v2/documentation.yml @@ -84,7 +84,7 @@ Statement Created: account_id, statement: { id, account (iban), name, bic, iban, type, amount (cents), date, - remittance_information + remittance_information, expected, reversal } } ``` diff --git a/box/business_processes/import_bank_statement.rb b/box/business_processes/import_bank_statement.rb index fd33dd1d..03dc0239 100644 --- a/box/business_processes/import_bank_statement.rb +++ b/box/business_processes/import_bank_statement.rb @@ -1,14 +1,10 @@ # frozen_string_literal: true require "cmxl" - require_relative "../models/account" require_relative "../models/bank_statement" require_relative "../../lib/checksum_generator" -# more general matching regex that covers both newlines and newlines with dashes -Cmxl.config[:statement_separator] = /(\n-?)(?=:20)/m - module Box module BusinessProcesses class ImportBankStatement diff --git a/box/business_processes/import_statements.rb b/box/business_processes/import_statements.rb index f1b3cc01..1370bbec 100644 --- a/box/business_processes/import_statements.rb +++ b/box/business_processes/import_statements.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true -require "cmxl" require "camt_parser" +require "cmxl" require_relative "../models/account" require_relative "../models/bank_statement" @@ -113,8 +113,10 @@ def self.statement_attributes_from_bank_transaction(transaction, bank_statement) eref: transaction.respond_to?(:eref) ? transaction.eref : transaction.sepa["EREF"], mref: transaction.respond_to?(:mref) ? transaction.mref : transaction.sepa["MREF"], svwz: transaction.respond_to?(:svwz) ? transaction.svwz : transaction.sepa["SVWZ"], - tx_id: transaction.try(:primanota) || transaction.try(:transaction_id), - creditor_identifier: transaction.respond_to?(:creditor_identifier) ? transaction.creditor_identifier : transaction.sepa["CRED"] + tx_id: transaction.try(:transaction_id), + creditor_identifier: transaction.respond_to?(:creditor_identifier) ? transaction.creditor_identifier : transaction.sepa["CRED"], + expected: transaction.try(:expected?), + reversal: transaction..try(:reversal?) } end end diff --git a/box/entities/statement.rb b/box/entities/statement.rb index dac62cbe..dccbc222 100644 --- a/box/entities/statement.rb +++ b/box/entities/statement.rb @@ -12,6 +12,8 @@ class Statement < Grape::Entity expose :bic expose :iban expose :type, documentation: {type: "Enum", desc: "Type of statement", values: %w[credit debit]} + expose :expected, documentation: {type: "Boolean", desc: "Expected statement are not yet confirmed"} + expose :reversal, documentation: {type: "Boolean", desc: "Reversal of a previous transaction"} expose :amount, documentation: {type: "Integer", desc: "Amount in cents"} expose :date expose(:remittance_information, documentation: {type: "String", desc: "Wire transfer reference"}) { |statement| statement[:svwz] || statement[:information] } diff --git a/box/entities/v2/transaction.rb b/box/entities/v2/transaction.rb index e6d3737b..8fd57c22 100644 --- a/box/entities/v2/transaction.rb +++ b/box/entities/v2/transaction.rb @@ -12,6 +12,8 @@ class Transaction < Grape::Entity expose :iban expose :bic expose :type + expose :expected + expose :reversal expose :amount, as: "amount_in_cents" expose :date, as: "executed_on" expose(:settled_at) { |trx| trx.settled ? trx.date : nil } diff --git a/box/jobs/fetch_statements.rb b/box/jobs/fetch_statements.rb index 9215b976..19d23207 100644 --- a/box/jobs/fetch_statements.rb +++ b/box/jobs/fetch_statements.rb @@ -2,7 +2,7 @@ require "sidekiq-scheduler" require "active_support/all" -require "camt_parser" +require "sepa_file_parser" require "cmxl" require "epics" require "sequel" @@ -84,7 +84,7 @@ def camt53(client, from, to) combined_camt = client.C53(from.to_s(:db), to.to_s(:db)) return unless combined_camt.any? - combined_camt.map { |chunk| CamtParser::String.parse(chunk).statements }.flatten + combined_camt.map { |chunk| SepaFileParser::String.parse(chunk).statements }.flatten end def mt940(client, from, to) diff --git a/box/jobs/fetch_upcoming_statements.rb b/box/jobs/fetch_upcoming_statements.rb index 6116c4aa..03e42cc1 100644 --- a/box/jobs/fetch_upcoming_statements.rb +++ b/box/jobs/fetch_upcoming_statements.rb @@ -2,8 +2,6 @@ require "sidekiq-scheduler" require "active_support/all" -require "camt_parser" -require "cmxl" require "epics" require "sequel" diff --git a/box/jobs/queue_fetch_statements.rb b/box/jobs/queue_fetch_statements.rb index d50d4ea9..7d63438a 100644 --- a/box/jobs/queue_fetch_statements.rb +++ b/box/jobs/queue_fetch_statements.rb @@ -2,8 +2,6 @@ require "sidekiq-scheduler" require "active_support/all" -require "camt_parser" -require "cmxl" require "epics" require "sequel" diff --git a/box/jobs/queue_fetch_upcoming_statements.rb b/box/jobs/queue_fetch_upcoming_statements.rb index bdfa8299..48fd258b 100644 --- a/box/jobs/queue_fetch_upcoming_statements.rb +++ b/box/jobs/queue_fetch_upcoming_statements.rb @@ -2,8 +2,6 @@ require "sidekiq-scheduler" require "active_support/all" -require "camt_parser" -require "cmxl" require "epics" require "sequel" diff --git a/box/models/statement.rb b/box/models/statement.rb index 256af97a..d1811af2 100644 --- a/box/models/statement.rb +++ b/box/models/statement.rb @@ -84,6 +84,14 @@ def debit? debit end + def reversal? + reversal + end + + def expected? + !!expected + end + def type debit? ? "debit" : "credit" end diff --git a/db/migrations/20250121163300_add_meta_data_to_statement.rb b/db/migrations/20250121163300_add_meta_data_to_statement.rb new file mode 100644 index 00000000..dc2545eb --- /dev/null +++ b/db/migrations/20250121163300_add_meta_data_to_statement.rb @@ -0,0 +1,8 @@ +# frozen_string_literal: true + +Sequel.migration do + change do + add_column :statements, :expected, :boolean + add_column :statements, :reversal, :boolean + end +end diff --git a/db/schema.rb b/db/schema.rb index c13bdb46..1d416c96 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -117,6 +117,8 @@ column :settled, "boolean", :default=>true column :sha, "text" column :tx_id, "text" + column :expected, "boolean" + column :reversal, "boolean" index [:sha], :name=>:statements_sha2_index, :unique=>true index [:sha_bak], :name=>:statements_sha_key, :unique=>true diff --git a/spec/apis/v2/transactions_spec.rb b/spec/apis/v2/transactions_spec.rb index bf56fb07..0b37c969 100644 --- a/spec/apis/v2/transactions_spec.rb +++ b/spec/apis/v2/transactions_spec.rb @@ -16,6 +16,8 @@ module Box amount_in_cents: :integer, executed_on: :date, type: :string, + expected: :boolean, + reversal: :boolean, reference: :string, end_to_end_reference: :string, settled_at: :string diff --git a/spec/business_processes/import_bank_statement_spec.rb b/spec/business_processes/import_bank_statement_spec.rb index e6553364..6facf9ff 100644 --- a/spec/business_processes/import_bank_statement_spec.rb +++ b/spec/business_processes/import_bank_statement_spec.rb @@ -1,7 +1,6 @@ # frozen_string_literal: true require "active_support/all" -require "cmxl" require_relative "../../box/models/account" require_relative "../../box/business_processes/import_bank_statement" diff --git a/spec/business_processes/import_statement_spec.rb b/spec/business_processes/import_statement_spec.rb index 19298a34..77324d50 100644 --- a/spec/business_processes/import_statement_spec.rb +++ b/spec/business_processes/import_statement_spec.rb @@ -1,7 +1,6 @@ # frozen_string_literal: true require "active_support/all" -require "cmxl" require_relative "../../box/models/account" require_relative "../../box/models/organization" @@ -257,6 +256,19 @@ def exec_link_action expect(described_class.create_statement(mt940_bank_statement, transaction)).to be_truthy end end + + context "importing vmk transaction that is expected" do + it "marks statements as expected" do + vmk_data = File.read("spec/fixtures/master_diesel_vmk_data.txt") + bank_statement = ImportBankStatement.from_mt940(vmk_data, account) + bank_transactions = described_class.parse_bank_statement(bank_statement) + transaction = bank_transactions.first + + described_class.create_statement(bank_statement, transaction) + + expect(Statement.first).to be_expected + end + end end end end diff --git a/spec/fabricators/statement_fabricator.rb b/spec/fabricators/statement_fabricator.rb index 221a9a5d..e87bfd6b 100644 --- a/spec/fabricators/statement_fabricator.rb +++ b/spec/fabricators/statement_fabricator.rb @@ -44,4 +44,6 @@ bic { STATEMENT_BICS.sample } eref { "eref-#{Fabricate.sequence(:eref)}" } svwz { Faker::Lorem.sentence } + expected { [true, false].sample } + reversal { [true, false].sample } end diff --git a/spec/fixtures/abnamro.mt940 b/spec/fixtures/abnamro.mt940 new file mode 100644 index 00000000..9e18d89b --- /dev/null +++ b/spec/fixtures/abnamro.mt940 @@ -0,0 +1,41 @@ +ABNANL2A +940 +ABNANL2A +:20:ABN AMRO BANK NV +:25:517852257 +:28:19321/1 +:60F:C110522EUR3236,28 +:61:1105240524D9,N192NONREF +:86:GIRO 428428 KPN - DIGITENNE BETALINGSKENM. 000000042188659 +5314606715 BETREFT FACTUUR D.D. 20-05-2011 +INCL. 1,44 BTW +:61:1105210523D11,59N426NONREF +:86:BEA NR:XXX1234 21.05.11/12.54 DIRCKIII FIL2500 KATWIJK,PAS999 +:61:1105230523D11,63N426NONREF +:86:BEA NR:XXX1234 23.05.11/09.08 DIGROS FIL1015 KATWIJK Z,PAS999 +:61:1105220523D11,8N426NONREF +:86:BEA NR:XXX1234 22.05.11/14.25 MC DONALDS A44 LEIDEN,PAS999 +:61:1105210523D13,45N426NONREF +:86:BEA NR:XXX1234 21.05.11/12.09 PRINCE FIL. 55 KATWIJK Z,PAS999 +:61:1105210523D15,49N426NONREF +:86:BEA NR:XXX1234 21.05.11/12.55 DIRX FIL6017 KATWIJK ZH ,PAS999 + +:61:1105210523D107,N426NONREF +:86:BEA NR:XXX1234 21.05.11/12.04 HANS ANDERS OPT./056 KAT,PAS999 +:61:1105220523D141,48N426NONREF +:86:BEA NR:XXX1234 22.05.11/13.45 MYCOM DEN HAAG S-GRAVEN,PAS999 +:62F:C110523EUR876,84 +- +ABNANL2A +940 +ABNANL2A +:20:ABN AMRO BANK NV +:25:517852257 +:28:19322/1 +:60F:C110523EUR2876,84 +:61:1105240524D9,49N426NONREF +:86:BEA NR:XXX1234 24.05.11/09.18 PETS PLACE KATWIJK KATWI,PAS999 +:61:1105240524D15,N426NONREF +:86:52.89.39.882 MYCOM DEN HAAG S-GRAVEN,PAS999 +:62F:C110524EUR1849,75 +- \ No newline at end of file diff --git a/spec/fixtures/handelbank.mt940 b/spec/fixtures/handelbank.mt940 new file mode 100644 index 00000000..712b2103 --- /dev/null +++ b/spec/fixtures/handelbank.mt940 @@ -0,0 +1,16 @@ +ä4: +:20:5566778899100112 +:25:10020030/1234567 +:28C:188/1 +:60F:C130928SEK0, +:62F:C130930SEK0, +:64:C130930SEK0, +-å +ä4: +:20:5566778899100169 +:25:10020030/1234567 +:28C:188/1 +:60F:C130928SEK0, +:62F:C130930SEK0, +:64:C130930SEK0, +-å \ No newline at end of file diff --git a/spec/fixtures/master_diesel_vmk_data.txt b/spec/fixtures/master_diesel_vmk_data.txt new file mode 100644 index 00000000..f2c76c58 --- /dev/null +++ b/spec/fixtures/master_diesel_vmk_data.txt @@ -0,0 +1,58 @@ +RLNWATWWAMS 00001 +942 01 +ELBA-INT +:20:20241212080001 +:25://10020030/1234567 +:28C:24241/001 +:34F:EUR0, +:13D:2412120800+0100 +:61:2412121212ED15162,57NDDTNONREF//950 +:86:999Kreuzmayr GmbH +Siehe Avis vom 12.12.24 +:61:2412121212ED6136,95NDDTNONREF//950 +:86:999Moser Wurst GmbH +1161235,1161236 +:61:2412121212ED2079,12NDDT0094324106//950 +:86:999JULIUS MEINL Austria GmbH +0094324106 +:61:2412121212ED1218,84NDDTNONREF//950 +:86:999unik GmbH +2200243181, EUR 1 218,84 +:61:2412121212ED223,NSUEEUIEV1IV//950 +:86:999ELBA-AUFTRAG EUIEV1IV +:61:2412121212ED190,08NSUEEUIEV1IM//950 +:86:999ELBA-AUFTRAG EUIEV1IM +:61:2412121212EC739,5NTRF186454//950 +:86:999G�ls�m Cakmak +186454 +:61:2412121212EC1000,NTRFNONREF//950 +:86:999IKO-DACH GmbH +:61:2412121212EC1836,25NTRFNONREF//950 +:86:999Int. Transporte A. Fasching Anton F +176471 +:61:2412121212EC2000,NTRF80235//950 +:86:999Valentin Klizan +80235 +:61:2412121212EC2233,5NTRF176417//950 +:86:999Miroslav Beganovic +176417 +:61:2412121212EC14409,79NTRFNONREF//950 +:86:999GLOBAL PAYMENTS S.R.O. +KOMB /REF 4850003118 111224 /B 14673,47 /D 219,73 /U 43,95 /G 269 +master DIESEL Tankstelle +:90D:6EUR25010,56 +:90C:6EUR22219,04 +- +RLNWATWWAMS 00001 +942 01 +ELBA-INT +:20:20241212100006 +:25://10020030/1234567 +:28C:24241/001 +:34F:EUR0, +:13D:2412121000+0100 +:61:2412121212EC43374,55NTRFNONREF//950 +:86:999Staack Pooltankstellen GmbH +RNR 176080 23781,44 0,00 RNR 175584 19593,11 0,00 +:90C:1EUR43374,55 + diff --git a/spec/fixtures/mt940.txt b/spec/fixtures/multiple-transactions.mt940 similarity index 100% rename from spec/fixtures/mt940.txt rename to spec/fixtures/multiple-transactions.mt940 diff --git a/spec/fixtures/mt942.txt b/spec/fixtures/multiple-transactions.mt942 similarity index 100% rename from spec/fixtures/mt942.txt rename to spec/fixtures/multiple-transactions.mt942 diff --git a/spec/fixtures/single_with_headers.mt940 b/spec/fixtures/single_with_headers.mt940 new file mode 100644 index 00000000..22a84b4e --- /dev/null +++ b/spec/fixtures/single_with_headers.mt940 @@ -0,0 +1,16 @@ +{1:D02AASDISLNETAXXXXXXXXXXXXX} +{2:E623XXXXXXXXAXXXN} +{4: +:20:1234567 +:21:9876543210 +:25:10020030/1234567 +:28C:5/1 +:60F:C160314EUR2187,95 +:61:0211011102DR800,NSTONONREF//55555 +:86:008?00DAUERAUFTRAG?100599?20Miete November?3010020030?31234567?32MUELLER?34339 +:61:0211021102CR3000,NTRFNONREF//55555 +:86:051?00UEBERWEISUNG?100599?20Gehalt Oktober?21Firma +Mustermann GmbH?3050060400?310847564700?32MUELLER?34339 +:62F:C160315EUR4387,95 +:86:Some random data +-} \ No newline at end of file diff --git a/spec/jobs/fetch_statements_spec.rb b/spec/jobs/fetch_statements_spec.rb index 7907f196..8fbd729a 100644 --- a/spec/jobs/fetch_statements_spec.rb +++ b/spec/jobs/fetch_statements_spec.rb @@ -34,7 +34,7 @@ module Jobs before do account.imported_at!(1.day.ago) allow_any_instance_of(EbicsUser).to receive(:client) { client } - allow(client).to receive(:STA).and_return(File.read("spec/fixtures/mt940.txt")) + allow(client).to receive(:STA).and_return(File.read("spec/fixtures/multiple-transactions.mt940")) allow(BusinessProcesses::ImportBankStatement).to receive(:from_cmxl).and_call_original allow(BusinessProcesses::ImportStatements).to receive(:from_bank_statement).and_call_original diff --git a/spec/jobs/fetch_upcoming_statements_spec.rb b/spec/jobs/fetch_upcoming_statements_spec.rb index 380076ee..2d54e5f6 100644 --- a/spec/jobs/fetch_upcoming_statements_spec.rb +++ b/spec/jobs/fetch_upcoming_statements_spec.rb @@ -41,10 +41,11 @@ module Jobs describe ".fetch_for_account" do let(:client) { double("Epics Client") } + let(:import_file) { File.read("spec/fixtures/multiple-transactions.mt942") } before do allow_any_instance_of(EbicsUser).to receive(:client) { client } - allow(client).to receive(:VMK).and_return(File.read("spec/fixtures/mt942.txt")) + allow(client).to receive(:VMK).and_return(import_file) allow(Account).to( receive(:[]).and_return(double("account", organization: double("orga", webhook_token: "token"))) ) @@ -53,14 +54,63 @@ module Jobs allow(BusinessProcesses::ImportStatements).to receive(:from_bank_statement).and_call_original end - it "imports all bank statements" do - included_vmk = 3 + context "with handelsbank format" do + before { Cmxl.config[:strip_headers] = true } + after { Cmxl.config[:strip_headers] = false } - job.fetch_for_account(account) + it "imports all bank statements" do + included_vmk = 3 - expect(BusinessProcesses::ImportBankStatement).to( - have_received(:from_cmxl).exactly(included_vmk).times - ) + job.fetch_for_account(account) + + expect(BusinessProcesses::ImportBankStatement).to( + have_received(:from_cmxl).exactly(included_vmk).times + ) + end + + xit "updates the account balance" do + job.fetch_for_account(account) + + expect(account.reload.balance_in_cents).to eql(0.0) + end + end + + context "with unstructured headers" do + let(:import_file) { File.read("spec/fixtures/master_diesel_vmk_data.txt") } + before { Cmxl.config[:strip_headers] = true } + after { Cmxl.config[:strip_headers] = false } + + it "imports all bank statements" do + included_vmk = 2 + + job.fetch_for_account(account) + + expect(BusinessProcesses::ImportBankStatement).to( + have_received(:from_cmxl).exactly(included_vmk).times + ) + end + + it "flags the statements as expected" do + job.fetch_for_account(account) + + expect(Statement.first).to be_expected + end + end + + context "with structured header" do + let(:import_file) { File.read("spec/fixtures/single_with_headers.mt940") } + before { Cmxl.config[:strip_headers] = true } + after { Cmxl.config[:strip_headers] = false } + + it "imports all bank statements" do + included_vmk = 1 + + job.fetch_for_account(account) + + expect(BusinessProcesses::ImportBankStatement).to( + have_received(:from_cmxl).exactly(included_vmk).times + ) + end end it "imports all statements for all bank statements" do @@ -82,6 +132,17 @@ module Jobs expect(BusinessProcesses::ImportStatements).not_to have_received(:from_bank_statement) end + it "logs error when ebics client raises business error" do + allow(client).to receive(:VMK).and_raise(Epics::Error::BusinessError, "foo") + allow(Box.logger).to receive(:error) + + job.fetch_for_account(account) + + expect(Box.logger).to( + have_received(:error).with("[Jobs::FetchUpcomingStatements] EBICS error. id=#{account.id} reason='EPICS_UNKNOWN - unknown'") + ) + end + context "with timeframe" do before { job.send(:options=, from: Date.new(2019, 6, 1), to: Date.new(2019, 10, 31)) }