Merge pull request #66 from railslove/feat/strip_header-default

feat: enable header parsing as default

Author: janz93

CHANGELOG.mdown

@@ -1,5 +1,10 @@
 # NEXT release
 
+# 2.2
+- `[ENHANCEMENT]` enable `strip_headers` option per default.
+- `[BUGFIX]` fix `strip_header` remove greedy regex logic
+- `[BUGFIX]` support details parsing from files on windows os
+
 # 2.1
 - `[REFACTOR]` improve file parser to work with windows line breaks and any known header formats
 - `[BUGFIX]` fix `strip_header` making it work on statements without headers

README.md

@@ -38,25 +38,11 @@ Or install it yourself as:
 
 ## Usage
 
-Simple usage:
+### Simple usage:
 
 ```ruby
+statements = Cmxl.parse(File.read('mt940.txt'))
 
-# Configuration:
-
-# statement divider regex to split the individual statements in one file - the default is standard and should be good for most files
-Cmxl.config[:statement_separator] = /\n-.\n/m
-
-# do you want an error to be raised when a line can not be parsed? default is true
-Cmxl.config[:raise_line_format_errors] = true
-
-# try to stip the SWIFT header data. This strips everything until the actual first MT940 field. (if parsing fails, try this!)
-Cmxl.config[:strip_headers] = true
-
-
-# Statment parsing:
-
-statements = Cmxl.parse(File.read('mt940.txt'), :encoding => 'ISO-8859-1') # parses the file and returns an array of statement objects. Please note: if no encoding is given Cmxl tries to guess the encoding from the content and converts it to UTF-8.
 statements.each do |s|
   puts s.reference
   puts s.generation_date
@@ -81,25 +67,58 @@ statements.each do |s|
     # ...
   end
 end
-
 ```
 
 Every object responds to `to_h` and let's you easily convert the data to a hash. Also every object responds to `to_json` which lets you easily represent the statements as JSON with your favorite JSON library.
 
-#### A note about encoding and file weirdnesses
+### File encoding options
 
 You probably will encounter encoding issues (hey, you are building banking applications!).
 We try to handle encoding and format weirdnesses as much as possible. If no encoding is passed we try to guess the encoding of the data and convert it to UTF8.
 In the likely case that you encounter encoding issues you can pass encoding options to `Cmxl.parse(<string>, <options hash>)`. It accepts the same options as [String#encode](http://ruby-doc.org/core-2.1.3/String.html#method-i-encode)
 If that fails, try to modify the file before you pass it to the parser - and please create an issue.
 
-### MT940 SWIFT header data
+```ruby
+Cmxl.parse(File.read('mt940.txt'), :encoding => 'ISO-8859-1')
+```
+
+## Global configurations:
+The gem offers option to adjust behavior for the gem
+
+### `statement_separator`
+statement divider regex to split the individual statements in one file
+
+|type|default|
+|----|-------|
+|regex|[`/\R+-[^\n\r]*\R*/m`](https://github.com/railslove/cmxl/blob/main/lib/cmxl.rb#L18)|
 
+```ruby
+Cmxl.config[:statement_separator] = ...
+Cmxl.parse(...)
+```
+
+### `raise_line_format_errors`
+do you want an error to be raised when a line can not be parsed? 
+
+|type|default|
+|----|-------|
+|boolean|[`true`](https://github.com/railslove/cmxl/blob/main/lib/cmxl.rb#L19)|
+
+```ruby
+Cmxl.config[:raise_line_format_errors] = ...
+Cmxl.parse(...)
+```
+
+### `strip_headers`
 Cmxl currently does not support parsing of the SWIFT headers (like {1:F01AXISINBBA ....)
 If your file comes with these headers try the `strip_headers` configuration option to strip data except the actual MT940 fields.
 
+|type|default|
+|----|-------|
+|boolean|[`false`](https://github.com/railslove/cmxl/blob/main/lib/cmxl.rb#L20)|
+
 ```ruby
-Cmxl.config[:strip_headers] = true
+Cmxl.config[:strip_headers] = ...
 Cmxl.parse(...)
 ```
 

lib/cmxl.rb

@@ -17,7 +17,7 @@ def self.config
     # \R is a platform independent newline but in the negated group `[^\n\r]` that did not seem to work.
     statement_separator: /\R+-[^\n\r]*\R*/m,
     raise_line_format_errors: true,
-    strip_headers: false
+    strip_headers: true
   }
 
   # Public: Parse a MT940 string

lib/cmxl/fields/statement_details.rb

@@ -7,7 +7,7 @@ class StatementDetails < Field
       class << self
         def parse(line)
           # remove line breaks as they are allowed via documentation but not needed for data-parsing
-          super line.delete("\n")
+          super line.gsub(/\R?/, '')
         end
       end
 

lib/cmxl/statement.rb

@@ -42,7 +42,6 @@ def parse!
 
     def strip_headers!
       source.gsub!(/\A.*?(?=^:)/m, '') # beginning: strip every line in the beginning that does not start with a :
-      source.gsub!(/^[^:]*\z/, '') # end: strip every line in the end that does not start with a :
       source.strip!
     end
 

lib/cmxl/version.rb

@@ -1,3 +1,3 @@
 module Cmxl
-  VERSION = '2.1'.freeze
+  VERSION = '2.2'.freeze
 end

spec/fields/statement_details_spec.rb

@@ -75,6 +75,47 @@
       )
     }
     it { expect(subject.to_hash).to eql(subject.to_h) }
+
+    context 'with newlines in windows format' do
+      it 'removes any newlines' do
+        data = ":86:171?00SEPA LASTSCHRIFT KUNDE?10281?\r\n20KREF+EREF+TRX-0A4A47C3-F846-4729?21-8A1B-5DF620F?22MREF+CAC97D2144174318AC18D9?23BF815BD4FB?24CRED+DE98ZZZ09999999999?25SVWZ+FOO TRX-0A4A47C3-F84?266-4729-8A1B-5DF620F?30HYVEDEMMXXX?31HUkkbbbsssskcccccccccccccccx?\r\n32Peter Pan?99?34171"
+        result = Cmxl::Fields::StatementDetails.parse(data)
+
+        expect(result.to_h).to eql(
+          'bic' => 'HYVEDEMMXXX',
+          'iban' => 'HUkkbbbsssskcccccccccccccccx',
+          'name' => 'Peter Pan',
+          'sepa' => {
+            'KREF' => '',
+            'EREF' => 'TRX-0A4A47C3-F846-4729-8A1B-5DF620F',
+            'MREF' => 'CAC97D2144174318AC18D9BF815BD4FB',
+            'CRED' => 'DE98ZZZ09999999999',
+            'SVWZ' => 'FOO TRX-0A4A47C3-F846-4729-8A1B-5DF620F'
+          },
+          'information' => 'KREF+EREF+TRX-0A4A47C3-F846-4729-8A1B-5DF620FMREF+CAC97D2144174318AC18D9BF815BD4FBCRED+DE98ZZZ09999999999SVWZ+FOO TRX-0A4A47C3-F846-4729-8A1B-5DF620F',
+          'description' => 'SEPA LASTSCHRIFT KUNDE',
+          'sub_fields' => {
+            '00' => 'SEPA LASTSCHRIFT KUNDE',
+            '10' => '281',
+            '20' => 'KREF+EREF+TRX-0A4A47C3-F846-4729',
+            '21' => '-8A1B-5DF620F',
+            '22' => 'MREF+CAC97D2144174318AC18D9',
+            '23' => 'BF815BD4FB',
+            '24' => 'CRED+DE98ZZZ09999999999',
+            '25' => 'SVWZ+FOO TRX-0A4A47C3-F84',
+            '26' => '6-4729-8A1B-5DF620F',
+            '30' => 'HYVEDEMMXXX',
+            '31' => 'HUkkbbbsssskcccccccccccccccx',
+            '32' => 'Peter Pan',
+            '34' => '171',
+            '99' => ''
+          },
+          'transaction_code' => '171',
+          'primanota' => '281',
+          'details' => '?00SEPA LASTSCHRIFT KUNDE?10281?20KREF+EREF+TRX-0A4A47C3-F846-4729?21-8A1B-5DF620F?22MREF+CAC97D2144174318AC18D9?23BF815BD4FB?24CRED+DE98ZZZ09999999999?25SVWZ+FOO TRX-0A4A47C3-F84?266-4729-8A1B-5DF620F?30HYVEDEMMXXX?31HUkkbbbsssskcccccccccccccccx?32Peter Pan?99?34171'
+        )
+      end
+    end
   end
 
   describe 'information parsing with empty fields on the end' do

spec/statement_spec.rb

@@ -214,6 +214,13 @@
     end
 
     context "when strip_headers is disabled" do
+      around do |example|
+        existing_value = Cmxl.config[:strip_headers]
+        Cmxl.config[:strip_headers] = false
+        example.run
+        Cmxl.config[:strip_headers] = existing_value
+      end
+
       it "raise an parsing error exception if headers are present" do
         data = <<~MT940.chomp
           {1:D02AASDISLNETAXXXXXXXXXXXXX}