Bugfix: handle invalid length record (#45)

* check 300 record has correct length
* test case for invalid length 300 record
* update changelog
* check empty cells on 400 records
* spec for empty cells on both 300 and 400 records
* 400 record with emptyb startInterval and endInterval values

CHANGELOG.md

@@ -1,5 +1,9 @@
 # AEMO Gem Changelog
 
+## 0.3.1
+
+* Catch invalid length  NEM12 records (300 and 400)
+
 ## 0.3.0
 
 * Refactor NMI allocations

Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    aemo (0.3.0.pre.rc2)
+    aemo (0.3.1)
       activesupport (>= 4.2.6, < 5.2)
       httparty (~> 0.15, >= 0.15.6)
       json (>= 1.7.5)

lib/aemo/nem12.rb

@@ -349,15 +349,19 @@ def parse_nem12_200(line, options = {})
     # @return [Array of hashes] the line parsed into a hash of information
     def parse_nem12_300(line, options = {})
       csv = line.parse_csv
-
       raise TypeError, 'Expected NMI Data Details to exist with IntervalLength specified' if @data_details.last.nil? || @data_details.last[:interval_length].nil?
+
+      # ref: AEMO's MDFF Spec NEM12 and NEM13 v1.01 (2014-05-14)
+      record_fixed_fields = %w[RecordIndicator IntervalDate QualityMethod ReasonCode ReasonDescription UpdateDatetime MSATSLoadDateTime]
       number_of_intervals = 1440 / @data_details.last[:interval_length]
+      raise TypeError, 'Invalid record length' if csv.length != record_fixed_fields.length + number_of_intervals
+
       intervals_offset = number_of_intervals + 2
 
       raise ArgumentError, 'RecordIndicator is not 300' if csv[0] != '300'
       raise ArgumentError, 'IntervalDate is not valid' if csv[1].match(/\d{8}/).nil? || csv[1] != Time.parse(csv[1].to_s).strftime('%Y%m%d')
       (2..(number_of_intervals + 1)).each do |i|
-        raise ArgumentError, "Interval number #{i - 1} is not valid" if csv[i].match(/\d+(\.\d+)?/).nil?
+        raise ArgumentError, "Interval number #{i - 1} is not valid" if csv[i].nil? || csv[i].match(/\d+(\.\d+)?/).nil?
       end
       raise ArgumentError, 'QualityMethod is not valid' unless csv[intervals_offset + 0].class == String
       raise ArgumentError, 'QualityMethod does not have valid length' unless [1, 3].include?(csv[intervals_offset + 0].length)
@@ -411,9 +415,9 @@ def parse_nem12_300(line, options = {})
     def parse_nem12_400(line)
       csv = line.parse_csv
       raise ArgumentError, 'RecordIndicator is not 400'     if csv[0] != '400'
-      raise ArgumentError, 'StartInterval is not valid'     if csv[1].match(/^\d+$/).nil?
-      raise ArgumentError, 'EndInterval is not valid'       if csv[2].match(/^\d+$/).nil?
-      raise ArgumentError, 'QualityMethod is not valid'     if csv[3].match(/^([AN]|([AEFNSV]\d{2}))$/).nil?
+      raise ArgumentError, 'StartInterval is not valid'     if csv[1].nil? || csv[1].match(/^\d+$/).nil?
+      raise ArgumentError, 'EndInterval is not valid'       if csv[2].nil? || csv[2].match(/^\d+$/).nil?
+      raise ArgumentError, 'QualityMethod is not valid'     if csv[3].nil? || csv[3].match(/^([AN]|([AEFNSV]\d{2}))$/).nil?
       # raise ArgumentError, 'ReasonCode is not valid'        if (csv[4].nil? && csv[3].match(/^ANE/)) || csv[4].match(/^\d{3}?$/) || csv[3].match(/^ANE/)
       # raise ArgumentError, 'ReasonDescription is not valid' if (csv[4].nil? && csv[3].match(/^ANE/)) || ( csv[5].match(/^$/) && csv[4].match(/^0$/) )
 

lib/aemo/version.rb

@@ -24,7 +24,7 @@
 # @author Joel Courtney <[email protected]>
 module AEMO
   # aemo version
-  VERSION = '0.3.0'
+  VERSION = '0.3.1'
 
   # aemo version split amongst different revisions
   MAJOR_VERSION, MINOR_VERSION, REVISION = VERSION.split('.').map(&:to_i)

spec/fixtures/NEM12-Errors/NEM12#EmptyCells300Record#CNRGYMDP#NEMMCO.csv

@@ -0,0 +1,4 @@
+100,NEM12,200505181432,CNRGYMDP,NEMMCO
+200,NEM1201002,E1,E1,E1,N1,01002,KWH,30,
+300,20050315,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,A,,,20050316014209,
+900

spec/fixtures/NEM12-Errors/NEM12#EmptyCells400Record#CNRGYMDP#NEMMCO.csv

@@ -0,0 +1,8 @@
+100,NEM12,200504210935,WBAYM,NEMMCO
+200,NEM1201010,E1E2,,E1,N1,01010,kWh,30,
+300,20050305,1832.51,1707.74,1608.67,1628.47,1644.39,1617.39,1539.87,1492.38,1499.55,1491.75,1427.61,1450.55,1487.92,1377.05,1371.61,1407.45,1249.92,1309.85,1296.15,1285.1,1342.47,1303.87,1255.95,1294.83,1361.27,1383.39,1403.07,1486.87,1613.53,1581.35,1466.69,1297.42,1169.94,1094.91,1459.1,1335.62,1278.71,1319.1,1292.94,1243.86,1299.5,1257.6,1287.33,1227.28,1293.34,1244.71,1282.68,1258.1,A,,,20050421081805,
+300,20050306,1254.43,1221.69,1208.19,1214.3,1271.1,1203.55,1161.82,1161.41,1170.96,1152.25,1118.34,1125.05,1186.6,1183.8,1215.59,1241.14,1250.63,1228.9,1224.77,1252.75,1238.03,1284.19,1264.23,1285.67,1322.01,1308.02,1290.7,1335.53,1368.84,1480.85,1223.51,1277.96,1300.66,1328.22,1265.31,1254.55,1231.74,1239.16,1174.06,1211.66,1172.19,1191.44,1231.58,1279.82,1253.19,1183.71,1181.86,1141.48,A,,,20050421081805,
+300,20050307,1208.34,1245.41,1242,1210.67,1389.96,1343.92,1419.53,1364.32,1422.17,1452.68,1548.35,1756.84,1995.47,2252.12,2629.64,2673.19,2682.86,2559.22,2740.62,2735.74,2797.83,2782.9,2849.85,2892.1,2760.26,2932.58,2916.79,3006.37,2997.92,2982.44,2843.39,2650.55,2366.95,2414.74,2433.92,2393,2481.81,2420.54,2339.02,2294.67,2267.39,2225.93,2215.23,2254.36,2215.43,2176.78,2097.44,1960.64,A,,,20050421081805,
+300,20050308,1911.77,1887.22,1848.15,1837.62,1894.4,1976.38,1962.79,1938.92,1904.96,1866.19,2055.67,2186.99,2448.23,2563.53,2637.81,2833.17,2848.7,2725.29,2878.08,2818.36,2841.61,2927.15,2929.52,2880,2799.11,2952.93,2960.64,2986.41,3047.51,3035.36,2899.8,2724.43,2620.27,2486.11,2507.44,2561.05,2583.66,2382.79,1849.87,1821.83,1839.49,2094.05,2273.56,2218.17,2256.87,2266.54,2199.18,2090.56,A,,,20050421081805,
+400,,,F56,45,
+900

spec/fixtures/NEM12-Errors/NEM12#InvalidIntervalDataLength#CNRGYMDP#NEMMCO.csv

@@ -0,0 +1,4 @@
+100,NEM12,200505181432,CNRGYMDP,NEMMCO
+200,NEM1201002,E1E2,E1,E1,N1,01002,KWH,30,
+300,20050315,300.000,266.100,191.550,247.800,288.600,280.800,282.450,206.100,407.700,432.600,435.000,491.850,600.900,541.950,474.600,565.350,548.550,491.850,593.250,602.400,571.350,450.150,509.400,559.950,522.000,520.950,541.200,538.050,484.800,330.900,329.250,331.650,330.750,333.750,335.250,294.150,185.250,184.800,186.450,256.800,329.700,320.100,316.500,321.150,A,,,20050316014209,
+900

spec/lib/aemo/nem12_spec.rb

@@ -67,9 +67,22 @@
   end
 
   describe '#parse_nem12_300' do
+    it 'should raise invalid record length error' do
+      bad_file = fixture(File.join('NEM12-Errors', 'NEM12#InvalidIntervalDataLength#CNRGYMDP#NEMMCO.csv'))
+      expect { AEMO::NEM12.parse_nem12_file(bad_file) }.to raise_error(TypeError, 'Invalid record length')
+    end
+
+    it 'should raise argument error on 300 empty cells' do
+      nem12_empty_cells_300_record = fixture(File.join('NEM12-Errors', 'NEM12#EmptyCells300Record#CNRGYMDP#NEMMCO.csv'))
+      expect { AEMO::NEM12.parse_nem12_file(nem12_empty_cells_300_record) }.to raise_error(ArgumentError)
+    end
   end
 
   describe '#parse_nem12_400' do
+    it 'should raise argument error on 400 empty cells' do
+      nem12_empty_cells_400_record = fixture(File.join('NEM12-Errors', 'NEM12#EmptyCells400Record#CNRGYMDP#NEMMCO.csv'))
+      expect { AEMO::NEM12.parse_nem12_file(nem12_empty_cells_400_record) }.to raise_error(ArgumentError)
+    end
   end
 
   describe '#parse_nem12_500' do