Merge release/2.30.0 to master

.clang-format

@@ -4,6 +4,7 @@ AccessModifierOffset: -4
 AlignAfterOpenBracket: Align
 AlignConsecutiveAssignments: true
 AlignConsecutiveDeclarations: false
+AlignConsecutiveMacros: true
 AlignEscapedNewlines: Left
 AlignOperands:   true
 AlignTrailingComments: true
@@ -66,6 +67,7 @@ IncludeCategories:
     Priority:        3
 IncludeIsMainRegex: '([-_](test|unittest))?$'
 IndentCaseLabels: true
+IndentPPDirectives: BeforeHash
 IndentWidth:     4
 IndentWrappedFunctionNames: false
 JavaScriptQuotes: Leave

.github/.cmake-options

@@ -0,0 +1 @@
+-DENABLE_EXTRA_TESTS=1

CMakeLists.txt

@@ -21,7 +21,7 @@ cmake_minimum_required( VERSION 3.12 FATAL_ERROR )
 find_package( ecbuild 3.7 REQUIRED HINTS ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/../ecbuild)
 
 # Initialise project
-project( eccodes VERSION 2.29.1 LANGUAGES CXX )
+project( eccodes VERSION 2.30.0 LANGUAGES CXX )
 set(CMAKE_CXX_STANDARD 11)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 
@@ -65,6 +65,11 @@ ecbuild_debug("ECCODES_BIG_ENDIAN=${ECCODES_BIG_ENDIAN}")
 ecbuild_debug("ECCODES_LITTLE_ENDIAN=${ECCODES_LITTLE_ENDIAN}")
 ecbuild_info("Operating system=${CMAKE_SYSTEM} (${EC_OS_BITS} bits)")
 
+# Only support 64 bit operating systems
+if( NOT EC_OS_BITS EQUAL "64" )
+    ecbuild_critical( "Operating system ${CMAKE_SYSTEM} (${EC_OS_BITS} bits) -- ecCodes only supports 64 bit platforms" )
+endif()
+
 ###############################################################################
 # some variables/options of this project
 
@@ -315,11 +320,6 @@ if( IEEE_LE )
     set( IEEE_BE 0 )
 endif()
 
-set( ECCODES_ON_LINUX_32BIT 0 )
-if( EC_OS_NAME MATCHES "linux" AND EC_OS_BITS EQUAL "32" )
-    set( ECCODES_ON_LINUX_32BIT 1 )
-endif()
-
 set( ECCODES_ON_WINDOWS 0 )
 if( EC_OS_NAME MATCHES "windows" )
     # Symbols need to be explicitly exported on Windows so we can link to dlls.
@@ -430,11 +430,12 @@ add_subdirectory( samples )
 add_subdirectory( ifs_samples ) # must come after samples
 
 # ecbuild_dont_pack( DIRS samples DONT_PACK_REGEX "*.grib" )
+ecbuild_dont_pack( FILES .cproject .project )
 ecbuild_dont_pack( DIRS
-     experimental deprecated doxygen confluence tests/tests.ecmwf
-     config m4 examples/F77 src/deprecated tools/deprecated ifs_samples/grib1_mlgrib2_ieee32
-     examples/examples.dev examples/extra examples/deprecated bamboo
-     fortran/fortranCtypes tigge/tools share/eccodes  .settings )
+    experimental deprecated doxygen confluence tests/deprecated tests/tests.ecmwf
+    src/deprecated tools/deprecated ifs_samples/grib1_mlgrib2_ieee32
+    examples/examples.dev examples/extra examples/deprecated bamboo
+    fortran/fortranCtypes tigge/tools share/eccodes  .settings )
 #ecbuild_dont_pack( DIRS data/bufr  DONT_PACK_REGEX "*.bufr" )
 #ecbuild_dont_pack( DIRS data/tigge DONT_PACK_REGEX "*.grib" )
 

bamboo/GCC-env.sh

@@ -1,19 +1,19 @@
 #!/bin/bash
 
-[[ $(uname) == "Darwin" ]] && return
+# [[ $(uname) == "Darwin" ]] && return
 
 # initialise module environment if it is not
-if [[ ! $(command -v module > /dev/null 2>&1) ]]; then
-  . /usr/local/apps/module/init/bash
-fi
+#if [[ ! $(command -v module > /dev/null 2>&1) ]]; then
+#   . /usr/local/apps/module/init/bash
+#fi
 
-module unload grib_api
-module unload eccodes
-module unload emos
-module unload fftw
-module unload libemos
-module unload metview
+#module unload grib_api
+#module unload eccodes
+#module unload emos
+#module unload fftw
+#module unload libemos
+#module unload metview
 
-module load cmake/3.16.5
-module load aec
+#module load cmake/3.16.5
+#module load aec
 

data/CMakeLists.txt

@@ -1,4 +1,7 @@
 # data/CMakeLists.txt
+set(ECBUILD_DOWNLOAD_TIMEOUT 60)
+set(ECBUILD_DOWNLOAD_RETRIES 3)
+
 add_subdirectory(tigge)
 add_subdirectory(bufr)
 add_subdirectory(metar)

data/bufr/CMakeLists.txt

@@ -12,13 +12,18 @@ file(READ "bufr_ref_files.txt" bufr_refs_to_download)
 string(REGEX REPLACE "\n" ";" bufr_refs_to_download "${bufr_refs_to_download}")
 
 # Exceptional case: download bufr files which have to be treated specially
-list(APPEND bufr_refs_to_download "vos308014_v3_26.bufr")  # See test ecc-197
+list(APPEND bufr_files_to_download "vos308014_v3_26.bufr")  # See test ecc-197
 
 if( ENABLE_EXTRA_TESTS )
     ecbuild_get_test_multidata(
-         TARGET eccodes_download_bufrs
-         NOCHECK
-        NAMES ${bufr_files_to_download} ${bufr_refs_to_download}
+        TARGET eccodes_download_bufrs
+        NOCHECK
+        NAMES ${bufr_files_to_download}
+    )
+    ecbuild_get_test_multidata(
+        TARGET eccodes_download_bufr_refs
+        NOCHECK
+        NAMES ${bufr_refs_to_download}
     )
 endif()
 

data/bufr/bufr_ref_files.txt

@@ -262,110 +262,6 @@ synop_multi_subset.bufr.desc.ref
 aaen_55.bufr.ls.ref
 syno_multi.bufr.header.ref
 207003.bufr.json.ref
-new_207003.bufr.ref
-new_aaen_55.bufr.ref
-new_aben_55.bufr.ref
-new_ahws_139.bufr.ref
-new_airc_142.bufr.ref
-new_airc_144.bufr.ref
-new_airs_57.bufr.ref
-new_alws_139.bufr.ref
-new_amda_144.bufr.ref
-new_amsa_55.bufr.ref
-new_amsb_55.bufr.ref
-new_amse_55.bufr.ref
-new_amsu_55.bufr.ref
-new_amv2_87.bufr.ref
-new_amv3_87.bufr.ref
-new_asbh_139.bufr.ref
-new_asbl_139.bufr.ref
-new_asca_139.bufr.ref
-new_asch_139.bufr.ref
-new_ascs_139.bufr.ref
-new_aseh_139.bufr.ref
-new_asel_139.bufr.ref
-new_ashs_139.bufr.ref
-new_atap_55.bufr.ref
-new_ateu_155.bufr.ref
-new_atms_201.bufr.ref
-new_atov_55.bufr.ref
-new_avhm_87.bufr.ref
-new_avhn_87.bufr.ref
-new_avhr_58.bufr.ref
-new_b002_95.bufr.ref
-new_b002_96.bufr.ref
-new_b003_56.bufr.ref
-new_b004_145.bufr.ref
-new_b005_87.bufr.ref
-new_b006_96.bufr.ref
-new_b007_31.bufr.ref
-new_bssh_170.bufr.ref
-new_bssh_176.bufr.ref
-new_bssh_178.bufr.ref
-new_bssh_180.bufr.ref
-new_btem_109.bufr.ref
-new_buoy_27.bufr.ref
-new_cmwi_87.bufr.ref
-new_cmwn_87.bufr.ref
-new_cnow_28.bufr.ref
-new_cori_156.bufr.ref
-new_crit_202.bufr.ref
-new_emsg_87.bufr.ref
-new_euwv_87.bufr.ref
-new_fy3a_154.bufr.ref
-new_fy3b_154.bufr.ref
-new_go15_87.bufr.ref
-new_goee_87.bufr.ref
-new_goes_87.bufr.ref
-new_grst_26.bufr.ref
-new_gst4_26.bufr.ref
-new_hirb_55.bufr.ref
-new_hirs_55.bufr.ref
-new_ias1_240.bufr.ref
-new_iasi_241.bufr.ref
-new_itwt_233.bufr.ref
-new_j2eo_216.bufr.ref
-new_j2nb_216.bufr.ref
-new_jaso_214.bufr.ref
-new_meta_140.bufr.ref
-new_mhen_55.bufr.ref
-new_mhsa_55.bufr.ref
-new_mhsb_55.bufr.ref
-new_mhse_55.bufr.ref
-new_modi_87.bufr.ref
-new_modw_87.bufr.ref
-new_monw_87.bufr.ref
-new_mytemp.bufr.ref
-new_new.bufr.ref
-new_ocea_131.bufr.ref
-new_ocea_132.bufr.ref
-new_ocea_133.bufr.ref
-new_ocea_21.bufr.ref
-new_pgps_110.bufr.ref
-new_pilo_91.bufr.ref
-new_s4kn_165.bufr.ref
-new_ship_13.bufr.ref
-new_ship_19.bufr.ref
-new_ship_9.bufr.ref
-new_smin_49.bufr.ref
-new_smis_49.bufr.ref
-new_smiu_49.bufr.ref
-new_smos_203.bufr.ref
-new_sn4k_165.bufr.ref
-new_soil_7.bufr.ref
-new_ssbt_127.bufr.ref
-new_stuk_7.bufr.ref
-new_syno_1.bufr.ref
-new_syno_3.bufr.ref
-new_syno_4.bufr.ref
-new_syno_multi.bufr.ref
-new_synop_multi_subset.bufr.ref
-new_temp_101.bufr.ref
-new_temp_102.bufr.ref
-new_temp_106.bufr.ref
-new_tmr7_129.bufr.ref
-new_tros_31.bufr.ref
-new_wavb_134.bufr.ref
 311001.bufr.ref
 uegabe.bufr.num.ref
 syno.bufr.out.ref

definitions/add_params_from_tsv.pl

@@ -105,38 +105,65 @@
 if ($SANITY_CHECK) {
     my %map_sn = ();   # map of shortNames
     my %map_pid = ();  # map of paramIds
+    my $sanity_error_count = 0;
     print "Checking sanity: uniqueness of paramId and shortName keys ...\n";
     while (<>) {
         chomp;
+        $lcount++;
         s/\r//g;  # Remove DOS carriage returns
         if ($first == 1) {
             $first = 0;
             next;
         }
-        $lcount++;
+
         ($paramId, $shortName, $name, $units) = split(/\t/);
 
-        die "Error: shortName=$shortName is duplicated (line ", $lcount+1, ")\n" if (exists $map_sn{$shortName});
+        die "Error: shortName=$shortName is duplicated (line $lcount)\n" if (exists $map_sn{$shortName});
         $map_sn{$shortName}++; # increment count in shortName map
 
-        die "Error: paramId=$paramId is duplicated (line ", $lcount+1, ")\n" if (exists $map_pid{$paramId});
+        die "Error: paramId=$paramId is duplicated (line $lcount)\n" if (exists $map_pid{$paramId});
         $map_pid{$paramId}++; # increment count in paramId map
 
-        die "Error: paramId=$paramId is not an integer (line ", $lcount+1, ")\n" if (!is_integer($paramId));
+        if (!is_integer($paramId)) {
+            warn "Error: paramId=$paramId is not an integer (line $lcount)\n";
+            $sanity_error_count++;
+        }
 
         my $x = $dbh->selectrow_array("select * from param.param where id = ?",undef,$paramId);
-        die "Error: paramId=$x exists in the database (line ", $lcount+1, ")\n" if (defined $x);
-
-        die "Error: Name '$name': ends in space" if ($name =~ / $/);
-        die "Error: Name '$name': starts with space" if ($name =~ /^ /);
+        if (defined $x) {
+            warn "Error: paramId=$x exists in the database (line $lcount)\n";
+            $sanity_error_count++;
+        }
 
-        # Will die if it fails
-        get_db_units_code($units);
+        if ($name =~ / $/) {
+            warn "Error: Name '$name': ends in space" ;
+            $sanity_error_count++;
+        }
+        if ($name =~ /^ /) {
+            warn "Error: Name '$name': starts with space" ;
+            $sanity_error_count++;
+        }
+        if ($name !~ /^[A-Z0-9]/) {
+            warn "Error: name \"$name\" should have uppercase 1st letter or digit (line $lcount)\n";
+            $sanity_error_count++;
+        }
 
         $x = $dbh->selectrow_array("select shortName from param.param where shortName = ?",undef,$shortName);
-        die "Error: shortName=$x exists in the database (line ", $lcount+1, ")\n" if (defined $x);
+        if (defined $x) {
+            warn "Error: shortName=$x exists in the database (line $lcount)\n";
+            $sanity_error_count++;
+        }
+
+        if (!check_units($units)) {
+            warn "Error: Database does not contain units=$units (line $lcount)\n";
+            $sanity_error_count++;
+        }
+    }
+    if ($sanity_error_count == 0) {
+        print "\nSanity checking completed. $lcount rows checked. No errors.\n";
+    } else {
+        die "\nSanity checking FAILED. $lcount rows checked. $sanity_error_count error(s).\n";
     }
-    print "\nSanity checking completed. $lcount rows checked. No errors.\nExiting.\n";
     exit 0;
 }
 
@@ -298,6 +325,13 @@ sub centre_as_str {
     return "ECMWF" if ($cc eq $centre_ecmwf);
     return "Unknown";
 }
+sub check_units {
+    my $u = shift;
+    my $unit_id = $dbh->selectrow_array("select id from units where name = ?",undef,$u);
+    return 0 if (!$unit_id);
+    return 1;
+}
+
 sub get_db_units_code {
     my $u = shift;
     my $unit_id = $dbh->selectrow_array("select id from units where name = ?",undef,$u);

definitions/grib1/section.1.def

@@ -137,8 +137,6 @@ codetable[1] timeRangeIndicator ('5.table',tablesLocalDir,tablesMasterDir) = 1 :
 
 unsigned[2] numberIncludedInAverage;
 
-meta mybits bits(numberIncludedInAverage,0,12);
-
 unsigned[1] numberMissingFromAveragesOrAccumulations;
 unsigned[1] centuryOfReferenceTimeOfData;
 

definitions/grib2/cfName.def

@@ -190,6 +190,18 @@
 	 scaledValueOfSecondFixedSurface = missing() ;
 	 scaleFactorOfSecondFixedSurface = missing() ;
 	}
+#Heat flux correction
+'heat_flux_correction' = {
+	 discipline = 10 ;
+	 parameterCategory = 3 ;
+	 parameterNumber = 18 ;
+	 typeOfFirstFixedSurface = 160 ;
+	 typeOfSecondFixedSurface = 255 ;
+	 scaledValueOfFirstFixedSurface = 0 ;
+	 scaleFactorOfFirstFixedSurface = 0 ;
+	 scaledValueOfSecondFixedSurface = missing() ;
+	 scaleFactorOfSecondFixedSurface = missing() ;
+	}
 #Eastward sea water velocity
 'eastward_sea_water_velocity' = {
 	 discipline = 10 ;