From d064296f9634b4e24a8f9d3650040dc7bd804b0d Mon Sep 17 00:00:00 2001 From: PatrickGoRaft Date: Mon, 9 Dec 2024 16:02:59 -0500 Subject: [PATCH] collaborative updates from the HMDA Dev team to resolve CVEs and platform bugs --- .../lar/parser/ModifiedLarCsvParser.scala | 1 + .../src/test/resources/modifiedlar.sql | 667 ++++++++++++++++++ .../lar/parser/ModifiedLarCsvParserSpec.scala | 6 +- 3 files changed, 671 insertions(+), 3 deletions(-) diff --git a/modified-lar/src/main/scala/hmda/publication/lar/parser/ModifiedLarCsvParser.scala b/modified-lar/src/main/scala/hmda/publication/lar/parser/ModifiedLarCsvParser.scala index 684fbb99fc..e4ea38c0e1 100644 --- a/modified-lar/src/main/scala/hmda/publication/lar/parser/ModifiedLarCsvParser.scala +++ b/modified-lar/src/main/scala/hmda/publication/lar/parser/ModifiedLarCsvParser.scala @@ -305,6 +305,7 @@ private def getOverallLoanLimit(year: Int) = { case 2020 => overallLoanLimit2020 case 2021 => overallLoanLimit2021 case 2022 => overallLoanLimit2022 + case 2023 => overallLoanLimit2023 case 2024 => overallLoanLimit2024 case 2025 => overallLoanLimit2025 diff --git a/modified-lar/src/test/resources/modifiedlar.sql b/modified-lar/src/test/resources/modifiedlar.sql index e825fed215..9014489ee5 100644 --- a/modified-lar/src/test/resources/modifiedlar.sql +++ b/modified-lar/src/test/resources/modifiedlar.sql @@ -1,3 +1,670 @@ +CREATE TABLE modifiedlar2025 ( + id integer NOT NULL, + uli character varying not null, + lei character varying NOT NULL, + loan_type integer, + loan_purpose integer, + preapproval integer, + construction_method character varying, + occupancy_type integer, + loan_amount character varying, + action_taken_type integer, + state character varying, + county character varying, + tract character varying, + ethnicity_applicant_1 character varying, + ethnicity_applicant_2 character varying, + ethnicity_applicant_3 character varying, + ethnicity_applicant_4 character varying, + ethnicity_applicant_5 character varying, + ethnicity_observed_applicant integer, + ethnicity_co_applicant_1 character varying, + ethnicity_co_applicant_2 character varying, + ethnicity_co_applicant_3 character varying, + ethnicity_co_applicant_4 character varying, + ethnicity_co_applicant_5 character varying, + ethnicity_observed_co_applicant integer, + race_applicant_1 character varying, + race_applicant_2 character varying, + race_applicant_3 character varying, + race_applicant_4 character varying, + race_applicant_5 character varying, + race_co_applicant_1 character varying, + race_co_applicant_2 character varying, + race_co_applicant_3 character varying, + race_co_applicant_4 character varying, + race_co_applicant_5 character varying, + race_observed_applicant integer, + race_observed_co_applicant integer, + sex_applicant integer, + sex_co_applicant integer, + observed_sex_applicant integer, + observed_sex_co_applicant integer, + age_applicant character varying, + applicant_age_greater_than_62 character varying, + age_co_applicant character varying, + coapplicant_age_greater_than_62 character varying, + income character varying, + purchaser_type integer, + rate_spread character varying, + hoepa_status integer, + lien_status integer, + credit_score_type_applicant integer, + credit_score_type_co_applicant integer, + denial_reason1 integer, + denial_reason2 integer, + denial_reason3 integer, + denial_reason4 integer, + total_loan_costs character varying, + total_points character varying, + origination_charges character varying, + discount_points character varying, + lender_credits character varying, + interest_rate character varying, + payment_penalty character varying, + debt_to_incode character varying, + loan_value_ratio character varying, + loan_term character varying, + rate_spread_intro character varying, + baloon_payment integer, + insert_only_payment integer, + amortization integer, + other_amortization integer, + property_value character varying, + home_security_policy integer, + lan_property_interest integer, + total_units character varying, + mf_affordable character varying, + application_submission integer, + payable integer, + aus1 integer, + aus2 integer, + aus3 integer, + aus4 integer, + aus5 integer, + reverse_mortgage integer, + line_of_credits integer, + business_or_commercial integer, + population character varying, + minority_population_percent character varying, + ffiec_med_fam_income character varying, + tract_to_msamd character varying, + owner_occupied_units character varying, + one_to_four_fam_units character varying, + msa_md integer, + created_at timestamp without time zone DEFAULT CURRENT_TIMESTAMP, + submission_id character varying, + msa_md_name character varying, + filing_year integer, + conforming_loan_limit character varying, + median_age integer, + median_age_calculated character varying, + median_income_percentage integer, + race_categorization character varying, + sex_categorization character varying, + ethnicity_categorization character varying, + percent_median_msa_income character varying, + dwelling_category character varying, + loan_product_type character varying, + action_taken_date integer, + checksum character varying +); +CREATE TABLE modifiedlar2024 ( + id integer NOT NULL, + uli character varying not null, + lei character varying NOT NULL, + loan_type integer, + loan_purpose integer, + preapproval integer, + construction_method character varying, + occupancy_type integer, + loan_amount character varying, + action_taken_type integer, + state character varying, + county character varying, + tract character varying, + ethnicity_applicant_1 character varying, + ethnicity_applicant_2 character varying, + ethnicity_applicant_3 character varying, + ethnicity_applicant_4 character varying, + ethnicity_applicant_5 character varying, + ethnicity_observed_applicant integer, + ethnicity_co_applicant_1 character varying, + ethnicity_co_applicant_2 character varying, + ethnicity_co_applicant_3 character varying, + ethnicity_co_applicant_4 character varying, + ethnicity_co_applicant_5 character varying, + ethnicity_observed_co_applicant integer, + race_applicant_1 character varying, + race_applicant_2 character varying, + race_applicant_3 character varying, + race_applicant_4 character varying, + race_applicant_5 character varying, + race_co_applicant_1 character varying, + race_co_applicant_2 character varying, + race_co_applicant_3 character varying, + race_co_applicant_4 character varying, + race_co_applicant_5 character varying, + race_observed_applicant integer, + race_observed_co_applicant integer, + sex_applicant integer, + sex_co_applicant integer, + observed_sex_applicant integer, + observed_sex_co_applicant integer, + age_applicant character varying, + applicant_age_greater_than_62 character varying, + age_co_applicant character varying, + coapplicant_age_greater_than_62 character varying, + income character varying, + purchaser_type integer, + rate_spread character varying, + hoepa_status integer, + lien_status integer, + credit_score_type_applicant integer, + credit_score_type_co_applicant integer, + denial_reason1 integer, + denial_reason2 integer, + denial_reason3 integer, + denial_reason4 integer, + total_loan_costs character varying, + total_points character varying, + origination_charges character varying, + discount_points character varying, + lender_credits character varying, + interest_rate character varying, + payment_penalty character varying, + debt_to_incode character varying, + loan_value_ratio character varying, + loan_term character varying, + rate_spread_intro character varying, + baloon_payment integer, + insert_only_payment integer, + amortization integer, + other_amortization integer, + property_value character varying, + home_security_policy integer, + lan_property_interest integer, + total_units character varying, + mf_affordable character varying, + application_submission integer, + payable integer, + aus1 integer, + aus2 integer, + aus3 integer, + aus4 integer, + aus5 integer, + reverse_mortgage integer, + line_of_credits integer, + business_or_commercial integer, + population character varying, + minority_population_percent character varying, + ffiec_med_fam_income character varying, + tract_to_msamd character varying, + owner_occupied_units character varying, + one_to_four_fam_units character varying, + msa_md integer, + created_at timestamp without time zone DEFAULT CURRENT_TIMESTAMP, + submission_id character varying, + msa_md_name character varying, + filing_year integer, + conforming_loan_limit character varying, + median_age integer, + median_age_calculated character varying, + median_income_percentage integer, + race_categorization character varying, + sex_categorization character varying, + ethnicity_categorization character varying, + percent_median_msa_income character varying, + dwelling_category character varying, + loan_product_type character varying, + action_taken_date integer, + checksum character varying +); +CREATE TABLE modifiedlar2023 ( + id integer NOT NULL, + uli character varying not null, + lei character varying NOT NULL, + loan_type integer, + loan_purpose integer, + preapproval integer, + construction_method character varying, + occupancy_type integer, + loan_amount character varying, + action_taken_type integer, + state character varying, + county character varying, + tract character varying, + ethnicity_applicant_1 character varying, + ethnicity_applicant_2 character varying, + ethnicity_applicant_3 character varying, + ethnicity_applicant_4 character varying, + ethnicity_applicant_5 character varying, + ethnicity_observed_applicant integer, + ethnicity_co_applicant_1 character varying, + ethnicity_co_applicant_2 character varying, + ethnicity_co_applicant_3 character varying, + ethnicity_co_applicant_4 character varying, + ethnicity_co_applicant_5 character varying, + ethnicity_observed_co_applicant integer, + race_applicant_1 character varying, + race_applicant_2 character varying, + race_applicant_3 character varying, + race_applicant_4 character varying, + race_applicant_5 character varying, + race_co_applicant_1 character varying, + race_co_applicant_2 character varying, + race_co_applicant_3 character varying, + race_co_applicant_4 character varying, + race_co_applicant_5 character varying, + race_observed_applicant integer, + race_observed_co_applicant integer, + sex_applicant integer, + sex_co_applicant integer, + observed_sex_applicant integer, + observed_sex_co_applicant integer, + age_applicant character varying, + applicant_age_greater_than_62 character varying, + age_co_applicant character varying, + coapplicant_age_greater_than_62 character varying, + income character varying, + purchaser_type integer, + rate_spread character varying, + hoepa_status integer, + lien_status integer, + credit_score_type_applicant integer, + credit_score_type_co_applicant integer, + denial_reason1 integer, + denial_reason2 integer, + denial_reason3 integer, + denial_reason4 integer, + total_loan_costs character varying, + total_points character varying, + origination_charges character varying, + discount_points character varying, + lender_credits character varying, + interest_rate character varying, + payment_penalty character varying, + debt_to_incode character varying, + loan_value_ratio character varying, + loan_term character varying, + rate_spread_intro character varying, + baloon_payment integer, + insert_only_payment integer, + amortization integer, + other_amortization integer, + property_value character varying, + home_security_policy integer, + lan_property_interest integer, + total_units character varying, + mf_affordable character varying, + application_submission integer, + payable integer, + aus1 integer, + aus2 integer, + aus3 integer, + aus4 integer, + aus5 integer, + reverse_mortgage integer, + line_of_credits integer, + business_or_commercial integer, + population character varying, + minority_population_percent character varying, + ffiec_med_fam_income character varying, + tract_to_msamd character varying, + owner_occupied_units character varying, + one_to_four_fam_units character varying, + msa_md integer, + created_at timestamp without time zone DEFAULT CURRENT_TIMESTAMP, + submission_id character varying, + msa_md_name character varying, + filing_year integer, + conforming_loan_limit character varying, + median_age integer, + median_age_calculated character varying, + median_income_percentage integer, + race_categorization character varying, + sex_categorization character varying, + ethnicity_categorization character varying, + percent_median_msa_income character varying, + dwelling_category character varying, + loan_product_type character varying, + action_taken_date integer, + 1 checksum character varying +); +CREATE TABLE modifiedlar2022 ( + id integer NOT NULL, + uli character varying not null, + lei character varying NOT NULL, + loan_type integer, + loan_purpose integer, + preapproval integer, + construction_method character varying, + occupancy_type integer, + loan_amount character varying, + action_taken_type integer, + state character varying, + county character varying, + tract character varying, + ethnicity_applicant_1 character varying, + ethnicity_applicant_2 character varying, + ethnicity_applicant_3 character varying, + ethnicity_applicant_4 character varying, + ethnicity_applicant_5 character varying, + ethnicity_observed_applicant integer, + ethnicity_co_applicant_1 character varying, + ethnicity_co_applicant_2 character varying, + ethnicity_co_applicant_3 character varying, + ethnicity_co_applicant_4 character varying, + ethnicity_co_applicant_5 character varying, + ethnicity_observed_co_applicant integer, + race_applicant_1 character varying, + race_applicant_2 character varying, + race_applicant_3 character varying, + race_applicant_4 character varying, + race_applicant_5 character varying, + race_co_applicant_1 character varying, + race_co_applicant_2 character varying, + race_co_applicant_3 character varying, + race_co_applicant_4 character varying, + race_co_applicant_5 character varying, + race_observed_applicant integer, + race_observed_co_applicant integer, + sex_applicant integer, + sex_co_applicant integer, + observed_sex_applicant integer, + observed_sex_co_applicant integer, + age_applicant character varying, + applicant_age_greater_than_62 character varying, + age_co_applicant character varying, + coapplicant_age_greater_than_62 character varying, + income character varying, + purchaser_type integer, + rate_spread character varying, + hoepa_status integer, + lien_status integer, + credit_score_type_applicant integer, + credit_score_type_co_applicant integer, + denial_reason1 integer, + denial_reason2 integer, + denial_reason3 integer, + denial_reason4 integer, + total_loan_costs character varying, + total_points character varying, + origination_charges character varying, + discount_points character varying, + lender_credits character varying, + interest_rate character varying, + payment_penalty character varying, + debt_to_incode character varying, + loan_value_ratio character varying, + loan_term character varying, + rate_spread_intro character varying, + baloon_payment integer, + insert_only_payment integer, + amortization integer, + other_amortization integer, + property_value character varying, + home_security_policy integer, + lan_property_interest integer, + total_units character varying, + mf_affordable character varying, + application_submission integer, + payable integer, + aus1 integer, + aus2 integer, + aus3 integer, + aus4 integer, + aus5 integer, + reverse_mortgage integer, + line_of_credits integer, + business_or_commercial integer, + population character varying, + minority_population_percent character varying, + ffiec_med_fam_income character varying, + tract_to_msamd character varying, + owner_occupied_units character varying, + one_to_four_fam_units character varying, + msa_md integer, + created_at timestamp without time zone DEFAULT CURRENT_TIMESTAMP, + submission_id character varying, + msa_md_name character varying, + filing_year integer, + conforming_loan_limit character varying, + median_age integer, + median_age_calculated character varying, + median_income_percentage integer, + race_categorization character varying, + sex_categorization character varying, + ethnicity_categorization character varying, + percent_median_msa_income character varying, + dwelling_category character varying, + loan_product_type character varying, + action_taken_date integer, + checksum character varying +); +CREATE TABLE modifiedlar2021 ( + id integer NOT NULL, + uli character varying not null, + lei character varying NOT NULL, + loan_type integer, + loan_purpose integer, + preapproval integer, + construction_method character varying, + occupancy_type integer, + loan_amount character varying, + action_taken_type integer, + state character varying, + county character varying, + tract character varying, + ethnicity_applicant_1 character varying, + ethnicity_applicant_2 character varying, + ethnicity_applicant_3 character varying, + ethnicity_applicant_4 character varying, + ethnicity_applicant_5 character varying, + ethnicity_observed_applicant integer, + ethnicity_co_applicant_1 character varying, + ethnicity_co_applicant_2 character varying, + ethnicity_co_applicant_3 character varying, + ethnicity_co_applicant_4 character varying, + ethnicity_co_applicant_5 character varying, + ethnicity_observed_co_applicant integer, + race_applicant_1 character varying, + race_applicant_2 character varying, + race_applicant_3 character varying, + race_applicant_4 character varying, + race_applicant_5 character varying, + race_co_applicant_1 character varying, + race_co_applicant_2 character varying, + race_co_applicant_3 character varying, + race_co_applicant_4 character varying, + race_co_applicant_5 character varying, + race_observed_applicant integer, + race_observed_co_applicant integer, + sex_applicant integer, + sex_co_applicant integer, + observed_sex_applicant integer, + observed_sex_co_applicant integer, + age_applicant character varying, + applicant_age_greater_than_62 character varying, + age_co_applicant character varying, + coapplicant_age_greater_than_62 character varying, + income character varying, + purchaser_type integer, + rate_spread character varying, + hoepa_status integer, + lien_status integer, + credit_score_type_applicant integer, + credit_score_type_co_applicant integer, + denial_reason1 integer, + denial_reason2 integer, + denial_reason3 integer, + denial_reason4 integer, + total_loan_costs character varying, + total_points character varying, + origination_charges character varying, + discount_points character varying, + lender_credits character varying, + interest_rate character varying, + payment_penalty character varying, + debt_to_incode character varying, + loan_value_ratio character varying, + loan_term character varying, + rate_spread_intro character varying, + baloon_payment integer, + insert_only_payment integer, + amortization integer, + other_amortization integer, + property_value character varying, + home_security_policy integer, + lan_property_interest integer, + total_units character varying, + mf_affordable character varying, + application_submission integer, + payable integer, + aus1 integer, + aus2 integer, + aus3 integer, + aus4 integer, + aus5 integer, + reverse_mortgage integer, + line_of_credits integer, + business_or_commercial integer, + population character varying, + minority_population_percent character varying, + ffiec_med_fam_income character varying, + tract_to_msamd character varying, + owner_occupied_units character varying, + one_to_four_fam_units character varying, + msa_md integer, + created_at timestamp without time zone DEFAULT CURRENT_TIMESTAMP, + submission_id character varying, + msa_md_name character varying, + filing_year integer, + conforming_loan_limit character varying, + median_age integer, + median_age_calculated character varying, + median_income_percentage integer, + race_categorization character varying, + sex_categorization character varying, + ethnicity_categorization character varying, + percent_median_msa_income character varying, + dwelling_category character varying, + loan_product_type character varying, + action_taken_date integer, + checksum character varying +); +CREATE TABLE modifiedlar2020 ( + + id integer NOT NULL, + uli character varying not null, + lei character varying NOT NULL, + loan_type integer, + loan_purpose integer, + preapproval integer, + construction_method character varying, + occupancy_type integer, + loan_amount character varying, + action_taken_type integer, + state character varying, + county character varying, + tract character varying, + ethnicity_applicant_1 character varying, + ethnicity_applicant_2 character varying, + ethnicity_applicant_3 character varying, + ethnicity_applicant_4 character varying, + ethnicity_applicant_5 character varying, + ethnicity_observed_applicant integer, + ethnicity_co_applicant_1 character varying, + ethnicity_co_applicant_2 character varying, + ethnicity_co_applicant_3 character varying, + ethnicity_co_applicant_4 character varying, + ethnicity_co_applicant_5 character varying, + ethnicity_observed_co_applicant integer, + race_applicant_1 character varying, + race_applicant_2 character varying, + race_applicant_3 character varying, + race_applicant_4 character varying, + race_applicant_5 character varying, + race_co_applicant_1 character varying, + race_co_applicant_2 character varying, + race_co_applicant_3 character varying, + race_co_applicant_4 character varying, + race_co_applicant_5 character varying, + race_observed_applicant integer, + race_observed_co_applicant integer, + sex_applicant integer, + sex_co_applicant integer, + observed_sex_applicant integer, + observed_sex_co_applicant integer, + age_applicant character varying, + applicant_age_greater_than_62 character varying, + age_co_applicant character varying, + coapplicant_age_greater_than_62 character varying, + income character varying, + purchaser_type integer, + rate_spread character varying, + hoepa_status integer, + lien_status integer, + credit_score_type_applicant integer, + credit_score_type_co_applicant integer, + denial_reason1 integer, + denial_reason2 integer, + denial_reason3 integer, + denial_reason4 integer, + total_loan_costs character varying, + total_points character varying, + origination_charges character varying, + discount_points character varying, + lender_credits character varying, + interest_rate character varying, + payment_penalty character varying, + debt_to_incode character varying, + loan_value_ratio character varying, + loan_term character varying, + rate_spread_intro character varying, + baloon_payment integer, + insert_only_payment integer, + amortization integer, + other_amortization integer, + property_value character varying, + home_security_policy integer, + lan_property_interest integer, + total_units character varying, + mf_affordable character varying, + application_submission integer, + payable integer, + aus1 integer, + aus2 integer, + aus3 integer, + aus4 integer, + aus5 integer, + reverse_mortgage integer, + line_of_credits integer, + business_or_commercial integer, + population character varying, + minority_population_percent character varying, + ffiec_med_fam_income character varying, + tract_to_msamd character varying, + owner_occupied_units character varying, + one_to_four_fam_units character varying, + msa_md integer, + created_at timestamp without time zone DEFAULT CURRENT_TIMESTAMP, + submission_id character varying, + msa_md_name character varying, + filing_year integer, + conforming_loan_limit character varying, + median_age integer, + median_age_calculated character varying, + median_income_percentage integer, + race_categorization character varying, + sex_categorization character varying, + ethnicity_categorization character varying, + percent_median_msa_income character varying, + dwelling_category character varying, + loan_product_type character varying, + action_taken_date integer, + checksum character varying +); CREATE TABLE modifiedlar2019 ( id integer NOT NULL, uli character varying not null, diff --git a/modified-lar/src/test/scala/hmda/publication/lar/parser/ModifiedLarCsvParserSpec.scala b/modified-lar/src/test/scala/hmda/publication/lar/parser/ModifiedLarCsvParserSpec.scala index 3df50a0ea8..e04409a7a2 100644 --- a/modified-lar/src/test/scala/hmda/publication/lar/parser/ModifiedLarCsvParserSpec.scala +++ b/modified-lar/src/test/scala/hmda/publication/lar/parser/ModifiedLarCsvParserSpec.scala @@ -9,7 +9,7 @@ class ModifiedLarCsvParserSpec extends WordSpec with ScalaCheckPropertyChecks wi "Parses LAR entries" in { forAll(larGen) { lar => val mlarFn = ModifiedLarCsvParser(lar.toCSV, _) - (2018 to 2023).foreach(year => mlarFn(year).lei shouldBe lar.larIdentifier.LEI) + (2018 to 2025).foreach(year => mlarFn(year).lei shouldBe lar.larIdentifier.LEI) } } @@ -17,7 +17,7 @@ class ModifiedLarCsvParserSpec extends WordSpec with ScalaCheckPropertyChecks wi forAll(larGen) { lar => val larWithChangedAge = lar.copy(applicant = lar.applicant.copy(age = 8888)) val mlarFn = ModifiedLarCsvParser(larWithChangedAge.toCSV, _) - (2018 to 2023).foreach { year => + (2018 to 2025).foreach { year => mlarFn(year).lei shouldBe lar.larIdentifier.LEI mlarFn(year).ageGreaterThanOrEqual62 shouldBe "NA" } @@ -64,7 +64,7 @@ class ModifiedLarCsvParserSpec extends WordSpec with ScalaCheckPropertyChecks wi forAll(larGen) { lar => val larWithChangedAge = lar.copy(property = lar.property.copy(totalUnits = 150)) val mlarFn = ModifiedLarCsvParser(larWithChangedAge.toCSV, _) - (2018 to 2023).foreach { year => + (2018 to 2025).foreach { year => mlarFn(year).lei shouldBe lar.larIdentifier.LEI mlarFn(year).totalUnits shouldBe ">149" }