From dfa58dbb55da1758f07ee7c3d7e83963f0be3dbd Mon Sep 17 00:00:00 2001 From: Graham Stark Date: Wed, 11 Dec 2024 07:06:29 +0000 Subject: [PATCH] 2 new blog posts --- oldsrc/HouseholdMappingFRS-no-hbai-notused.jl | 1454 ----------------- oldsrc/HouseholdMappingFRS_HBAI.jl | 665 -------- oldsrc/Incomes.jl.using-constants | 989 ----------- scripts/create-scottish-la-weights.jl | 12 +- 4 files changed, 6 insertions(+), 3114 deletions(-) delete mode 100644 oldsrc/HouseholdMappingFRS-no-hbai-notused.jl delete mode 100644 oldsrc/HouseholdMappingFRS_HBAI.jl delete mode 100644 oldsrc/Incomes.jl.using-constants diff --git a/oldsrc/HouseholdMappingFRS-no-hbai-notused.jl b/oldsrc/HouseholdMappingFRS-no-hbai-notused.jl deleted file mode 100644 index 4267d8d7..00000000 --- a/oldsrc/HouseholdMappingFRS-no-hbai-notused.jl +++ /dev/null @@ -1,1454 +0,0 @@ -# -# This contains most of the functions used to create our model dataset from raw FRS/SHS/HBAI data. -# -# This is for 2015-2019 FRS/HBAI; you likely need to alter this by hand -# as years are added. -# -# -using DataFrames -using CSV - -using ScottishTaxBenefitModel -using .Utils -using .Definitions -using .Randoms: mybigrandstr -using .GeneralTaxComponents: RateBands, WEEKS_PER_YEAR - -export CreateData - -const MONTHS = Dict( - "JAN" => 1, - "FEB" => 2, - "MAR" => 3, - "APR" => 4, - "MAY" => 5, - "JUN" => 6, - "JUL" => 7, - "AUG" => 8, - "SEP" => 9, - "OCT" => 10, - "NOV" => 11, - "DEC" => 12 ) - -""" -hacky hack to hack AA, etc into l/m/h, sometimes without the m -""" -function map123( v :: Union{Missing,Real}, amts :: Vector ) :: Integer - n = size(amts)[1] - @assert n in [1,2] - # println(n) - r = -1 - if (! ismissing(v)) && v > 0 - if n == 1 - r = v <= amts[1] ? 1 : 3 - else - # println("r=$r") - if v <= amts[1] - r = 1 - elseif v <= amts[2] - r = 2 - else - r = 3 - end - end - end - @assert r in [-1,1,2,3] - return r -end - -""" -hacky hack to hack PIP, etc into l/h -""" -function map12( v :: Union{Missing,Real}, amt :: Real ) :: Integer - r = -1 - if (! ismissing(v)) && v > 0 - r = v <= amt ? 1 : 2 - end - @assert r in [-1,1,2] - return r -end - -# -# @returns ns for the JSType enum -1=no 1=cont, 2=income 3=mixed -# -function make_jsa_type( frs_res::DataFrame, sernum :: Integer, benunit :: Integer, head :: Bool )::Tuple - ad_frs = frs_res[((frs_res.sernum.==sernum ).& - (frs_res.benunit.==benunit)), [:jsatyphd,:jsatypsp,:esatyphd,:esatypsp]] - @assert size( ad_frs )[1] .== 1 - af = ad_frs[1,:] - jsa = head ? af.jsatyphd : af.jsatypsp - # fixme refactor - jtype = -1 - if jsa == -1 - jtype = -1 - elseif jsa in [1,3] - jtype = 1 - elseif jsa in [2,4] - jtype = 2 - elseif jsa in [5,6] - jtype = 3 - else - @assert false "$jsa not mapped" - end - etype = -1 - esa = head ? af.esatyphd : af.esatypsp - if esa == -1 - etype = -1 - elseif esa in [1,3] - etype = 1 - elseif esa in [2,4] - etype = 2 - elseif esa in [5,6] - etype = 3 - else - @assert false "$esa not mapped" - end - return( jtype, etype ) - - - - # see benefits PDF file - # 1 = Contributory - # 2 = Income Based - # 3 = Contributory (Imputed) - # 4 = Income Based (Imputed) - # 5 = Both contributory and income based - # 6 = Both contributory and income based (Imputed) - -end - -# -# BU head on HBAI classification. Near dup of `get_incs_from_hbai` below. -# -function is_bu_head( - sernum :: Integer, - benunit :: Integer, - person :: Integer ) :: Bool - ad_hbai = hbai_res[((hbai_res.sernum.==sernum ).& - ((hbai_res.personhd.==person).|(hbai_res.personsp.==person)) .& - (hbai_res.benunit.==benunit)), :] - @assert size( ad_hbai )[1] > 0 - ar = ad_hbai[1,:] - return ar.personhd == person -end - - -function initialise_person(n::Integer)::DataFrame - pers = DataFrame( - data_year = Vector{Union{Int64,Missing}}(missing, n), - hid = Vector{Union{BigInt,Missing}}(missing, n), - pid = Vector{Union{BigInt,Missing}}(missing, n), - pno = Vector{Union{Integer,Missing}}(missing, n), - is_hrp = Vector{Union{Integer,Missing}}(missing, n), - - from_child_record = Vector{Union{Integer,Missing}}(missing, n), - default_benefit_unit = Vector{Union{Integer,Missing}}(missing, n), - age = Vector{Union{Integer,Missing}}(missing, n), - sex = Vector{Union{Integer,Missing}}(missing, n), - ethnic_group = Vector{Union{Integer,Missing}}(missing, n), - marital_status = Vector{Union{Integer,Missing}}(missing, n), - highest_qualification = Vector{Union{Integer,Missing}}(missing, n), - sic = Vector{Union{Integer,Missing}}(missing, n), - occupational_classification = Vector{Union{Integer,Missing}}(missing, n), - public_or_private = Vector{Union{Integer,Missing}}(missing, n), - principal_employment_type = Vector{Union{Integer,Missing}}(missing, n), - socio_economic_grouping = Vector{Union{Integer,Missing}}(missing, n), - age_completed_full_time_education = Vector{Union{Integer,Missing}}(missing, n), - years_in_full_time_work = Vector{Union{Integer,Missing}}(missing, n), - employment_status = Vector{Union{Integer,Missing}}(missing, n), - usual_hours_worked = Vector{Union{Real,Missing}}(missing, n), - actual_hours_worked = Vector{Union{Real,Missing}}(missing, n), - - - - age_started_first_job = Vector{Union{Real,Missing}}(missing, n), - # for widow's benefits - type_of_bereavement_allowance = Vector{Union{Real,Missing}}(missing, n), - had_children_when_bereaved = Vector{Union{Real,Missing}}(missing, n), - - pay_includes_ssp = Vector{Union{Integer,Missing}}(missing, n), - pay_includes_smp = Vector{Union{Integer,Missing}}(missing, n), - pay_includes_spp = Vector{Union{Integer,Missing}}(missing, n), - pay_includes_sap = Vector{Union{Integer,Missing}}(missing, n), - pay_includes_mileage = Vector{Union{Integer,Missing}}(missing, n), - pay_includes_motoring_expenses = Vector{Union{Integer,Missing}}(missing, n), - - income_wages = Vector{Union{Real,Missing}}(missing, n), - income_self_employment_income = Vector{Union{Real,Missing}}(missing, n), - income_self_employment_expenses = Vector{Union{Real,Missing}}(missing, n), - income_self_employment_losses = Vector{Union{Real,Missing}}(missing, n), - income_odd_jobs = Vector{Union{Real,Missing}}(missing, n), # FIXME UNUSED - income_private_pensions = Vector{Union{Real,Missing}}(missing, n), - income_national_savings = Vector{Union{Real,Missing}}(missing, n), - income_bank_interest = Vector{Union{Real,Missing}}(missing, n), - income_stocks_shares = Vector{Union{Real,Missing}}(missing, n), - income_individual_savings_account = Vector{Union{Real,Missing}}(missing, n), - # income_dividends = Vector{Union{Real,Missing}}(missing, n), # FIXME not used needs deleted use stocks_shares instead - income_property = Vector{Union{Real,Missing}}(missing, n), - income_royalties = Vector{Union{Real,Missing}}(missing, n), - income_bonds_and_gilts = Vector{Union{Real,Missing}}(missing, n), - income_other_investment_income = Vector{Union{Real,Missing}}(missing, n), - income_other_income = Vector{Union{Real,Missing}}(missing, n), - income_alimony_and_child_support_received = Vector{Union{Real,Missing}}(missing, n), - income_health_insurance = Vector{Union{Real,Missing}}(missing, n), - income_alimony_and_child_support_paid = Vector{Union{Real,Missing}}(missing, n), - income_care_insurance = Vector{Union{Real,Missing}}(missing, n), - income_trade_unions_etc = Vector{Union{Real,Missing}}(missing, n), - income_friendly_societies = Vector{Union{Real,Missing}}(missing, n), - income_work_expenses = Vector{Union{Real,Missing}}(missing, n), - income_avcs = Vector{Union{Real,Missing}}(missing, n), - income_other_deductions = Vector{Union{Real,Missing}}(missing, n), - income_loan_repayments = Vector{Union{Real,Missing}}(missing, n), - income_student_loan_repayments = Vector{Union{Real,Missing}}(missing, n), - income_pension_contributions_employer = Vector{Union{Real,Missing}}(missing, n), - income_pension_contributions_employee = Vector{Union{Real,Missing}}(missing, n), - income_education_allowances = Vector{Union{Real,Missing}}(missing, n), - income_foster_care_payments = Vector{Union{Real,Missing}}(missing, n), - income_student_grants = Vector{Union{Real,Missing}}(missing, n), - income_student_loans = Vector{Union{Real,Missing}}(missing, n), - income_income_tax = Vector{Union{Real,Missing}}(missing, n), - income_national_insurance = Vector{Union{Real,Missing}}(missing, n), - income_local_taxes = Vector{Union{Real,Missing}}(missing, n), - income_free_school_meals = Vector{Union{Real,Missing}}(missing, n), - income_dlaself_care = Vector{Union{Real,Missing}}(missing, n), - income_dlamobility = Vector{Union{Real,Missing}}(missing, n), - income_child_benefit = Vector{Union{Real,Missing}}(missing, n), - income_pension_credit = Vector{Union{Real,Missing}}(missing, n), - income_state_pension = Vector{Union{Real,Missing}}(missing, n), - income_bereavement_allowance_or_widowed_parents_allowance_or_bereavement = Vector{Union{ - Real, - Missing - }}( - missing, - n - ), - income_armed_forces_compensation_scheme = Vector{Union{Real,Missing}}(missing, n), - income_war_widows_or_widowers_pension = Vector{Union{Real,Missing}}(missing, n), - income_severe_disability_allowance = Vector{Union{Real,Missing}}(missing, n), - income_attendance_allowance = Vector{Union{Real,Missing}}(missing, n), - income_carers_allowance = Vector{Union{Real,Missing}}(missing, n), - income_jobseekers_allowance = Vector{Union{Real,Missing}}(missing, n), - income_industrial_injury_disablement_benefit = Vector{Union{Real,Missing}}( - missing, - n - ), - income_employment_and_support_allowance = Vector{Union{Real,Missing}}(missing, n), - income_incapacity_benefit = Vector{Union{Real,Missing}}(missing, n), - income_income_support = Vector{Union{Real,Missing}}(missing, n), - income_maternity_allowance = Vector{Union{Real,Missing}}(missing, n), - income_maternity_grant_from_social_fund = Vector{Union{Real,Missing}}(missing, n), - income_funeral_grant_from_social_fund = Vector{Union{Real,Missing}}(missing, n), - income_any_other_ni_or_state_benefit = Vector{Union{Real,Missing}}(missing, n), - income_trade_union_sick_or_strike_pay = Vector{Union{Real,Missing}}(missing, n), - income_friendly_society_benefits = Vector{Union{Real,Missing}}(missing, n), - income_private_sickness_scheme_benefits = Vector{Union{Real,Missing}}(missing, n), - income_accident_insurance_scheme_benefits = Vector{Union{Real,Missing}}(missing, n), - income_hospital_savings_scheme_benefits = Vector{Union{Real,Missing}}(missing, n), - income_government_training_allowances = Vector{Union{Real,Missing}}(missing, n), - income_guardians_allowance = Vector{Union{Real,Missing}}(missing, n), - income_widows_payment = Vector{Union{Real,Missing}}(missing, n), - income_unemployment_or_redundancy_insurance = Vector{Union{Real,Missing}}( - missing, - n - ), - income_winter_fuel_payments = Vector{Union{Real,Missing}}(missing, n), - income_dwp_third_party_payments_is_or_pc = Vector{Union{Real,Missing}}(missing, n), - income_dwp_third_party_payments_jsa_or_esa = Vector{Union{Real,Missing}}( - missing, - n - ), - income_social_fund_loan_repayment_from_is_or_pc = Vector{Union{Real,Missing}}( - missing, - n - ), - income_social_fund_loan_repayment_from_jsa_or_esa = Vector{Union{Real,Missing}}( - missing, - n - ), - income_extended_hb = Vector{Union{Real,Missing}}(missing, n), - income_permanent_health_insurance = Vector{Union{Real,Missing}}(missing, n), - income_any_other_sickness_insurance = Vector{Union{Real,Missing}}(missing, n), - income_critical_illness_cover = Vector{Union{Real,Missing}}(missing, n), - income_working_tax_credit = Vector{Union{Real,Missing}}(missing, n), - income_child_tax_credit = Vector{Union{Real,Missing}}(missing, n), - income_working_tax_credit_lump_sum = Vector{Union{Real,Missing}}(missing, n), - income_child_tax_credit_lump_sum = Vector{Union{Real,Missing}}(missing, n), - income_housing_benefit = Vector{Union{Real,Missing}}(missing, n), - income_universal_credit = Vector{Union{Real,Missing}}(missing, n), - income_personal_independence_payment_daily_living = Vector{Union{Real,Missing}}( - missing, - n - ), - income_personal_independence_payment_mobility = Vector{Union{Real,Missing}}( - missing, - n - ), - income_a_loan_from_the_dwp_and_dfc = Vector{Union{Real,Missing}}(missing, n), - income_a_loan_or_grant_from_local_authority = Vector{Union{Real,Missing}}( - missing, - n - ), - income_social_fund_loan_uc = Vector{Union{Real,Missing}}(missing, n), - income_other_benefits = Vector{Union{Real,Missing}}(missing, n), - - wages_frs = Vector{Union{Real,Missing}}(missing, n), - self_emp_frs = Vector{Union{Real,Missing}}(missing, n), - wages_hbai = Vector{Union{Real,Missing}}(missing, n), - self_emp_hbai = Vector{Union{Real,Missing}}(missing, n), - - jsa_type = Vector{Union{Integer,Missing}}(missing, n), - esa_type = Vector{Union{Integer,Missing}}(missing, n), - dlaself_care_type = Vector{Union{Integer,Missing}}(missing, n), - dlamobility_type = Vector{Union{Integer,Missing}}(missing, n), - attendance_allowance_type = Vector{Union{Integer,Missing}}(missing, n), - personal_independence_payment_daily_living_type = Vector{Union{Integer,Missing}}(missing, n), - personal_independence_payment_mobility_type = Vector{Union{Integer,Missing}}(missing, n), - - over_20_k_saving = - Vector{Union{Integer,Missing}}( - missing, n), - asset_current_account = Vector{Union{Real,Missing}}(missing, n), - asset_nsb_ordinary_account = Vector{Union{Real,Missing}}(missing, n), - asset_nsb_investment_account = Vector{Union{Real,Missing}}(missing, n), - asset_not_used = Vector{Union{Real,Missing}}(missing, n), - asset_savings_investments_etc = Vector{Union{Real,Missing}}(missing, n), - asset_government_gilt_edged_stock = Vector{Union{Real,Missing}}(missing, n), - asset_unit_or_investment_trusts = Vector{Union{Real,Missing}}(missing, n), - asset_stocks_shares_bonds_etc = Vector{Union{Real,Missing}}(missing, n), - asset_pep = Vector{Union{Real,Missing}}(missing, n), - asset_national_savings_capital_bonds = Vector{Union{Real,Missing}}(missing, n), - asset_index_linked_national_savings_certificates = Vector{Union{Real,Missing}}( - missing, - n - ), - asset_fixed_interest_national_savings_certificates = Vector{Union{Real,Missing}}( - missing, - n - ), - asset_pensioners_guaranteed_bonds = Vector{Union{Real,Missing}}(missing, n), - asset_saye = Vector{Union{Real,Missing}}(missing, n), - asset_premium_bonds = Vector{Union{Real,Missing}}(missing, n), - asset_national_savings_income_bonds = Vector{Union{Real,Missing}}(missing, n), - asset_national_savings_deposit_bonds = Vector{Union{Real,Missing}}(missing, n), - asset_first_option_bonds = Vector{Union{Real,Missing}}(missing, n), - asset_yearly_plan = Vector{Union{Real,Missing}}(missing, n), - asset_isa = Vector{Union{Real,Missing}}(missing, n), - asset_fixd_rate_svngs_bonds_or_grntd_incm_bonds_or_grntd_growth_bonds = Vector{Union{ - Real, - Missing - }}( - missing, - n - ), - asset_geb = Vector{Union{Real,Missing}}(missing, n), - asset_basic_account = Vector{Union{Real,Missing}}(missing, n), - asset_credit_unions = Vector{Union{Real,Missing}}(missing, n), - asset_endowment_policy_not_linked = Vector{Union{Real,Missing}}(missing, n), - contracted_out_of_serps = Vector{Union{Integer,Missing}}(missing, n), - registered_blind = Vector{Union{Integer,Missing}}(missing, n), - registered_partially_sighted = Vector{Union{Integer,Missing}}(missing, n), - registered_deaf = Vector{Union{Integer,Missing}}(missing, n), - disability_vision = Vector{Union{Integer,Missing}}(missing, n), - disability_hearing = Vector{Union{Integer,Missing}}(missing, n), - disability_mobility = Vector{Union{Integer,Missing}}(missing, n), - disability_dexterity = Vector{Union{Integer,Missing}}(missing, n), - disability_learning = Vector{Union{Integer,Missing}}(missing, n), - disability_memory = Vector{Union{Integer,Missing}}(missing, n), - disability_mental_health = Vector{Union{Integer,Missing}}(missing, n), - disability_stamina = Vector{Union{Integer,Missing}}(missing, n), - disability_socially = Vector{Union{Integer,Missing}}(missing, n), - disability_other_difficulty = Vector{Union{Integer,Missing}}(missing, n), - health_status = Vector{Union{Integer,Missing}}(missing, n), - - has_long_standing_illness = Vector{Union{Integer,Missing}}(missing, n), - adls_are_reduced = Vector{Union{Integer,Missing}}(missing, n), - how_long_adls_reduced = Vector{Union{Integer,Missing}}(missing, n), - - is_informal_carer = Vector{Union{Integer,Missing}}(missing, n), - receives_informal_care_from_non_householder = Vector{Union{Integer,Missing}}( - missing, - n - ), - hours_of_care_received = Vector{Union{Real,Missing}}(missing, n), - hours_of_care_given = Vector{Union{Real,Missing}}(missing, n), - hours_of_childcare = Vector{Union{Real,Missing}}(missing, n), - cost_of_childcare = Vector{Union{Real,Missing}}(missing, n), - childcare_type = Vector{Union{Integer,Missing}}(missing, n), - employer_provides_child_care = Vector{Union{Integer,Missing}}(missing, n), - - - - company_car_fuel_type = Vector{Union{Integer,Missing}}(missing, n), - company_car_value = Vector{Union{Real,Missing}}(missing, n), - company_car_contribution = Vector{Union{Real,Missing}}(missing, n), - fuel_supplied = Vector{Union{Real,Missing}}(missing, n), - - relationship_to_hoh = Vector{Union{Integer,Missing}}(missing, n), - relationship_1 = Vector{Union{Integer,Missing}}(missing, n), - relationship_2 = Vector{Union{Integer,Missing}}(missing, n), - relationship_3 = Vector{Union{Integer,Missing}}(missing, n), - relationship_4 = Vector{Union{Integer,Missing}}(missing, n), - relationship_5 = Vector{Union{Integer,Missing}}(missing, n), - relationship_6 = Vector{Union{Integer,Missing}}(missing, n), - relationship_7 = Vector{Union{Integer,Missing}}(missing, n), - relationship_8 = Vector{Union{Integer,Missing}}(missing, n), - relationship_9 = Vector{Union{Integer,Missing}}(missing, n), - relationship_10 = Vector{Union{Integer,Missing}}(missing, n), - relationship_11 = Vector{Union{Integer,Missing}}(missing, n), - relationship_12 = Vector{Union{Integer,Missing}}(missing, n), - relationship_13 = Vector{Union{Integer,Missing}}(missing, n), - relationship_14 = Vector{Union{Integer,Missing}}(missing, n), - relationship_15 = Vector{Union{Integer,Missing}}(missing, n), - onerand = Vector{String}(undef,n) - ) - -end - -const HH_TYPE_HINTS = [ - :region => Standard_Region, - :ct_band => CT_Band, - :tenure => Tenure_Type -] - - - -function initialise_household(n::Integer)::DataFrame - # .. example check - # FIXME change all VectorUnion to fill(0,n) - # select value,count(value),label from dictionaries.enums where dataset='frs' and tables='househol' and variable_name='hhcomps' group by value,label; - return DataFrame( - data_year = Vector{Union{Integer,Missing}}(missing, n), - interview_year = Vector{Union{Integer,Missing}}(missing, n), - interview_month = Vector{Union{Integer,Missing}}(missing, n), - quarter = Vector{Union{Integer,Missing}}(missing, n), - hid = Vector{Union{BigInt,Missing}}(missing, n), - tenure = Vector{Union{Integer,Missing}}(missing, n), - region = Vector{Union{Integer,Missing}}(missing, n), - ct_band = Vector{Union{Integer,Missing}}(missing, n), - council_tax = Vector{Union{Real,Missing}}(missing, n), - water_and_sewerage = Vector{Union{Real,Missing}}(missing, n), - mortgage_payment = Vector{Union{Real,Missing}}(missing, n), - mortgage_interest = Vector{Union{Real,Missing}}(missing, n), - years_outstanding_on_mortgage = Vector{Union{Integer,Missing}}(missing, n), - mortgage_outstanding = Vector{Union{Real,Missing}}(missing, n), - year_house_bought = Vector{Union{Integer,Missing}}(missing, n), - gross_rent = Vector{Union{Real,Missing}}(missing, n), - rent_includes_water_and_sewerage = Vector{Union{Integer,Missing}}(missing, n), - other_housing_charges = Vector{Union{Real,Missing}}(missing, n), - gross_housing_costs = Vector{Union{Real,Missing}}(missing, n), - total_income = Vector{Union{Real,Missing}}(missing, n), - total_wealth = Vector{Union{Real,Missing}}(missing, n), - house_value = Vector{Union{Real,Missing}}(missing, n), - weight = Vector{Union{Real,Missing}}(missing, n), - council = fill( "", n ), - nhs_board = fill( "", n ), - bedrooms = fill( 0, n ), - onerand = Vector{String}(undef,n) - ) -end - -# -# the way this seems to work: if deduc1 in job record -# is > 0, the employee contrib here is set to -1 -# -function process_penprovs(a_pens::DataFrame)::Tuple - npens = size(a_pens)[1] - penconts_employer = 0.0 - penconts_employee = 0.0 - for p in 1:npens - pen = a_pens[p,:] - pc = safe_inc(0.0, pen.penamt) - if pen.penamtpd == 95 - pc /= 52.0 - end - if pen.pencon in [1,4,5] # ish ... - penconts_employee += pc - elseif pen.pencon == 2 # employer - penconts_employer += pc - elseif pen.pencon == 3 # oth employer and employee - penconts_employer += pc/2 - penconts_employee += pc/2 - end - end - # FIXME something about SERPS - (penconts_employee,penconts_employer) -end - -function process_pensions(a_pens::DataFrame)::NamedTuple - npens = size(a_pens)[1] - private_pension = 0.0 - tax = 0.0 - for p in 1:npens - private_pension = safe_inc(private_pension, a_pens[p, :penpay]) - private_pension = safe_inc(private_pension, a_pens[p, :ptamt]) # tax - private_pension = safe_inc(private_pension, a_pens[p, :penpd2]) # other deduction - tax = safe_inc(tax, a_pens[p, :ptamt]) - end - return (pension = private_pension, tax = tax) -end - -const NS_RATE = 0.01/WEEKS_PER_YEAR - -# | 2016 | accounts | nsamt | 1 | Jan-50 | Jan_50 | 0 -# | 2016 | accounts | nsamt | 2 | 51 - 100 | v_51_100 | 0 -# | 2016 | accounts | nsamt | 3 | 101 - 250 | v_101_250 | 0 -# | 2016 | accounts | nsamt | 4 | 251 - 500 | v_251_500 | 0 -# | 2016 | accounts | nsamt | 5 | 501 - 1000 | v_501_1000 | 0 -# | 2016 | accounts | nsamt | 6 | 1001 - 2000 | v_1001_2000 | 0 -# | 2016 | accounts | nsamt | 7 | 2001 - 3000 | v_2001_3000 | 0 -# | 2016 | accounts | nsamt | 8 | 3001 - 5000 | v_3001_5000 | 0 -# | 2016 | accounts | nsamt | 9 | 5001 - 10,000 | v_5001_10_000 | 0 -# | 2016 | accounts | nsamt | 10 | 10,001 - 20,000 | v_10_001_20_000 | 0 -# | 2016 | accounts | nsamt | 11 | 20,001 - 30,000 | v_20_001_30_000 | 0 -# | 2016 | accounts | nsamt | 12 | 30,001 or over | v_30_001_or_over | 0 -const NSAMT_ENUM_MIDPOINTS = [ - 25.0, - 75.0, - 175.0, - 375.0, - 750.0, - 1_500.0, - 2_500.0, - 4_000.0, - 7_500.0, - 15_000.0, - 25_000.0, - 40_000.0 -] - -""" -infer amounts from holdings (nsamt) assuming 1% pa interest rate -see: https://www.nsandi.com/historical-interest-rates for rates -why FRS records like this I have no idea -""" -function infer_national_savings_income( nsamt :: Integer )::Real - @assert ! (nsamt in [1:12]) "nsr out of range for enum: $nsamt" - NSAMT_ENUM_MIDPOINTS[ nsamt ]*NS_RATE -end - -function map_investment_income!(model_adult::DataFrameRow, accounts::DataFrame) - naccts = size(accounts)[1] - - model_adult.income_national_savings = 0.0 - model_adult.income_bank_interest = 0.0 - model_adult.income_stocks_shares = 0.0 - model_adult.income_individual_savings_account = 0.0 - model_adult.income_property = 0.0 - model_adult.income_royalties = 0.0 - model_adult.income_bonds_and_gilts = 0.0 - model_adult.income_other_investment_income = 0.0 - - - for i in 1:naccts - v = max(0.0, accounts[i, :accint]) # FIXME national savings stuff appears to be coded -1 for missing - if accounts[i, :invtax] == 1 - # FIXME is this right for dividends anymore? - v /= 0.8 - end - # FIXME building society - check with other models - # FIXME go over assignment to broad types against income - # tax book - atype = Account_Type(accounts[i, :account]) - nsamt = accounts[i, :nsamt] - # - # for national savings, amount held is recorded - # for the rest acctoint = interest pw from account - if nsamt > 0 - model_adult.income_national_savings += - infer_national_savings_income( nsamt ) # FIXME appears to be all zero! - elseif atype in [ - Current_account, - Basic_Account, - NSB_Investment_account, - NSB_Direct_Saver - - ] - model_adult.income_bank_interest += v - elseif atype in [ - National_Savings_capital_bonds, - Index_Linked_National_Savings_Certificates, - Fixed_Interest_National_Savings_Certificates, - National_Savings_income_bonds, - National_Savings_deposit_bonds - ] - ## this should never happen given, but does.. - # the weird way the FRS records National Savings as stocks - # nsamt should always be set for these records & handled above. - # @assert false - println( "atype = $atype but nsamt is $nsamt" ) - elseif atype in [ - Stocks_Shares_Bonds_etc, - Member_of_Share_Club] - model_adult.income_stocks_shares += v - elseif atype in [ISA] - model_adult.income_individual_savings_account += v - elseif atype in [ - SAYE, - Savings_investments_etc, - Unit_or_Investment_Trusts, - Endowment_Policy_Not_Linked, - Profit_sharing, - Credit_Unions, - Yearly_Plan, - Premium_bonds, - Company_Share_Option_Plans, - Post_Office_Card_Account, - Pensioners_Guaranteed_Bonds - ] - model_adult.income_other_investment_income += v - elseif atype in [ - Guaranteed_Equity_Bond, - Fixed_Rate_Savings_or_Guaranteed_Income_or_Guaranteed_Growth_Bonds, - First_Option_bonds, - Government_Gilt_Edged_Stock] - model_adult.income_bonds_and_gilts += v - else - @assert false "failed to map $atype" - end - end # accounts loop -end # map_investment_income - -function map_alimony(frs_person::DataFrameRow, a_maint::DataFrame)::Real - nmaints = size(a_maint)[1] - alimony = 0.0 # note: not including children - if frs_person.alimny == 1 # receives alimony - if frs_person.alius == 2 # not usual - alimony = safe_inc(0.0, frs_person.aluamt) - else - alimony = safe_inc(0.0, frs_person.aliamt) - end - end - for c in 1:nmaints - alimony = safe_inc(alimony, a_maint[c, :mramt]) - end - alimony -end - -function map_car_value( cv :: Integer ) :: Real - v = 0.0 - @assert cv <= 10 "cv out-of-range = $cv" - if cv < 0 - v = 0.0 - elseif cv == 1 - v = 5_000.0 - elseif cv == 2 - v = 11_500.0 - elseif cv == 3 - v = 14_500.0 - elseif cv == 4 - v = 17_500.0 - elseif cv == 5 - v = 20_500.0 - elseif cv == 6 - v = 23_500.0 - elseif cv == 7 - v = 27_500.0 - elseif cv == 8 - v = 35_000.0 - elseif cv == 9 - v = 45_000.0 - elseif cv == 10 - v = 20_000 # Don't_know = 10 - end - v -end - -""" -process the "r01..r014 and relhrp codes. Note we're adding 'this person' (=0) rather than missing as in the raw data" -""" -function process_relationships!( model_person :: DataFrameRow, frs_person :: DataFrameRow ) - relhh = safe_assign( frs_person.relhrp ) - if (relhh == -1 ) - relhh = 0 # map 'this person'; note hrp/head no longer needs to be 1 - end - model_person.relationship_to_hoh = relhh - for i in 1:14 - rel = i < 10 ? "r0" : "r" - relfrs = Symbol( "$(rel)$i" ) # :r10 or :r02 and so on - relmod = Symbol( "relationship_$(i)") # :relationship_10 or :relationship_2 - relp = safe_assign(frs_person[relfrs]) - if (frs_person.person == i) & (relp == -1) # again "this person = 0; makes mapping code (and just reading output) easier - relp = 0 - end - model_person[relmod] = relp - end -end - -function process_job_rec!(model_adult::DataFrameRow, a_job::DataFrame) - njobs = size(a_job)[1] - - earnings = 0.0 - actual_hours = 0.0 - usual_hours = 0.0 - health_insurance = 0.0 - alimony_and_child_support_paid = 0.0 - # care_insurance = 0.0 - trade_unions_etc = 0.0 - friendly_societies = 0.0 - work_expenses = 0.0 - pension_contributions_employee = 0.0 - avcs = 0.0 - other_deductions = 0.0 - student_loan_repayments = 0.0 - loan_repayments = 0.0 - self_employment_income = 0.0 - self_employment_expenses = 0.0 - self_employment_losses = 0.0 - tax = 0.0 - principal_employment_type = -1 - public_or_private = -1 - - company_car_fuel_type = 0 - company_car_value = 0.0 - company_car_contribution = 0.0 - fuel_supplied = 0.0 - - for j in 1:njobs - jb = a_job[j,:] # 1 row - if j == 1 # take 1st record job for all of these - principal_employment_type = safe_assign(jb.etype) - public_or_private = safe_assign(jb.jobsect) - end - usual_hours = safe_inc(usual_hours, jb.dvushr) - actual_hours = safe_inc(actual_hours, jb.jobhours) - - # alimony_and_child_support_paid = safe_inc( alimony_and_child_support_paid , a_job[j,udeduc0X]) - # care_insurance = safe_inc( care_insurance , jb.othded0X - # note these are *Usual* deductions - # "1... contribution *by you* to a Pension or superannuation scheme?" - # I *think* these contributions - pension_contributions_employee = safe_inc(pension_contributions_employee, jb.udeduc1) - avcs = safe_inc(avcs, jb.udeduc2) - trade_unions_etc = safe_inc(trade_unions_etc, jb.udeduc3) - friendly_societies = safe_inc(friendly_societies, jb.udeduc4) - other_deductions = safe_inc(other_deductions, jb.udeduc5) - loan_repayments = safe_inc(loan_repayments, jb.udeduc6) - health_insurance = safe_inc(health_insurance, jb.udeduc7) - other_deductions = safe_inc(other_deductions, jb.udeduc8) - student_loan_repayments = safe_inc(student_loan_repayments, jb.udeduc9) - work_expenses = safe_inc(work_expenses, jb.umotamt)# CARS FIXME add to this - - if jb.inclpay1 == 1 - model_adult.pay_includes_ssp = 1 - end - if jb.inclpay2 == 1 - model_adult.pay_includes_smp = 1 - end - # it refund .. 3 - if jb.inclpay4 == 1 - model_adult.pay_includes_mileage = 1 - end - if jb.inclpay5 == 1 - model_adult.pay_includes_motoring_expenses = 1 - end - if jb.inclpay6 == 1 - model_adult.pay_includes_spp = 1 - end - if jb.inclpay7 == 1 - model_adult.pay_includes_sap = 1 - end - - # self employment - if jb.prbefore > 0.0 - self_employment_income += jb.prbefore - elseif jb.profit1 > 0.0 - @assert jb.profit2 in [1, 2] - if jb.profit2 == 1 - self_employment_income += jb.profit1 - else - self_employment_losses += jb.profit1 - end - elseif jb.seincamt > 0.0 - self_employment_income += jb.seincamt - end - # setax = safe_inc(0.0, jb.setaxamt) - # tax += setax / 52.0 - - # earnings - addBonus = false - if jb.ugross > 0.0 # take usual when last not usual - earnings += jb.ugross - addBonus = true - elseif jb.grwage > 0.0 # then take last - earnings += jb.grwage - addBonus = true - elseif jb.ugrspay > 0.0 # then take total pay, but don't add bonuses - earnings += jb.ugrspay - end - if addBonus - for i in 1:6 - bon = Symbol(string("bonamt", i)) - tax = Symbol(string("bontax", i)) - if a_job[j, bon] > 0.0 - bon = a_job[j, bon] - if a_job[j, tax] == 2 - bon /= (1 - 0.22) # fixme hack basic rate - end - earnings += bon / 52.0 # fixwme weeks per year - end - end # bonuses loop - end # add bonuses - # cars - - company_car_fuel_type = jb.fueltyp - mv = map_car_value(jb.carval) - # println( mv ) - company_car_value = safe_inc(company_car_value, mv ) - company_car_contribution = safe_inc(company_car_contribution, jb.caramt) - fuel_supplied = safe_inc(fuel_supplied, jb.fuelamt) - - end # jobs loop - - model_adult.usual_hours_worked = usual_hours - model_adult.actual_hours_worked = actual_hours - model_adult.income_wages = earnings - model_adult.principal_employment_type = principal_employment_type - model_adult.public_or_private = public_or_private - ## FIXME look at this mapping again: pcodes - model_adult.income_health_insurance = health_insurance - # model_adult.income_# care_insurance = # care_insurance - model_adult.income_trade_unions_etc = trade_unions_etc - model_adult.income_friendly_societies = friendly_societies - model_adult.income_work_expenses = work_expenses - model_adult.income_pension_contributions_employee = pension_contributions_employee - model_adult.income_avcs = avcs - model_adult.income_other_deductions = other_deductions - model_adult.income_student_loan_repayments = student_loan_repayments # fixme maybe "slrepamt" or "slreppd" - model_adult.income_loan_repayments = loan_repayments # fixme maybe "slrepamt" or "slreppd" - - model_adult.income_self_employment_income = self_employment_income - model_adult.income_self_employment_expenses = self_employment_expenses - model_adult.income_self_employment_losses = self_employment_losses - - model_adult.company_car_fuel_type = company_car_fuel_type - model_adult.company_car_value = company_car_value - model_adult.company_car_contribution = company_car_contribution - model_adult.fuel_supplied = fuel_supplied - -end - -""" -Convoluted - take the benefit enum, make ... -FIXME: some represent one-off payments (winter fuel..) so maybe weeklyise, but all that -really matters is whether they are present -""" -function process_benefits!( model_adult::DataFrameRow, a_benefits::DataFrame) - nbens = size(a_benefits)[1] - for i in instances(Incomes_Type) - if i >= dlaself_care && i <= personal_independence_payment_mobility - ikey = make_sym_for_frame("income", i) - model_adult[ikey] = 0.0 - end - end - for b in 1:nbens - bno = a_benefits[b, :benefit] - if !(bno in [46, 47]) # 2015 receipt in last 6 months of tax credits - btype = Benefit_Type(bno) - # println( "bno=$bno BenefitType=$btype") - if btype <= Personal_Independence_Payment_Mobility - ikey = make_sym_for_frame("income", btype) - model_adult[ikey] = safe_inc(model_adult[ikey], a_benefits[b, :benamt]) - end - end - end -end - -""" -Convoluted - take the benefit enum, make ... -""" -function process_assets!(model_adult::DataFrameRow, an_asset::DataFrame) - nassets = size(an_asset)[1] - for i in instances(Asset_Type) - if (i > Missing_Asset_Type) - ikey = make_sym_for_asset(i) - model_adult[ikey] = 0.0 - end - end - for a in 1:nassets - ano = an_asset[a, :assetype] - atype = Asset_Type(ano) - ikey = make_sym_for_asset(atype) - v = an_asset[a, :howmuch] - if an_asset[a, :howmuche] > 0 - v = an_asset[a, :howmuche] - end - model_adult[ikey] = safe_inc(model_adult[ikey], v) - end -end - -function infer_hours_of_care(hourtot::Integer)::Real - hrs = Dict( - 0 => 0.0, - 1 => 2.0, - 2 => 7.0, - 3 => 14.0, - 4 => 27.5, - 5 => 42.5, - 6 => 75.0, - 7 => 100.0, - 8 => 10.0, - 9 => 27.5, - 10 => 50.0 - ) - h = 0.0 - if hourtot in keys(hrs) - h = hrs[hourtot] - end - h -end - -""" - remap child care type from pre-2017 version to 2017+ -""" -function map_child_care( year :: Integer, care ) :: Integer - if ismissing( care ) || care < -1 - care = -1 - end - if year >= 2017 - return care - end - if care > 0 # remap to2015/16 care to 2017+ - m = Dict( - 1=>1, - 2=>2, - 3=>3, - 4=>5, - 5=>4, - 6=>5, - 7=>4, - 8=>7, - 9=>8, - 10=>9, - 11=>10, - 12=>10, - 13=>11, - 14=>12, - 15=>13, - 16=>14, - 17=>15, - 18=>16, - 19=>17, - 20=>18 - ) - care = m[care] - end - care -end - -## FIXME add Oddjobs here for non-hbai case - -function create_adults( - year::Integer, - frs_adults::DataFrame, - accounts::DataFrame, - benunit::DataFrame, - extchild::DataFrame, - maint::DataFrame, - penprov::DataFrame, - # admin::DataFrame, - care::DataFrame, - mortcont::DataFrame, - pension::DataFrame, - govpay::DataFrame, - mortgage::DataFrame, - assets::DataFrame, - chldcare::DataFrame, - househol::DataFrame, - oddjob::DataFrame, - benefits::DataFrame, - endowmnt::DataFrame, - job::DataFrame, - frsx :: DataFrame )::DataFrame - - num_adults = size(frs_adults)[1] - adult_model = initialise_person(num_adults) - adno = 0 - for pn in 1:num_adults - if pn % 1000 == 0 - println("adults: on year $year, pno $pn") - end - - frs_person = frs_adults[pn, :] - sernum = frs_person.sernum - adno += 1 - model_adult = adult_model[adno, :] - model_adult.onerand = mybigrandstr() - ## also for children - model_adult.pno = frs_person.person - model_adult.hid = frs_person.sernum - model_adult.is_hrp = (frs_person.hrpid == 1) ? 1 : 0 - - model_adult.pid = get_pid(FRS, year, frs_person.sernum, frs_person.person) - model_adult.from_child_record = 0 - model_adult.data_year = year - model_adult.default_benefit_unit = frs_person.benunit - model_adult.age = frs_person.age80 - model_adult.sex = safe_assign(frs_person.sex) - model_adult.ethnic_group = safe_assign(frs_person.ethgr3) - - - - model_adult.jsa_type, model_adult.esa_type = make_jsa_type( - frsx, - frs_person.sernum, - frs_person.benunit, - hdsp ) - - # plan 'B' wages and SE from HBAI; first work out hd/spouse so we can extract right ones - # is_hbai_spouse = ( model_hbai.personsp == model_hbai.person ) - # is_hbai_head = ( model_hbai.personhd == model_hbai.person ) - # @assert is_hbai_head || is_hbai_spouse "neither head nor spouse" - - ## adult only - a_job = job[((job.sernum.==frs_person.sernum).&(job.benunit.==frs_person.benunit).&(job.person.==frs_person.person)), :] - a_benunit = benunit[((frs_person.benunit .== benunit.benunit).&(frs_person.sernum.==benunit.sernum)),:] - a_benunit = a_benunit[1,:] - model_adult.over_20_k_saving = 0 - if hdsp - ts = safe_assign(a_benunit.totsav) - if ts >= 5 - model_adult.over_20_k_saving = 1 - end - end - # println( "model_adult.over_20_k_saving=$(model_adult.over_20_k_saving)") - - a_pension = pension[((pension.sernum.==frs_person.sernum).&(pension.benunit.==frs_person.benunit).&(pension.person.==frs_person.person)), :] - a_penprov = penprov[((penprov.sernum.==frs_person.sernum).&(penprov.benunit.==frs_person.benunit).&(penprov.person.==frs_person.person)), :] - an_asset = assets[((assets.sernum.==frs_person.sernum).&(assets.benunit.==frs_person.benunit).&(assets.person.==frs_person.person)), :] - an_account = accounts[((accounts.sernum.==frs_person.sernum).&(accounts.benunit.==frs_person.benunit).&(accounts.person.==frs_person.person)), :] - a_maint = maint[((maint.sernum.==frs_person.sernum).&(maint.benunit.==frs_person.benunit).&(maint.person.==frs_person.person)), :] - a_oddjob = oddjob[((oddjob.sernum.==frs_person.sernum).&(oddjob.benunit.==frs_person.benunit).&(oddjob.person.==frs_person.person)), :] - a_benefits = benefits[((benefits.sernum.==frs_person.sernum).&(benefits.benunit.==frs_person.benunit).&(benefits.person.==frs_person.person)), :] - npens = size(a_pension)[1] - nassets = size(an_asset)[1] - naaccounts = size(an_account)[1] - nojs = size(a_oddjob)[1] - - model_adult.marital_status = safe_assign(frs_person.marital) - model_adult.highest_qualification = safe_assign(frs_person.dvhiqual) - model_adult.sic = safe_assign(frs_person.sic) - - model_adult.socio_economic_grouping = safe_assign(Integer(trunc(frs_person.nssec))) - model_adult.age_completed_full_time_education = safe_assign(frs_person.tea) - model_adult.years_in_full_time_work = safe_inc(0, frs_person.ftwk) - model_adult.employment_status = safe_assign(frs_person.empstati) - model_adult.occupational_classification = safe_assign(frs_person.soc2010) - - process_job_rec!(model_adult, a_job) - # FIXME some duplication here - hbaidata = get_incs_from_hbai( - hbai_res, - frs_person.sernum, - frs_person.benunit, - frs_person.person ) # fixme probably only need to check sernum - println( "frs_person.person=$(frs_person.person) frs_person.sernum=$(frs_person.sernum) frs_person.benunit=$(frs_person.benunit)") - - println( "hbaidata=$(hbaidata)") - @assert model_adult.sex == hbaidata.sex - @assert model_adult.age == hbaidata.age - if( override_se_and_wage_with_hbai ) - model_adult.income_wages = hbaidata.wages - model_adult.income_self_employment_income = hbaidata.selfemp - model_adult.income_self_employment_losses = 0.0 - model_adult.income_self_employment_expenses = 0.0 - end - # - # new - assign a total earnings/se figure from both hbai and frs - # so we can compare the two. This is in reaction to the - # oddly low Gini/Palma when using HBAI/SPI'd earnings - # - model_adult.wages_hbai = hbaidata.wages - model_adult.self_emp_hbai = hbaidata.selfemp - model_adult.wages_frs = safe_inc( 0.0, frs_person.inearns ) - model_adult.self_emp_frs = safe_inc( 0.0, frs_person.incseo2 ) - - penstuff = process_pensions(a_pension) - model_adult.income_private_pensions = penstuff.pension - model_adult.income_income_tax += penstuff.tax - - # FIXME CHECK THIS - adding PENCONT and also from work pension contributions - double counting? - (employee,employer) = process_penprovs(a_penprov) - - model_adult.income_pension_contributions_employee = safe_inc( employee, model_adult.income_pension_contributions_employee ) - model_adult.income_pension_contributions_employer = safe_inc( employer, model_adult.income_pension_contributions_employer ) - - map_investment_income!(model_adult, an_account) - model_adult.income_property = safe_inc(0.0, frs_person.royyr1) - if frs_person.rentprof == 2 # it's a loss - model_adult.income_property *= -1 # a loss - end - model_adult.income_royalties = safe_inc(0.0, frs_person.royyr2) - model_adult.income_other_income = safe_inc(0.0, frs_person.royyr3) # sleeping partners - model_adult.income_other_income = safe_inc( - model_adult.income_other_income, - frs_person.royyr4 - ) # overseas pensions - # payments from charities, bbysitting .. - # model_adult.income_other_income = safe_inc( model_adult.income_other_income, frs_person.[x] - model_adult.income_alimony_and_child_support_received = map_alimony( - frs_person, - a_maint - ) - - model_adult.income_odd_jobs = 0.0 - for o in 1:nojs - model_adult.income_odd_jobs = safe_inc( - model_adult.income_odd_jobs, - a_oddjob[o, :ojamt] - ) - end - model_adult.income_odd_jobs /= 4.0 # since it's monthly - - ## TODO babysitting,chartities (secure version only??) - ## TODO alimony and childcare PAID ?? // 2015/6 only - ## TODO allowances from absent spouses apamt apdamt - - ## TODO income_education_allowances - - model_adult.income_foster_care_payments = max(0.0,coalesce(frs_person.allpd3,0.0)) - - - ## TODO income_student_grants - ## TODO income_student_loans - ## TODO income_income_tax - ## TODO income_national_insurance - ## TODO income_local_taxes - - process_benefits!(model_adult, a_benefits) - process_assets!(model_adult, an_asset) - - ## also for child - model_adult.registered_blind = (frs_person.spcreg1 == 1 ? 1 : 0) - model_adult.registered_partially_sighted = (frs_person.spcreg2 == 1 ? 1 : 0) - model_adult.registered_deaf = (frs_person.spcreg3 == 1 ? 1 : 0) - - model_adult.disability_vision = (frs_person.disd01 == 1 ? 1 : 0) # cdisd kids .. - model_adult.disability_hearing = (frs_person.disd02 == 1 ? 1 : 0) - model_adult.disability_mobility = (frs_person.disd03 == 1 ? 1 : 0) - model_adult.disability_dexterity = (frs_person.disd04 == 1 ? 1 : 0) - model_adult.disability_learning = (frs_person.disd05 == 1 ? 1 : 0) - model_adult.disability_memory = (frs_person.disd06 == 1 ? 1 : 0) - model_adult.disability_mental_health = (frs_person.disd07 == 1 ? 1 : 0) - model_adult.disability_stamina = (frs_person.disd08 == 1 ? 1 : 0) - model_adult.disability_socially = (frs_person.disd09 == 1 ? 1 : 0) - model_adult.disability_other_difficulty = (frs_person.disd10 == 1 ? 1 : 0) - - model_adult.has_long_standing_illness = (frs_person.health1 == 1 ? 1 : 0) - model_adult.how_long_adls_reduced = (frs_person.limitl < 0 ? -1 : frs_person.limitl) - model_adult.adls_are_reduced = (frs_person.condit < 0 ? -1 : frs_person.condit) # missings to 'not at all' - - model_adult.age_started_first_job = safe_assign( frs_person.jobbyr ) - # 2017/18 only - if year >= 2017 - model_adult.type_of_bereavement_allowance = safe_assign( frs_person.wid ) - end - model_adult.had_children_when_bereaved = safe_assign( frs_person.w2 ) - - # dindividual_savings_accountbility_other_difficulty = Vector{Union{Real,Missing}}(missing, n), - model_adult.health_status = safe_assign(frs_person.heathad) - model_adult.hours_of_care_received = safe_inc(0.0, frs_person.hourcare) - model_adult.hours_of_care_given = infer_hours_of_care(frs_person.hourtot) # also kid - - model_adult.is_informal_carer = (frs_person.carefl == 1 ? 1 : 0) # also kid - process_relationships!( model_adult, frs_person ) - # - # illness benefit levels - # See the note on this in docs/ - model_adult.dlaself_care_type = map123( model_adult.income_dlaself_care, [30, 60 ] ) - model_adult.dlamobility_type = map123(model_adult.income_dlamobility, [30] ) - model_adult.attendance_allowance_type = map123( model_adult.income_attendance_allowance, [65] ) - model_adult.personal_independence_payment_daily_living_type = map12( model_adult.income_personal_independence_payment_daily_living, 65 ) - model_adult.personal_independence_payment_mobility_type = map12( model_adult.income_personal_independence_payment_mobility, 30 ) - end # adult loop - println("final adno $adno") - return adult_model[1:adno, :] -end # proc create_adult - -# -# FIXME This doesn't drop children from hhls dropped from the HBAI; harmless but it confused me ... -# -function create_children( - year::Integer, - frs_children::DataFrame, - childcare::DataFrame, - benefits:: DataFrame -)::DataFrame - num_children = size(frs_children)[1] - child_model = initialise_person(num_children) - ccount = 0 - for chno in 1:num_children - if chno % 1000 == 0 - println("on year $year, chno $chno") - end - frs_person = frs_children[chno, :] - - a_childcare = childcare[((childcare.sernum.==frs_person.sernum).&(childcare.benunit.==frs_person.benunit).&(childcare.person.==frs_person.person)), :] - nchildcares = size(a_childcare)[1] - - sernum = frs_person.sernum - ccount += 1 - ## also for children - model_child = child_model[ccount, :] - - model_child.pno = frs_person.person - model_child.hid = frs_person.sernum - model_child.pid = get_pid(FRS, year, frs_person.sernum, frs_person.person) - model_child.from_child_record = 1 - - model_child.data_year = year - model_child.default_benefit_unit = frs_person.benunit - model_child.age = frs_person.age - model_child.sex = safe_assign(frs_person.sex) - # model_child.ethnic_group = safe_assign(frs_person.ethgr3) - ## also for child - # println( "frs_person.chlimitl='$(frs_person.chlimitl)'") - model_child.has_long_standing_illness = (frs_person.chealth1 == 1 ? 1 : 0) - model_child.how_long_adls_reduced = (frs_person.chlimitl < 0 ? -1 : frs_person.chlimitl) - model_child.adls_are_reduced = (frs_person.chcond < 0 ? -1 : frs_person.chcond) # missings to 'not at all' - model_child.over_20_k_saving = 0 - - model_child.registered_blind = (frs_person.spcreg1 == 1 ? 1 : 0) - model_child.registered_partially_sighted = (frs_person.spcreg2 == 1 ? 1 : 0) - model_child.registered_deaf = (frs_person.spcreg3 == 1 ? 1 : 0) - - model_child.disability_vision = (frs_person.cdisd01 == 1 ? 1 : 0) # cdisd kids .. - model_child.disability_hearing = (frs_person.cdisd02 == 1 ? 1 : 0) - model_child.disability_mobility = (frs_person.cdisd03 == 1 ? 1 : 0) - model_child.disability_dexterity = (frs_person.cdisd04 == 1 ? 1 : 0) - model_child.disability_learning = (frs_person.cdisd05 == 1 ? 1 : 0) - model_child.disability_memory = (frs_person.cdisd06 == 1 ? 1 : 0) - model_child.disability_mental_health = (frs_person.cdisd07 == 1 ? 1 : 0) - model_child.disability_stamina = (frs_person.cdisd08 == 1 ? 1 : 0) - model_child.disability_socially = (frs_person.cdisd09 == 1 ? 1 : 0) - # dindividual_savings_accountbility_other_difficulty = Vector{Union{Real,Missing}}(missing, n), - model_child.health_status = safe_assign(frs_person.heathch) - model_child.income_wages = safe_inc( 0.0, frs_person.chearns ) - model_child.income_other_investment_income = safe_inc( 0.0, frs_person.chsave ) - model_child.income_other_income = safe_inc( 0.0, frs_person.chrinc ) - model_child.income_free_school_meals = 0.0 - for t in [:fsbval,:fsfvval,:fsmlkval,:fsmval] - model_child.income_free_school_meals = safe_inc( model_child.income_free_school_meals, frs_person[t] ) - end - model_child.is_informal_carer = (frs_person.carefl == 1 ? 1 : 0) # also kid - process_relationships!( model_child, frs_person ) - # TODO education grants, all the other good child stuff EMA - - model_child.cost_of_childcare = 0.0 - model_child.hours_of_childcare = 0.0 - for c in 1:nchildcares - if c == 1 # type of care from 1st instance - model_child.childcare_type = - map_child_care( year, a_childcare[c, :chlook] ) - model_child.employer_provides_child_care = (a_childcare[c, :emplprov] == 2 ? - 1 : 0) - end - model_child.cost_of_childcare = safe_inc( - model_child.cost_of_childcare, - a_childcare[c, :chamt] - ) - model_child.hours_of_childcare = safe_inc( - model_child.hours_of_childcare, - a_childcare[c, :chhr] - ) - end # child care loop - model_child.onerand = mybigrandstr() - - # - # - # this is zero length - # a_oddjob = oddjob[((oddjob.sernum.==frs_person.sernum).&(oddjob.benunit.==frs_person.benunit).&(oddjob.person.==frs_person.person)), :] - # this isn't - a_benefits = benefits[((benefits.sernum.==frs_person.sernum).&(benefits.benunit.==frs_person.benunit).&(benefits.person.==frs_person.person)), :] - sb = size( a_benefits )[1] - println( "sb = $sb") - # @assert sb in [0,1] - process_benefits!( model_child, a_benefits ) - - end # chno loop - return child_model[1:ccount,:] # send them all back ... -end - -function create_household( - year::Integer, - frs_household::DataFrame, - renter::DataFrame, - mortgage::DataFrame, - mortcont::DataFrame, - owner::DataFrame )::DataFrame - - num_households = size(frs_household)[1] - hh_model = initialise_household(num_households) - hhno = 0 - - for hn in 1:num_households - if hn % 1000 == 0 - println("on year $year, hid $hn") - end - hh = frs_household[hn, :] - sernum = hh.sernum - ad1_hbai = hbai_res[(hbai_res.sernum.==hh.sernum), :][1,:] - hhno += 1 - dd = split(hh.intdate, "/") - hh_model[hhno, :interview_year] = parse(Int64, dd[3]) - interview_month = parse(Int8, dd[1]) - hh_model[hhno, :interview_month] = interview_month - hh_model[hhno, :quarter] = div(interview_month - 1, 3) + 1 - - hh_model[hhno, :hid] = sernum - hh_model[hhno, :data_year] = year - hh_model[hhno, :tenure] = hh.tentyp2 > 0 ? hh.tentyp2 : -1 - hh_model[hhno, :region] = hh.gvtregn > 0 ? hh.gvtregn : -1 - hh_model[hhno, :ct_band] = hh.ctband > 0 ? hh.ctband : -1 - hh_model[hhno, :weight] = hh.gross4 - # hh_model[hhno, :tenure] = hh.tentyp2 > 0 ? Tenure_Type(hh.tentyp2) : - # Missing_Tenure_Type - # hh_model[hhno, :region] = hh.gvtregn > 0 ? Standard_Region(hh.gvtregn) : - # Missing_Standard_Region - # hh_model[hhno, :ct_band] = hh.ctband > 0 ? CT_Band(hh.ctband) : Missing_CT_Band - # - # council_tax::Real - # FIXME this is rounded to £ - if hh_model[hhno, :region] == 299999999 # Scotland - hh_model[hhno, :water_and_sewerage] = safe_assign(ad1_hbai.cwathh) - elseif hh_model[hhno, :region] == 399999999 # Nireland - hh_model[hhno, :water_and_sewerage] = 0.0 # FIXME - else # - hh_model[hhno, :water_and_sewerage] = safe_assign(ad1_hbai.watsewhh) - end - # hh_model[hhno, :mortgage_payment] - hh_model[hhno, :mortgage_interest] = ad1_hbai.hbxmort - - # TODO - # years_outstanding_on_mortgage::Integer - # mortgage_outstanding::Real - # year_house_bought::Integer - # FIXME rounded to £1 - hh_model[hhno, :gross_rent] = max(0.0, hh.hhrent) # rentg Gross rent including Housing Benefit or rent Net amount of last rent payment - - rents = renter[(renter.sernum.==sernum), :] - nrents = size(rents)[1] - hh_model[hhno, :rent_includes_water_and_sewerage] = false - for r in 1:nrents - if (rents[r, :wsinc] in [1, 2, 3]) - hh_model[hhno, :rent_includes_water_and_sewerage] = true - end - end - ohc = 0.0 - ohc = safe_inc(ohc, hh.chrgamt1) - ohc = safe_inc(ohc, hh.chrgamt2) - ohc = safe_inc(ohc, hh.chrgamt3) - ohc = safe_inc(ohc, hh.chrgamt4) - ohc = safe_inc(ohc, hh.chrgamt5) - ohc = safe_inc(ohc, hh.chrgamt6) - ohc = safe_inc(ohc, hh.chrgamt7) - ohc = safe_inc(ohc, hh.chrgamt8) - ohc = safe_inc(ohc, hh.chrgamt9) - hh_model[hhno, :other_housing_charges] = ohc - hh_model[hhno, :bedrooms] = hh.bedroom6 - hh_model[hhno, :onerand] = mybigrandstr() - # TODO - # gross_housing_costs::Real - # total_income::Real - # total_wealth::Real - # house_value::Real - # people::People_Dict - end - hh_model[1:hhno, :] -end - - -function loadfrs(which::AbstractString, year::Integer)::DataFrame - filename = "$(FRS_DIR)/$(year)/tab/$(which).tab" - loadtoframe(filename) -end - -function create_data() - # - # every time I do this the HBAI has been re-arranged - # now, there's one file for the years 2015-2019; despite the name - # the variables we use aren't uprated. See: - # `5828_hbai_1920_harmonised_dataset_variables_guide.xlsx` - # in the doc bundle - # - - for year in 2015:2019 - print("on year $year ") - appendb = year > 2015 - y = year - 2000 - ystr = "$(y)$(y+1)" - # we only want this massive thing for a couple of - # benefit variables. - frsx = loadfrs( "frs$ystr", year ) - - accounts = loadfrs("accounts", year) - benunit = loadfrs("benunit", year) - extchild = loadfrs("extchild", year) - maint = loadfrs("maint", year) - penprov = loadfrs("penprov", year) - care = loadfrs("care", year) - mortcont = loadfrs("mortcont", year) - pension = loadfrs("pension", year) - adult = loadfrs("adult", year) - child = loadfrs("child", year) - govpay = loadfrs("govpay", year) - mortgage = loadfrs("mortgage", year) - assets = loadfrs("assets", year) - chldcare = loadfrs("chldcare", year) - househol = loadfrs("househol", year) - oddjob = loadfrs("oddjob", year) - rentcont = loadfrs("rentcont", year) - benefits = loadfrs("benefits", year) - endowmnt = loadfrs("endowmnt", year) - job = loadfrs("job", year) - owner = loadfrs("owner", year) - renter = loadfrs("renter", year) - # this years subset of HBAI - model_children_yr = create_children( - year, - child, - chldcare, - benefits ) - - # append!(model_people, model_children_yr) - model_adults_yr = create_adults( - year, - adult, - accounts, - benunit, - extchild, - maint, - penprov, - # admin, - care, - mortcont, - pension, - govpay, - mortgage, - assets, - chldcare, - househol, - oddjob, - benefits, - endowmnt, - job, - frsx ) - # append!(model_people, model_adults_yr) - model_households_yr = create_household( - year, - househol, - renter, - mortgage, - mortcont, - owner ) - # append!(model_households, model_households_yr) - println( "on year $year") - println( "hhlds") - CSV.write("$(MODEL_DATA_DIR)/model_households.tab", model_households_yr, delim = "\t", append=appendb) - println( "adults") - CSV.write("$(MODEL_DATA_DIR)/model_people.tab", model_adults_yr, delim = "\t", append=appendb) - println( "children") - CSV.write("$(MODEL_DATA_DIR)/model_people.tab", model_children_yr, delim = "\t", append=true) - end - # CSV.write("$(MODEL_DATA_DIR)model_households.tab", model_households, delim = "\t") - # CSV.write("$(MODEL_DATA_DIR)model_people.tab", model_people, delim = "\t") -end diff --git a/oldsrc/HouseholdMappingFRS_HBAI.jl b/oldsrc/HouseholdMappingFRS_HBAI.jl deleted file mode 100644 index 4a1cbeb9..00000000 --- a/oldsrc/HouseholdMappingFRS_HBAI.jl +++ /dev/null @@ -1,665 +0,0 @@ -# -# This contains most of the functions used to create our model dataset from raw FRS/SHS/HBAI data. -# -# This is for 2015-2019 FRS/HBAI; you likely need to alter this by hand -# as years are added. -# -# -using DataFrames -using CSV - -using ScottishTaxBenefitModel -using .Utils -using .Definitions -using .Randoms: mybigrandstr -using .GeneralTaxComponents: RateBands, WEEKS_PER_YEAR - -export CreateData - -include( "frs_hbai_creation_libs.jl") - -function is_in_hbai( - hbai_res :: DataFrame, - sernum::Integer, - benunit :: Integer, - person :: Integer ) :: Bool - - ad_hbai = hbai_res[((hbai_res.sernum.==sernum ).& - ((hbai_res.personhd.==person).|(hbai_res.personsp.==person)) .& - (hbai_res.benunit.==benunit)), :] - return size( ad_hbai )[1]>0 -end - - -function is_in_hbai( - hbai_res :: DataFrame, - sernum::Integer ) :: Bool - - ad_hbai = hbai_res[(hbai_res.sernum.==sernum ), :] - return size( ad_hbai )[1]>0 -end - -# -# BU head on HBAI classification. Near dup of `get_incs_from_hbai` below. -# -function is_bu_head( - hbai_res :: DataFrame, - sernum::Integer, - benunit :: Integer, - person :: Integer ) :: Bool - ad_hbai = hbai_res[((hbai_res.sernum.==sernum ).& - ((hbai_res.personhd.==person).|(hbai_res.personsp.==person)) .& - (hbai_res.benunit.==benunit)), :] - @assert size( ad_hbai )[1] > 0 - ar = ad_hbai[1,:] - return ar.personhd == person -end - -function get_incs_from_hbai( - hbai_res :: DataFrame, - sernum::Integer, - benunit :: Integer, - person :: Integer ) :: NamedTuple - - ad_hbai = hbai_res[((hbai_res.sernum.==sernum ).& - ((hbai_res.personhd.==person).|(hbai_res.personsp.==person)) .& - (hbai_res.benunit.==benunit)), :] - @assert size( ad_hbai )[1] > 0 - ar = ad_hbai[1,:] - if ar.personhd == person - return ( - age=safe_assign(ar.agehd,0.0), - sex=safe_assign(ar.sexhd,0.0), - wages=safe_assign(ar.esgjobhd,0.0), - selfemp=safe_assign(ar.esgrsehd,0.0)) - elseif ar.personsp == person - return ( - age=safe_assign(ar.agesp,0.0), - sex=safe_assign(ar.sexsp,0.0), - wages=safe_assign(ar.esgjobsp,0.0), - selfemp=safe_assign(ar.esgrsesp,0.0)) - else - @assert false "$person is neither head or spouse in hbai assignment; sernum=$sernum benunit=$benunit" - end -end - - -## FIXME add Oddjobs here for non-hbai case - -function create_adults( - year::Integer, - frs_adults::DataFrame, - accounts::DataFrame, - benunit::DataFrame, - extchild::DataFrame, - maint::DataFrame, - penprov::DataFrame, - # admin::DataFrame, - care::DataFrame, - mortcont::DataFrame, - pension::DataFrame, - govpay::DataFrame, - mortgage::DataFrame, - assets::DataFrame, - chldcare::DataFrame, - househol::DataFrame, - oddjob::DataFrame, - benefits::DataFrame, - endowmnt::DataFrame, - job::DataFrame, - hbai_res::DataFrame, - frsx :: DataFrame, - override_se_and_wage_with_hbai :: Bool = true )::DataFrame - - num_adults = size(frs_adults)[1] - adult_model = initialise_person(num_adults) - adno = 0 - hbai_year = year - 1993 - println("hbai_year $hbai_year") - for pn in 1:num_adults - if pn % 1000 == 0 - println("adults: on year $year, pno $pn") - end - - frs_person = frs_adults[pn, :] - sernum = frs_person.sernum - if is_in_hbai( - hbai_res, - frs_person.sernum, - frs_person.benunit, - frs_person.person ) # fixme probably only need to check sernum - adno += 1 - model_adult = adult_model[adno, :] - model_adult.onerand = mybigrandstr() - ## also for children - model_adult = adult_model[adno, :] - model_adult.pno = frs_person.person - model_adult.hid = frs_person.sernum - model_adult.is_hrp = (frs_person.hrpid == 1) ? 1 : 0 - - model_adult.pid = get_pid( FRS, year, frs_person.sernum, frs_person.person ) - model_adult.from_child_record = 0 - model_adult.data_year = year - model_adult.default_benefit_unit = frs_person.benunit - model_adult.age = frs_person.age80 - model_adult.sex = safe_assign(frs_person.sex) - model_adult.ethnic_group = safe_assign(frs_person.ethgr3) - - hdsp = is_bu_head( - hbai_res, - frs_person.sernum, - frs_person.benunit, - frs_person.person ) - - model_adult.jsa_type, model_adult.esa_type = make_jsa_type( - frsx, - frs_person.sernum, - frs_person.benunit, - hdsp ) - - # plan 'B' wages and SE from HBAI; first work out hd/spouse so we can extract right ones - # is_hbai_spouse = ( model_hbai.personsp == model_hbai.person ) - # is_hbai_head = ( model_hbai.personhd == model_hbai.person ) - # @assert is_hbai_head || is_hbai_spouse "neither head nor spouse" - - ## adult only - a_job = job[((job.sernum.==frs_person.sernum).&(job.benunit.==frs_person.benunit).&(job.person.==frs_person.person)), :] - a_benunit = benunit[((frs_person.benunit .== benunit.benunit).&(frs_person.sernum.==benunit.sernum)),:] - a_benunit = a_benunit[1,:] - model_adult.over_20_k_saving = 0 - if hdsp - ts = safe_assign(a_benunit.totsav) - if ts >= 5 - model_adult.over_20_k_saving = 1 - end - end - # println( "model_adult.over_20_k_saving=$(model_adult.over_20_k_saving)") - - a_pension = pension[((pension.sernum.==frs_person.sernum).&(pension.benunit.==frs_person.benunit).&(pension.person.==frs_person.person)), :] - a_penprov = penprov[((penprov.sernum.==frs_person.sernum).&(penprov.benunit.==frs_person.benunit).&(penprov.person.==frs_person.person)), :] - an_asset = assets[((assets.sernum.==frs_person.sernum).&(assets.benunit.==frs_person.benunit).&(assets.person.==frs_person.person)), :] - an_account = accounts[((accounts.sernum.==frs_person.sernum).&(accounts.benunit.==frs_person.benunit).&(accounts.person.==frs_person.person)), :] - a_maint = maint[((maint.sernum.==frs_person.sernum).&(maint.benunit.==frs_person.benunit).&(maint.person.==frs_person.person)), :] - a_oddjob = oddjob[((oddjob.sernum.==frs_person.sernum).&(oddjob.benunit.==frs_person.benunit).&(oddjob.person.==frs_person.person)), :] - a_benefits = benefits[((benefits.sernum.==frs_person.sernum).&(benefits.benunit.==frs_person.benunit).&(benefits.person.==frs_person.person)), :] - npens = size(a_pension)[1] - nassets = size(an_asset)[1] - naaccounts = size(an_account)[1] - nojs = size(a_oddjob)[1] - - model_adult.marital_status = safe_assign(frs_person.marital) - model_adult.highest_qualification = safe_assign(frs_person.dvhiqual) - model_adult.sic = safe_assign(frs_person.sic) - - model_adult.socio_economic_grouping = safe_assign(Integer(trunc(frs_person.nssec))) - model_adult.age_completed_full_time_education = safe_assign(frs_person.tea) - model_adult.years_in_full_time_work = safe_inc(0, frs_person.ftwk) - model_adult.employment_status = safe_assign(frs_person.empstati) - model_adult.occupational_classification = safe_assign(frs_person.soc2010) - - process_job_rec!(model_adult, a_job) - # FIXME some duplication here - hbaidata = get_incs_from_hbai( - hbai_res, - frs_person.sernum, - frs_person.benunit, - frs_person.person ) # fixme probably only need to check sernum - @assert model_adult.sex == hbaidata.sex - @assert model_adult.age == hbaidata.age - if( override_se_and_wage_with_hbai ) - model_adult.income_wages = hbaidata.wages - model_adult.income_self_employment_income = hbaidata.selfemp - model_adult.income_self_employment_losses = 0.0 - model_adult.income_self_employment_expenses = 0.0 - end - # - # new - assign a total earnings/se figure from both hbai and frs - # so we can compare the two. This is in reaction to the - # oddly low Gini/Palma when using HBAI/SPI'd earnings - # - model_adult.wages_hbai = hbaidata.wages - model_adult.self_emp_hbai = hbaidata.selfemp - model_adult.wages_frs = safe_inc( 0.0, frs_person.inearns ) - model_adult.self_emp_frs = safe_inc( 0.0, frs_person.incseo2 ) - - penstuff = process_pensions(a_pension) - model_adult.income_private_pensions = penstuff.pension - model_adult.income_income_tax += penstuff.tax - - # FIXME CHECK THIS - adding PENCONT and also from work pension contributions - double counting? - (employee,employer) = process_penprovs(a_penprov) - - model_adult.income_pension_contributions_employee = safe_inc( employee, model_adult.income_pension_contributions_employee ) - model_adult.income_pension_contributions_employer = safe_inc( employer, model_adult.income_pension_contributions_employer ) - - map_investment_income!(model_adult, an_account) - model_adult.income_property = safe_inc(0.0, frs_person.royyr1) - if frs_person.rentprof == 2 # it's a loss - model_adult.income_property *= -1 # a loss - end - model_adult.income_royalties = safe_inc(0.0, frs_person.royyr2) - model_adult.income_other_income = safe_inc(0.0, frs_person.royyr3) # sleeping partners - model_adult.income_other_income = safe_inc( - model_adult.income_other_income, - frs_person.royyr4 - ) # overseas pensions - # payments from charities, bbysitting .. - # model_adult.income_other_income = safe_inc( model_adult.income_other_income, frs_person.[x] - model_adult.income_alimony_and_child_support_recieved, - model_adult.income_alimony_and_child_support_paid = map_alimony( - frs_person, - a_maint - ) - - model_adult.income_odd_jobs = 0.0 - for o in 1:nojs - model_adult.income_odd_jobs = safe_inc( - model_adult.income_odd_jobs, - a_oddjob[o, :ojamt] - ) - end - model_adult.income_odd_jobs /= 4.0 # since it's monthly - - ## TODO babysitting,chartities (secure version only??) - ## TODO alimony and childcare PAID ?? // 2015/6 only - ## TODO allowances from absent spouses apamt apdamt - - ## TODO income_education_allowances - - model_adult.income_foster_care_payments = max(0.0,coalesce(frs_person.allpd3,0.0)) - - - ## TODO income_student_grants - ## TODO income_student_loans - ## TODO income_income_tax - ## TODO income_national_insurance - ## TODO income_local_taxes - - process_benefits!(model_adult, a_benefits) - process_assets!(model_adult, an_asset) - - ## also for child - model_adult.registered_blind = (frs_person.spcreg1 == 1 ? 1 : 0) - model_adult.registered_partially_sighted = (frs_person.spcreg2 == 1 ? 1 : 0) - model_adult.registered_deaf = (frs_person.spcreg3 == 1 ? 1 : 0) - - model_adult.disability_vision = (frs_person.disd01 == 1 ? 1 : 0) # cdisd kids .. - model_adult.disability_hearing = (frs_person.disd02 == 1 ? 1 : 0) - model_adult.disability_mobility = (frs_person.disd03 == 1 ? 1 : 0) - model_adult.disability_dexterity = (frs_person.disd04 == 1 ? 1 : 0) - model_adult.disability_learning = (frs_person.disd05 == 1 ? 1 : 0) - model_adult.disability_memory = (frs_person.disd06 == 1 ? 1 : 0) - model_adult.disability_mental_health = (frs_person.disd07 == 1 ? 1 : 0) - model_adult.disability_stamina = (frs_person.disd08 == 1 ? 1 : 0) - model_adult.disability_socially = (frs_person.disd09 == 1 ? 1 : 0) - model_adult.disability_other_difficulty = (frs_person.disd10 == 1 ? 1 : 0) - - model_adult.has_long_standing_illness = (frs_person.health1 == 1 ? 1 : 0) - model_adult.how_long_adls_reduced = (frs_person.limitl < 0 ? -1 : frs_person.limitl) - model_adult.adls_are_reduced = (frs_person.condit < 0 ? -1 : frs_person.condit) # missings to 'not at all' - - model_adult.age_started_first_job = safe_assign( frs_person.jobbyr ) - # 2017/18 only - if year >= 2017 - model_adult.type_of_bereavement_allowance = safe_assign( frs_person.wid ) - end - model_adult.had_children_when_bereaved = safe_assign( frs_person.w2 ) - - # dindividual_savings_accountbility_other_difficulty = Vector{Union{Real,Missing}}(missing, n), - model_adult.health_status = safe_assign(frs_person.heathad) - model_adult.hours_of_care_received = safe_inc(0.0, frs_person.hourcare) - model_adult.hours_of_care_given = infer_hours_of_care(frs_person.hourtot) # also kid - - model_adult.is_informal_carer = (frs_person.carefl == 1 ? 1 : 0) # also kid - process_relationships!( model_adult, frs_person ) - # - # illness benefit levels - # See the note on this in docs/ - model_adult.dlaself_care_type = map123( model_adult.income_dlaself_care, [30, 60 ] ) - model_adult.dlamobility_type = map123(model_adult.income_dlamobility, [30] ) - model_adult.attendance_allowance_type = map123( model_adult.income_attendance_allowance, [65] ) - model_adult.personal_independence_payment_daily_living_type = map12( model_adult.income_personal_independence_payment_daily_living, 65 ) - model_adult.personal_independence_payment_mobility_type = map12( model_adult.income_personal_independence_payment_mobility, 30 ) - end # if in HBAI - end # adult loop - println("final adno $adno") - return adult_model[1:adno, :] -end # proc create_adult - -# -# FIXME This doesn't drop children from hhls dropped from the HBAI; harmless but it confused me ... -# -function create_children( - year::Integer, - frs_children::DataFrame, - childcare::DataFrame, - hbai_res::DataFrame, - benefits:: DataFrame -)::DataFrame - num_children = size(frs_children)[1] - child_model = initialise_person(num_children) - ccount = 0 - for chno in 1:num_children - if chno % 1000 == 0 - println("on year $year, chno $chno") - end - frs_person = frs_children[chno, :] - if is_in_hbai( hbai_res, frs_person.sernum ) - - a_childcare = childcare[((childcare.sernum.==frs_person.sernum).&(childcare.benunit.==frs_person.benunit).&(childcare.person.==frs_person.person)), :] - nchildcares = size(a_childcare)[1] - - sernum = frs_person.sernum - ccount += 1 - ## also for children - model_child = child_model[ccount, :] - - model_child.pno = frs_person.person - model_child.hid = frs_person.sernum - model_child.pid = get_pid(FRS, year, frs_person.sernum, frs_person.person) - model_child.from_child_record = 1 - - model_child.data_year = year - model_child.default_benefit_unit = frs_person.benunit - model_child.age = frs_person.age - model_child.sex = safe_assign(frs_person.sex) - # model_child.ethnic_group = safe_assign(frs_person.ethgr3) - ## also for child - # println( "frs_person.chlimitl='$(frs_person.chlimitl)'") - model_child.has_long_standing_illness = (frs_person.chealth1 == 1 ? 1 : 0) - model_child.how_long_adls_reduced = (frs_person.chlimitl < 0 ? -1 : frs_person.chlimitl) - model_child.adls_are_reduced = (frs_person.chcond < 0 ? -1 : frs_person.chcond) # missings to 'not at all' - model_child.over_20_k_saving = 0 - - model_child.registered_blind = (frs_person.spcreg1 == 1 ? 1 : 0) - model_child.registered_partially_sighted = (frs_person.spcreg2 == 1 ? 1 : 0) - model_child.registered_deaf = (frs_person.spcreg3 == 1 ? 1 : 0) - - model_child.disability_vision = (frs_person.cdisd01 == 1 ? 1 : 0) # cdisd kids .. - model_child.disability_hearing = (frs_person.cdisd02 == 1 ? 1 : 0) - model_child.disability_mobility = (frs_person.cdisd03 == 1 ? 1 : 0) - model_child.disability_dexterity = (frs_person.cdisd04 == 1 ? 1 : 0) - model_child.disability_learning = (frs_person.cdisd05 == 1 ? 1 : 0) - model_child.disability_memory = (frs_person.cdisd06 == 1 ? 1 : 0) - model_child.disability_mental_health = (frs_person.cdisd07 == 1 ? 1 : 0) - model_child.disability_stamina = (frs_person.cdisd08 == 1 ? 1 : 0) - model_child.disability_socially = (frs_person.cdisd09 == 1 ? 1 : 0) - # dindividual_savings_accountbility_other_difficulty = Vector{Union{Real,Missing}}(missing, n), - model_child.health_status = safe_assign(frs_person.heathch) - model_child.income_wages = safe_inc( 0.0, frs_person.chearns ) - model_child.income_other_investment_income = safe_inc( 0.0, frs_person.chsave ) - model_child.income_other_income = safe_inc( 0.0, frs_person.chrinc ) - model_child.income_free_school_meals = 0.0 - for t in [:fsbval,:fsfvval,:fsmlkval,:fsmval] - model_child.income_free_school_meals = safe_inc( model_child.income_free_school_meals, frs_person[t] ) - end - model_child.is_informal_carer = (frs_person.carefl == 1 ? 1 : 0) # also kid - process_relationships!( model_child, frs_person ) - # TODO education grants, all the other good child stuff EMA - - model_child.cost_of_childcare = 0.0 - model_child.hours_of_childcare = 0.0 - for c in 1:nchildcares - if c == 1 # type of care from 1st instance - model_child.childcare_type = - map_child_care( year, a_childcare[c, :chlook] ) - model_child.employer_provides_child_care = (a_childcare[c, :emplprov] == 2 ? - 1 : 0) - end - model_child.cost_of_childcare = safe_inc( - model_child.cost_of_childcare, - a_childcare[c, :chamt] - ) - model_child.hours_of_childcare = safe_inc( - model_child.hours_of_childcare, - a_childcare[c, :chhr] - ) - end # child care loop - model_child.onerand = mybigrandstr() - - # - # - # this is zero length - # a_oddjob = oddjob[((oddjob.sernum.==frs_person.sernum).&(oddjob.benunit.==frs_person.benunit).&(oddjob.person.==frs_person.person)), :] - # this isn't - a_benefits = benefits[((benefits.sernum.==frs_person.sernum).&(benefits.benunit.==frs_person.benunit).&(benefits.person.==frs_person.person)), :] - sb = size( a_benefits )[1] - println( "sb = $sb") - # @assert sb in [0,1] - process_benefits!( model_child, a_benefits ) - - end # if in HBAI - end # chno loop - return child_model[1:ccount,:] # send them all back ... -end - -function create_household( - year::Integer, - frs_household::DataFrame, - renter::DataFrame, - mortgage::DataFrame, - mortcont::DataFrame, - owner::DataFrame, - hbai_res::DataFrame )::DataFrame - - num_households = size(frs_household)[1] - hh_model = initialise_household(num_households) - hhno = 0 - # hbai_year = year - 1993 - - for hn in 1:num_households - if hn % 1000 == 0 - println("on year $year, hid $hn") - end - hh = frs_household[hn, :] - sernum = hh.sernum - if is_in_hbai( hbai_res, hh.sernum ) # only non-missing in HBAI - ad1_hbai = hbai_res[(hbai_res.sernum.==hh.sernum), :][1,:] - hhno += 1 - dd = split(hh.intdate, "/") - hh_model[hhno, :interview_year] = parse(Int64, dd[3]) - interview_month = parse(Int8, dd[1]) - hh_model[hhno, :interview_month] = interview_month - hh_model[hhno, :quarter] = div(interview_month - 1, 3) + 1 - - hh_model[hhno, :hid] = sernum - hh_model[hhno, :data_year] = year - hh_model[hhno, :tenure] = hh.tentyp2 > 0 ? hh.tentyp2 : -1 - hh_model[hhno, :dwelling] = hh.typeacc > 0 ? hh.typeacc : -1 - hh_model[hhno, :region] = hh.gvtregn > 0 ? hh.gvtregn : -1 - hh_model[hhno, :ct_band] = hh.ctband > 0 ? hh.ctband : -1 - hh_model[hhno, :weight] = hh.gross4 - # hh_model[hhno, :tenure] = hh.tentyp2 > 0 ? Tenure_Type(hh.tentyp2) : - # Missing_Tenure_Type - # hh_model[hhno, :region] = hh.gvtregn > 0 ? Standard_Region(hh.gvtregn) : - # Missing_Standard_Region - # hh_model[hhno, :ct_band] = hh.ctband > 0 ? CT_Band(hh.ctband) : Missing_CT_Band - # - # council_tax::Real - # FIXME this is rounded to £ - if hh_model[hhno, :region] == 299999999 # Scotland - hh_model[hhno, :water_and_sewerage] = safe_assign(ad1_hbai.cwathh) - elseif hh_model[hhno, :region] == 399999999 # Nireland - hh_model[hhno, :water_and_sewerage] = 0.0 # FIXME - else # - hh_model[hhno, :water_and_sewerage] = safe_assign(ad1_hbai.watsewhh) - end - # hh_model[hhno, :mortgage_payment] - hh_model[hhno, :mortgage_interest] = ad1_hbai.hbxmort - - # TODO - # years_outstanding_on_mortgage::Integer - # mortgage_outstanding::Real - # year_house_bought::Integer - # FIXME rounded to £1 - hh_model[hhno, :gross_rent] = max(0.0, hh.hhrent) # rentg Gross rent including Housing Benefit or rent Net amount of last rent payment - - rents = renter[(renter.sernum.==sernum), :] - nrents = size(rents)[1] - hh_model[hhno, :rent_includes_water_and_sewerage] = false - for r in 1:nrents - if (rents[r, :wsinc] in [1, 2, 3]) - hh_model[hhno, :rent_includes_water_and_sewerage] = true - end - end - ohc = 0.0 - ohc = safe_inc(ohc, hh.chrgamt1) - ohc = safe_inc(ohc, hh.chrgamt2) - ohc = safe_inc(ohc, hh.chrgamt3) - ohc = safe_inc(ohc, hh.chrgamt4) - ohc = safe_inc(ohc, hh.chrgamt5) - ohc = safe_inc(ohc, hh.chrgamt6) - ohc = safe_inc(ohc, hh.chrgamt7) - ohc = safe_inc(ohc, hh.chrgamt8) - ohc = safe_inc(ohc, hh.chrgamt9) - hh_model[hhno, :other_housing_charges] = ohc - hh_model[hhno, :bedrooms] = hh.bedroom6 - hh_model[hhno, :onerand] = mybigrandstr() - hh_model[hhno, :original_gross_income] = hh.hhinc - # TODO - # gross_housing_costs::Real - # total_wealth::Real - # house_value::Real - # people::People_Dict - end - end - hh_model[1:hhno, :] -end - -#= - -not used anymore - -const HBAIS = Dict( - - 2018 => "h1819.tab", - 2017 => "h1718.tab", - 2016 => "hbai1617_g4.tab", - 2015 => "hbai1516_g4.tab", - 2014 => "hbai1415_g4.tab", - 2013 => "hbai1314_g4.tab", - 2012 => "hbai1213_g4.tab", - 2011 => "hbai1112_g4.tab", - 2010 => "hbai1011_g4.tab", - 2009 => "hbai0910_g4.tab", - 2008 => "hbai0809_g4.tab", - 2007 => "hbai0708_g4.tab", - 2006 => "hbai0607_g4.tab", - 2005 => "hbai0506_g4.tab", - 2004 => "hbai0405_g4.tab", - 2003 => "hbai0304_g4.tab" -) -=# - -function loadfrs(which::AbstractString, year::Integer)::DataFrame - filename = "$(L_FRS_DIR)/$(year)/tab/$(which).tab" - loadtoframe(filename) -end - -function create_data(;start_year::Int, end_year::Int, hbai::String) - # - # every time I do this the HBAI has been re-arranged - # now, there's one file for the years 2015-2019; despite the name - # the variables we use aren't uprated. See: - # `5828_hbai_1920_harmonised_dataset_variables_guide.xlsx` - # in the doc bundle - # - hbai_res = loadtoframe("$(L_HBAI_DIR)/tab/$(hbai).tab") - # hbai_res = vcat( hbai_res, loadtoframe("$(HBAI_DIR)/tab/i1820e_1920prices.tab")) - - model_households = initialise_household(0) - model_people = initialise_person(0) - - for year in start_year:end_year - hbai_year = year - 1993 - print("on year $year ") - appendb = year > 2015 - y = year - 2000 - ystr = "$(y)$(y+1)" - # we only want this massive thing for a couple of - # benefit variables. - #frsx = "" - # if year < 2019 - frsx = loadfrs( "frs$ystr", year ) - # else - # frsx=CSV.File( "$(L_FRS_DIR)/$(year)/tab/frs1920.tab", missingstring=["", " "]) |> DataFrame - # - # end - accounts = loadfrs("accounts", year) - adult = loadfrs("adult", year) - assets = loadfrs("assets", year) - benefits = loadfrs("benefits", year) - benunit = loadfrs("benunit", year) - care = loadfrs("care", year) - child = loadfrs("child", year) - chldcare = loadfrs("chldcare", year) - endowmnt = loadfrs("endowmnt", year) - extchild = loadfrs("extchild", year) - govpay = loadfrs("govpay", year) - househol = loadfrs("househol", year) - job = loadfrs("job", year) - maint = loadfrs("maint", year) - mortcont = loadfrs("mortcont", year) - mortgage = loadfrs("mortgage", year) - oddjob = loadfrs("oddjob", year) - owner = loadfrs("owner", year) - penprov = loadfrs("penprov", year) - pension = loadfrs("pension", year) - rentcont = loadfrs("rentcont", year) - renter = loadfrs("renter", year) - # - # 2021 renames ... these are all the same variables - # - renameif!( adult, ["nssec20"=> "nssec", "soc2020"=>"soc2010"]) # 2021 change; this seems to be the same variable - - model_children_yr = create_children( - year, - child, - chldcare, - hbai_res[(hbai_res.year .== hbai_year),:], benefits ) - - append!(model_people, model_children_yr) - model_adults_yr = create_adults( - year, - adult, - accounts, - benunit, - extchild, - maint, - penprov, - # admin, - care, - mortcont, - pension, - govpay, - mortgage, - assets, - chldcare, - househol, - oddjob, - benefits, - endowmnt, - job, - hbai_res[(hbai_res.year .== hbai_year),:], - frsx ) - append!(model_people, model_adults_yr) - model_households_yr = create_household( - year, - househol, - renter, - mortgage, - mortcont, - owner, - hbai_res[(hbai_res.year .== hbai_year),:] ) - append!(model_households, model_households_yr) - println( "on year $year") - println( "hhlds") - # CSV.write("$(MODEL_DATA_DIR)/model_households-$(start_year)-$(end_year).tab", model_households_yr, delim = "\t", append=appendb) - # println( "adults") - # CSV.write("$(MODEL_DATA_DIR)/model_people.tab-$(start_year)-$(end_year)", model_adults_yr, delim = "\t", append=appendb) - # println( "children") - # CSV.write("$(MODEL_DATA_DIR)/model_people.tab-$(start_year)-$(end_year)", model_children_yr, delim = "\t", append=true) - end - CSV.write("$(MODEL_DATA_DIR)model_households-$(start_year)-$(end_year).tab", model_households, delim = "\t") - CSV.write("$(MODEL_DATA_DIR)model_people-$(start_year)-$(end_year).tab", model_people, delim = "\t") -end \ No newline at end of file diff --git a/oldsrc/Incomes.jl.using-constants b/oldsrc/Incomes.jl.using-constants deleted file mode 100644 index 2930c99c..00000000 --- a/oldsrc/Incomes.jl.using-constants +++ /dev/null @@ -1,989 +0,0 @@ -module Incomes -# -# This module implements a list of incomes, roughly based on FRS incomes, -# implemented with constants and -# fixed length arrays. It's used by the calculation and results modules for -# fast computation. In the household module, incomes are modelled as a Dictionary. -# [Results.jl] has a routine to map between the two. -# -# TODO Investigate Named Arrays -# - -using Base: String -using StaticArrays -using DataFrames -using ScottishTaxBenefitModel.RunSettings: Settings - - # declarations ---------------- - const WAGES = 1 - const SELF_EMPLOYMENT_INCOME = 2 - const ODD_JOBS = 3 - const PRIVATE_PENSIONS = 4 - const NATIONAL_SAVINGS = 5 - const BANK_INTEREST = 6 - const STOCKS_SHARES = 7 - const INDIVIDUAL_SAVINGS_ACCOUNT = 8 - const PROPERTY = 9 - const ROYALTIES = 10 - const BONDS_AND_GILTS = 11 - const OTHER_INVESTMENT_INCOME = 12 - const OTHER_INCOME = 13 - const ALIMONY_AND_CHILD_SUPPORT_RECEIVED = 14 - const PRIVATE_SICKNESS_SCHEME_BENEFITS = 15 - const ACCIDENT_INSURANCE_SCHEME_BENEFITS = 16 - const HOSPITAL_SAVINGS_SCHEME_BENEFITS = 17 - const UNEMPLOYMENT_OR_REDUNDANCY_INSURANCE = 18 - const PERMANENT_HEALTH_INSURANCE = 19 - const ANY_OTHER_SICKNESS_INSURANCE = 20 - const CRITICAL_ILLNESS_COVER = 21 - const TRADE_UNION_SICK_OR_STRIKE_PAY = 22 - const SPARE_INC_1 = 23 - const SPARE_INC_2 = 24 - const SPARE_INC_3 = 25 - const SPARE_INC_4 = 26 - const SPARE_INC_5 = 27 - - - const HEALTH_INSURANCE = 28 - const ALIMONY_AND_CHILD_SUPPORT_PAID = 29 - const TRADE_UNIONS_ETC = 30 - const FRIENDLY_SOCIETIES = 31 - const WORK_EXPENSES = 32 - const AVCS = 33 - const OTHER_DEDUCTIONS = 34 - const LOAN_REPAYMENTS = 35 - const PENSION_CONTRIBUTIONS_EMPLOYEE = 36 - const PENSION_CONTRIBUTIONS_EMPLOYER = 37 - const SPARE_DEDUCT_1 = 38 - const SPARE_DEDUCT_2 = 39 - const SPARE_DEDUCT_3 = 40 - const SPARE_DEDUCT_4 = 41 - const SPARE_DEDUCT_5 = 42 - - const INCOME_TAX = 43 - const NATIONAL_INSURANCE = 44 - const LOCAL_TAXES = 45 - const SOCIAL_FUND_LOAN_REPAYMENT = 46 - const STUDENT_LOAN_REPAYMENTS = 47 - const CARE_INSURANCE = 48 - const SPARE_TAX_1 = 49 - const SPARE_TAX_2 = 50 - const SPARE_TAX_3 = 51 - const SPARE_TAX_4 = 52 - const SPARE_TAX_5 = 53 - - const CHILD_BENEFIT = 54 - const STATE_PENSION = 55 - const BEREAVEMENT_ALLOWANCE = 56 - const ARMED_FORCES_COMPENSATION_SCHEME = 57 - const WAR_WIDOWS_PENSION = 58 - const SEVERE_DISABILITY_ALLOWANCE = 59 - const ATTENDANCE_ALLOWANCE = 60 - const CARERS_ALLOWANCE = 61 - const INDUSTRIAL_INJURY_BENEFIT = 62 - const INCAPACITY_BENEFIT = 63 - const PERSONAL_INDEPENDENCE_PAYMENT_DAILY_LIVING = 64 - const PERSONAL_INDEPENDENCE_PAYMENT_MOBILITY = 65 - const DLA_SELF_CARE = 66 - const DLA_MOBILITY = 67 - const EDUCATION_ALLOWANCES = 68 - const FOSTER_CARE_PAYMENTS = 69 - const MATERNITY_ALLOWANCE = 70 - const MATERNITY_GRANT = 71 - const FUNERAL_GRANT = 72 - const ANY_OTHER_NI_OR_STATE_BENEFIT = 73 - const FRIENDLY_SOCIETY_BENEFITS = 74 - const GOVERNMENT_TRAINING_ALLOWANCES = 75 - const CONTRIB_JOBSEEKERS_ALLOWANCE = 76 - const GUARDIANS_ALLOWANCE = 77 - const WIDOWS_PAYMENT = 78 - const WINTER_FUEL_PAYMENTS = 79 - # legacy mt benefits - const WORKING_TAX_CREDIT = 80 - const CHILD_TAX_CREDIT = 81 - const NON_CONTRIB_EMPLOYMENT_AND_SUPPORT_ALLOWANCE = 82 - const INCOME_SUPPORT = 83 - const PENSION_CREDIT = 84 - const SAVINGS_CREDIT = 85 - const NON_CONTRIB_JOBSEEKERS_ALLOWANCE = 86 # FIXME JOBSEEKERS->JOB_SEEKERS everwhere - const HOUSING_BENEFIT = 87 - - const FREE_SCHOOL_MEALS = 88 - const UNIVERSAL_CREDIT = 89 - const OTHER_BENEFITS = 90 - const STUDENT_GRANTS = 91 - const STUDENT_LOANS = 92 - const COUNCIL_TAX_BENEFIT = 93 - const CONTRIB_EMPLOYMENT_AND_SUPPORT_ALLOWANCE = 94 - const SCOTTISH_CARERS_SUPPLEMENT = 95 - const SPARE_BEN_1 = 96 - const SPARE_BEN_2 = 97 - const SPARE_BEN_3 = 98 - const SPARE_BEN_4 = 99 - const SPARE_BEN_5 = 100 - const SPARE_BEN_6 = 101 - const SPARE_BEN_7 = 102 - const SPARE_BEN_8 = 103 - const SPARE_BEN_9 = 104 - - const NON_CALCULATED = WAGES:SPARE_INC_5 - const BENEFITS = CHILD_BENEFIT:SPARE_BEN_9 - const LEGACY_MTBS = WORKING_TAX_CREDIT:HOUSING_BENEFIT - - const MEANS_TESTED_BENS = WORKING_TAX_CREDIT:UNIVERSAL_CREDIT - const NON_MEANS_TESTED_BENS = setdiff( BENEFITS, MEANS_TESTED_BENS ) - const INCOME_TAXES = INCOME_TAX:NATIONAL_INSURANCE - - const CALCULATED = INCOME_TAX:SCOTTISH_CARERS_SUPPLEMENT - const SICKNESS_ILLNESS = SEVERE_DISABILITY_ALLOWANCE:DLA_MOBILITY - const DEDUCTIONS = HEALTH_INSURANCE:SPARE_DEDUCT_5 - const INC_ARRAY_SIZE = SPARE_BEN_9 - const PASSED_THROUGH_BENEFITS = [ - ARMED_FORCES_COMPENSATION_SCHEME, - WAR_WIDOWS_PENSION, - SEVERE_DISABILITY_ALLOWANCE, - INDUSTRIAL_INJURY_BENEFIT, - INCAPACITY_BENEFIT, - EDUCATION_ALLOWANCES, - FOSTER_CARE_PAYMENTS, - MATERNITY_GRANT, - FUNERAL_GRANT, - ANY_OTHER_NI_OR_STATE_BENEFIT, - GUARDIANS_ALLOWANCE, - FREE_SCHOOL_MEALS, - OTHER_BENEFITS, - STUDENT_GRANTS, - STUDENT_LOANS ] - const ALL_INCOMES = union( NON_CALCULATED, BENEFITS ) - - const DIRECT_TAXES_AND_DEDUCTIONS = union(INCOME_TAXES,DEDUCTIONS) - - # exports ---------------- - export ALL_INCOMES - export NET_INCOME - export DIRECT_TAXES_AND_DEDUCTIONS - export DIRECT_TAXES - export NON_MEANS_TESTED_BENS - export WAGES - export SELF_EMPLOYMENT_INCOME - export ODD_JOBS - export PRIVATE_PENSIONS - export NATIONAL_SAVINGS - export BANK_INTEREST - export STOCKS_SHARES - export INDIVIDUAL_SAVINGS_ACCOUNT - export PROPERTY - export ROYALTIES - export BONDS_AND_GILTS - export OTHER_INVESTMENT_INCOME - export OTHER_INCOME - export ALIMONY_AND_CHILD_SUPPORT_RECEIVED - export PRIVATE_SICKNESS_SCHEME_BENEFITS - export ACCIDENT_INSURANCE_SCHEME_BENEFITS - export HOSPITAL_SAVINGS_SCHEME_BENEFITS - export UNEMPLOYMENT_OR_REDUNDANCY_INSURANCE - export PERMANENT_HEALTH_INSURANCE - export ANY_OTHER_SICKNESS_INSURANCE - export CRITICAL_ILLNESS_COVER - export TRADE_UNION_SICK_OR_STRIKE_PAY - export SPARE_INC_1 - export SPARE_INC_2 - export SPARE_INC_3 - export SPARE_INC_4 - export SPARE_INC_5 - export HEALTH_INSURANCE - export ALIMONY_AND_CHILD_SUPPORT_PAID - export TRADE_UNIONS_ETC - export FRIENDLY_SOCIETIES - export WORK_EXPENSES - export AVCS - export OTHER_DEDUCTIONS - export LOAN_REPAYMENTS - export PENSION_CONTRIBUTIONS_EMPLOYEE - export PENSION_CONTRIBUTIONS_EMPLOYER - export SPARE_DEDUCT_1 - export SPARE_DEDUCT_2 - export SPARE_DEDUCT_3 - export SPARE_DEDUCT_4 - export SPARE_DEDUCT_5 - # taxation - export INCOME_TAX - export NATIONAL_INSURANCE - export LOCAL_TAXES - export SOCIAL_FUND_LOAN_REPAYMENT - export STUDENT_LOAN_REPAYMENTS - export CARE_INSURANCE - export SPARE_TAX_1 - export SPARE_TAX_2 - export SPARE_TAX_3 - export SPARE_TAX_4 - export SPARE_TAX_5 - # benefits - export CHILD_BENEFIT - export STATE_PENSION - export BEREAVEMENT_ALLOWANCE - export ARMED_FORCES_COMPENSATION_SCHEME - export WAR_WIDOWS_PENSION - export SEVERE_DISABILITY_ALLOWANCE # obselete not modelled - export ATTENDANCE_ALLOWANCE - export CARERS_ALLOWANCE - export INDUSTRIAL_INJURY_BENEFIT - export INCAPACITY_BENEFIT # obselete not modelled - export PERSONAL_INDEPENDENCE_PAYMENT_DAILY_LIVING - export PERSONAL_INDEPENDENCE_PAYMENT_MOBILITY - export DLA_SELF_CARE - export DLA_MOBILITY - export EDUCATION_ALLOWANCES - export FOSTER_CARE_PAYMENTS - export MATERNITY_ALLOWANCE - export MATERNITY_GRANT - export FUNERAL_GRANT - export ANY_OTHER_NI_OR_STATE_BENEFIT - export FRIENDLY_SOCIETY_BENEFITS - export GOVERNMENT_TRAINING_ALLOWANCES - export CONTRIB_JOBSEEKERS_ALLOWANCE - export GUARDIANS_ALLOWANCE - export WIDOWS_PAYMENT - export WINTER_FUEL_PAYMENTS - export WORKING_TAX_CREDIT - export CHILD_TAX_CREDIT - export CONTRIB_EMPLOYMENT_AND_SUPPORT_ALLOWANCE - export NON_CONTRIB_EMPLOYMENT_AND_SUPPORT_ALLOWANCE - export INCOME_SUPPORT - export PENSION_CREDIT - export SAVINGS_CREDIT - export NON_CONTRIB_JOBSEEKERS_ALLOWANCE - export HOUSING_BENEFIT - export FREE_SCHOOL_MEALS - export UNIVERSAL_CREDIT - export OTHER_BENEFITS - export STUDENT_GRANTS - export STUDENT_LOANS - export COUNCIL_TAX_BENEFIT - export SPARE_BEN_1 # scottish carers supplement - export SPARE_BEN_2 - export SPARE_BEN_3 - export SPARE_BEN_4 - export SPARE_BEN_5 - export SPARE_BEN_6 - export SPARE_BEN_7 - export SPARE_BEN_8 - export SPARE_BEN_9 - export SCOTTISH_CARERS_SUPPLEMENT - - export GROSS_INCOME - export BENEFITS - export LEGACY_MTBS - export MEANS_TESTED_BENS - export INCOME_TAXES - export CALCULATED - export DEDUCTIONS - export SICKNESS_ILLNESS - export INC_ARRAY_SIZE - export ALL_INCOMES - export PASSED_THROUGH_BENEFITS - - export iname - export make_static_incs - export make_mutable_incs - export make_a - export IncludedItems - export isum - export any_positive - - const ISet = Set{Int} - const ZSet = Set{Int}() - - struct IncludedItems - included :: AbstractArray{<:Integer} - deducted :: Union{Nothing,AbstractArray{<:Integer}} - end - - function isum( a :: AbstractArray{T}, - included; - deducted = nothing ) :: T where T - s = zero(T) - for i in included - s += a[i] - end - if ! isnothing(deducted) - for i in deducted - s -= a[i] - end - end - return s - end - - function isum( a :: AbstractArray, - which :: IncludedItems ) - return isum( a, which.included, deducted=which.deducted) - end - - function any_positive( a :: AbstractArray, - which ) :: Bool - for i in which - if a[i] > 0 - return true - end - end - return false - end - - function make_static_incs( - T :: Type; - ones = ZSet, - minusones = ZSet - ) :: SVector # {INC_ARRAY_SIZE,T} where T - v = zeros(T, INC_ARRAY_SIZE) - for i in ones - v[i] = one(T) - end - for i in minusones - v[i] = -one(T) - end - return SVector{INC_ARRAY_SIZE,T}(v) - end - - function make_mutable_incs( - T :: Type; - ones = ZSet, - minusones = ZSet ) :: MVector # {INC_ARRAY_SIZE,T} where T - v = zeros(T, INC_ARRAY_SIZE) - for i in ones - v[i] = one(T) - end - for i in minusones - v[i] = -one(T) - end - return MVector{INC_ARRAY_SIZE,T}(v) - end - - function make_a( T :: Type ) :: SizedVector - return SizedVector{INC_ARRAY_SIZE,T}(zeros(T, INC_ARRAY_SIZE)) - end - - function iname(i::Integer)::String - @assert i in 1:INC_ARRAY_SIZE - if i == WAGES - return "Wages" - elseif i == SELF_EMPLOYMENT_INCOME - return "Self Employment Income" - elseif i == ODD_JOBS - return "Odd Jobs" - elseif i == PRIVATE_PENSIONS - return "Private Pensions" - elseif i == NATIONAL_SAVINGS - return "National Savings" - elseif i == BANK_INTEREST - return "Bank Interest" - elseif i == STOCKS_SHARES - return "Stocks Shares" - elseif i == INDIVIDUAL_SAVINGS_ACCOUNT - return "Individual Savings Account" - elseif i == PROPERTY - return "Property" - elseif i == ROYALTIES - return "Royalties" - elseif i == BONDS_AND_GILTS - return "Bonds And Gilts" - elseif i == OTHER_INVESTMENT_INCOME - return "Other Investment Income" - elseif i == OTHER_INCOME - return "Other Income" - elseif i == ALIMONY_AND_CHILD_SUPPORT_RECEIVED - return "Alimony And Child Support Received" - elseif i == PRIVATE_SICKNESS_SCHEME_BENEFITS - return "Private Sickness Scheme Benefits" - elseif i == ACCIDENT_INSURANCE_SCHEME_BENEFITS - return "Accident Insurance Scheme Benefits" - elseif i == HOSPITAL_SAVINGS_SCHEME_BENEFITS - return "Hospital Savings Scheme Benefits" - elseif i == UNEMPLOYMENT_OR_REDUNDANCY_INSURANCE - return "Unemployment Or Redundancy Insurance" - elseif i == PERMANENT_HEALTH_INSURANCE - return "Permanent Health Insurance" - elseif i == ANY_OTHER_SICKNESS_INSURANCE - return "Any Other Sickness Insurance" - elseif i == CRITICAL_ILLNESS_COVER - return "Critical Illness Cover" - elseif i == TRADE_UNION_SICK_OR_STRIKE_PAY - return "Trade Union Sick Or Strike Pay" - elseif i == SPARE_INC_1 - return "Spare Inc 1" - elseif i == SPARE_INC_2 - return "Spare Inc 2" - elseif i == SPARE_INC_3 - return "Spare Inc 3" - elseif i == SPARE_INC_4 - return "Spare Inc 4" - elseif i == SPARE_INC_5 - return "Spare Inc 5" - elseif i == HEALTH_INSURANCE - return "Health Insurance" - elseif i == ALIMONY_AND_CHILD_SUPPORT_PAID - return "Alimony And Child Support Paid" - elseif i == TRADE_UNIONS_ETC - return "Trade Unions Etc" - elseif i == FRIENDLY_SOCIETIES - return "Friendly Societies" - elseif i == WORK_EXPENSES - return "Work Expenses" - elseif i == AVCS - return "Avcs" - elseif i == OTHER_DEDUCTIONS - return "Other Deductions" - elseif i == LOAN_REPAYMENTS - return "Loan Repayments" - elseif i == PENSION_CONTRIBUTIONS_EMPLOYEE - return "Pension Contributions Employee" - elseif i == PENSION_CONTRIBUTIONS_EMPLOYER - return "Pension Contributions Employer" - elseif i == SPARE_DEDUCT_1 - return "Spare Deduct 1" - elseif i == SPARE_DEDUCT_2 - return "Spare Deduct 2" - elseif i == SPARE_DEDUCT_3 - return "Spare Deduct 3" - elseif i == SPARE_DEDUCT_4 - return "Spare Deduct 4" - elseif i == SPARE_DEDUCT_5 - return "Spare Deduct 5" - elseif i == INCOME_TAX - return "Income Tax" - elseif i == NATIONAL_INSURANCE - return "National Insurance" - elseif i == LOCAL_TAXES - return "Local Taxes" - elseif i == SOCIAL_FUND_LOAN_REPAYMENT - return "Social Fund Loan Repayment" - elseif i == STUDENT_LOAN_REPAYMENTS - return "Student Loan Repayments" - elseif i == CARE_INSURANCE - return "Care Insurance" - elseif i == SPARE_TAX_1 - return "Spare Tax 1" - elseif i == SPARE_TAX_2 - return "Spare Tax 2" - elseif i == SPARE_TAX_3 - return "Spare Tax 3" - elseif i == SPARE_TAX_4 - return "Spare Tax 4" - elseif i == SPARE_TAX_5 - return "Spare Tax 5" - elseif i == CHILD_BENEFIT - return "Child Benefit" - elseif i == STATE_PENSION - return "State Pension" - elseif i == BEREAVEMENT_ALLOWANCE - return "Bereavement Allowance" - elseif i == ARMED_FORCES_COMPENSATION_SCHEME - return "Armed Forces Compensation Scheme" - elseif i == WAR_WIDOWS_PENSION - return "War Widows Pension" - elseif i == SEVERE_DISABILITY_ALLOWANCE - return "Severe Disability Allowance" - elseif i == ATTENDANCE_ALLOWANCE - return "Attendence Allowance" - elseif i == CARERS_ALLOWANCE - return "Carers Allowance" - elseif i == INDUSTRIAL_INJURY_BENEFIT - return "Industrial Injury Benefit" - elseif i == INCAPACITY_BENEFIT - return "Incapacity Benefit" - elseif i == PERSONAL_INDEPENDENCE_PAYMENT_DAILY_LIVING - return "Personal Independence Payment Daily Living" - elseif i == PERSONAL_INDEPENDENCE_PAYMENT_MOBILITY - return "Personal Independence Payment Mobility" - elseif i == DLA_SELF_CARE - return "Dla Self Care" - elseif i == DLA_MOBILITY - return "Dla Mobility" - elseif i == EDUCATION_ALLOWANCES - return "Education Allowances" - elseif i == FOSTER_CARE_PAYMENTS - return "Foster Care Payments" - elseif i == MATERNITY_ALLOWANCE - return "Maternity Allowance" - elseif i == MATERNITY_GRANT - return "Maternity Grant" - elseif i == FUNERAL_GRANT - return "Funeral Grant" - elseif i == ANY_OTHER_NI_OR_STATE_BENEFIT - return "Any Other Ni Or State Benefit" - elseif i == FRIENDLY_SOCIETY_BENEFITS - return "Friendly Society Benefits" - elseif i == GOVERNMENT_TRAINING_ALLOWANCES - return "Government Training Allowances" - elseif i == CONTRIB_JOBSEEKERS_ALLOWANCE - return "Contrib Jobseekers Allowance" - elseif i == GUARDIANS_ALLOWANCE - return "Guardians Allowance" - elseif i == WIDOWS_PAYMENT - return "Widows Payment" - elseif i == WINTER_FUEL_PAYMENTS - return "Winter Fuel Payments" - elseif i == WORKING_TAX_CREDIT - return "Working Tax Credit" - elseif i == CHILD_TAX_CREDIT - return "Child Tax Credit" - elseif i == CONTRIB_EMPLOYMENT_AND_SUPPORT_ALLOWANCE - return "Contributory Employment And Support Allowance" - elseif i == NON_CONTRIB_EMPLOYMENT_AND_SUPPORT_ALLOWANCE - return "Income-Related Employment And Support Allowance" - elseif i == INCOME_SUPPORT - return "Income Support" - elseif i == PENSION_CREDIT - return "Pension Credit" - elseif i == SAVINGS_CREDIT - return "Savings Credit" - elseif i == NON_CONTRIB_JOBSEEKERS_ALLOWANCE - return "Non Contrib Jobseekers Allowance" - elseif i == HOUSING_BENEFIT - return "Housing Benefit" - elseif i == UNIVERSAL_CREDIT - return "Universal Credit" - elseif i == OTHER_BENEFITS - return "Other Benefits" - elseif i == STUDENT_GRANTS - return "Student Grants" - elseif i == STUDENT_LOANS - return "Student Loans" - elseif i == FREE_SCHOOL_MEALS - return "Free School Meals" - elseif i == COUNCIL_TAX_BENEFIT - return "Council Tax Rebate" - elseif i == SPARE_BEN_1 - return "Spare Ben 1" - elseif i == SPARE_BEN_2 - return "Spare Ben 2" - elseif i == SPARE_BEN_3 - return "Spare Ben 3" - elseif i == SPARE_BEN_4 - return "Spare Ben 4" - elseif i == SPARE_BEN_5 - return "Spare Ben 5" - elseif i == SPARE_BEN_6 - return "Spare Ben 6" - elseif i == SPARE_BEN_7 - return "Spare Ben 7" - elseif i == SPARE_BEN_8 - return "Spare Ben 8" - elseif i == SPARE_BEN_9 - return "Spare Ben 9" - elseif i == SCOTTISH_CARERS_SUPPLEMENT - return "Spare Ben 10" - end - @assert false "$i not mapped in iname" - end # iname - - # FIXME what if we din't export these ones, deleted '_INCOME' and instead forced Incomes.SAVINGS ... - export LEGACY_MT_INCOME - export GROSS_INCOME - export LEGACY_HB_INCOME - export LEGACY_PC_INCOME - export LEGACY_SAVINGS_CREDIT_INCOME - export DIVIDEND_INCOME - export EXEMPT_INCOME - export ALL_TAXABLE_INCOME - export NON_SAVINGS_INCOME - export SAVINGS_INCOME - export DEFAULT_PASSPORTED_BENS - export UC_OTHER_INCOME - export UC_EARNED_INCOME - - const SAVINGS_INCOME = [BANK_INTEREST, BONDS_AND_GILTS,OTHER_INVESTMENT_INCOME] - - """ - TODO check this carefully against WTC,PC and IS chapters - note this doesn't include wages and TaxBenefitSystem - which are handled in the `calc_incomes` function. - poss. have 2nd complete version for WTC/CTC - """ - const UC_EARNED_INCOME = IncludedItems( - [ - WAGES - # se treated seperately - ], - [ - INCOME_TAX, - NATIONAL_INSURANCE, - PENSION_CONTRIBUTIONS_EMPLOYEE - ## ? student loan repayments? - ] - ) - - export - LEGACY_CAP_BENEFITS, - UC_CAP_BENEFITS - - const COMMON_CAP_BENEFITS = [ - CONTRIB_EMPLOYMENT_AND_SUPPORT_ALLOWANCE, - CONTRIB_JOBSEEKERS_ALLOWANCE, - WIDOWS_PAYMENT, - MATERNITY_ALLOWANCE, - SEVERE_DISABILITY_ALLOWANCE, - INCAPACITY_BENEFIT, - CHILD_BENEFIT, - SEVERE_DISABILITY_ALLOWANCE - ] - - const UC_CAP_BENEFITS = union([UNIVERSAL_CREDIT], COMMON_CAP_BENEFITS ) - - const LEGACY_CAP_BENEFITS = union( - [HOUSING_BENEFIT, - CHILD_TAX_CREDIT, - NON_CONTRIB_EMPLOYMENT_AND_SUPPORT_ALLOWANCE, - NON_CONTRIB_JOBSEEKERS_ALLOWANCE, - INCOME_SUPPORT], - COMMON_CAP_BENEFITS - ) - - # FIXME check these lists again very carefully indeed. - # FIXME the CPAG guide describes a very convoluted way - # of calculating, which requires 2 lists of earned and unearned income - # I think that's important just for - const UC_OTHER_INCOME = [ - # WAGES, - # SELF_EMPLOYMENT_INCOME, # counts, but see minimum level of earnings regs 19/20 p119 - OTHER_INCOME, - CARERS_ALLOWANCE, - ALIMONY_AND_CHILD_SUPPORT_RECEIVED, # FIXME THERE IS A 15 DISREGARD SEE PP 438 - EDUCATION_ALLOWANCES, - FOSTER_CARE_PAYMENTS, - STATE_PENSION, - PRIVATE_PENSIONS, - BEREAVEMENT_ALLOWANCE, - WAR_WIDOWS_PENSION, - CONTRIB_JOBSEEKERS_ALLOWANCE, ## CONTRIBUTION BASED Only - CONTRIB_EMPLOYMENT_AND_SUPPORT_ALLOWANCE, - INDUSTRIAL_INJURY_BENEFIT, - INCAPACITY_BENEFIT, - MATERNITY_ALLOWANCE, - MATERNITY_GRANT, - FUNERAL_GRANT, - ANY_OTHER_NI_OR_STATE_BENEFIT, - TRADE_UNION_SICK_OR_STRIKE_PAY, - FRIENDLY_SOCIETY_BENEFITS, - WORKING_TAX_CREDIT , - PRIVATE_SICKNESS_SCHEME_BENEFITS, - ACCIDENT_INSURANCE_SCHEME_BENEFITS, - HOSPITAL_SAVINGS_SCHEME_BENEFITS, - GOVERNMENT_TRAINING_ALLOWANCES, - GUARDIANS_ALLOWANCE, - WIDOWS_PAYMENT, - UNEMPLOYMENT_OR_REDUNDANCY_INSURANCE ] - ## FIXME CHECK THIS list - ## NOTE wages, se are treated seperately - const LEGACY_MT_INCOME = IncludedItems( - [ - OTHER_INCOME, - CARERS_ALLOWANCE, - ALIMONY_AND_CHILD_SUPPORT_RECEIVED, # FIXME THERE IS A 15 DISREGARD SEE PP 438 - EDUCATION_ALLOWANCES, - FOSTER_CARE_PAYMENTS, - STATE_PENSION, - PRIVATE_PENSIONS, - BEREAVEMENT_ALLOWANCE, - WAR_WIDOWS_PENSION, - CONTRIB_EMPLOYMENT_AND_SUPPORT_ALLOWANCE, - CONTRIB_JOBSEEKERS_ALLOWANCE, ## CONTRIBUTION BASED Only - INDUSTRIAL_INJURY_BENEFIT, - INCAPACITY_BENEFIT, - MATERNITY_ALLOWANCE, - MATERNITY_GRANT, - FUNERAL_GRANT, - ANY_OTHER_NI_OR_STATE_BENEFIT, - TRADE_UNION_SICK_OR_STRIKE_PAY, - FRIENDLY_SOCIETY_BENEFITS, - WORKING_TAX_CREDIT , - PRIVATE_SICKNESS_SCHEME_BENEFITS, - ACCIDENT_INSURANCE_SCHEME_BENEFITS, - HOSPITAL_SAVINGS_SCHEME_BENEFITS, - GOVERNMENT_TRAINING_ALLOWANCES, - GUARDIANS_ALLOWANCE, - WIDOWS_PAYMENT, - UNEMPLOYMENT_OR_REDUNDANCY_INSURANCE - ], - [ - STUDENT_LOAN_REPAYMENTS, - ALIMONY_AND_CHILD_SUPPORT_PAID - ] - ) - - const GROSS_INCOME = IncludedItems( - [ - WAGES, - SELF_EMPLOYMENT_INCOME, - ODD_JOBS, - PRIVATE_PENSIONS, - NATIONAL_SAVINGS, - BANK_INTEREST, - STOCKS_SHARES, - INDIVIDUAL_SAVINGS_ACCOUNT, - PROPERTY, - ROYALTIES, - BONDS_AND_GILTS, - OTHER_INVESTMENT_INCOME, - OTHER_INCOME, - ALIMONY_AND_CHILD_SUPPORT_RECEIVED, - PRIVATE_SICKNESS_SCHEME_BENEFITS, - ACCIDENT_INSURANCE_SCHEME_BENEFITS, - HOSPITAL_SAVINGS_SCHEME_BENEFITS, - UNEMPLOYMENT_OR_REDUNDANCY_INSURANCE, - PERMANENT_HEALTH_INSURANCE, - ANY_OTHER_SICKNESS_INSURANCE, - CRITICAL_ILLNESS_COVER, - TRADE_UNION_SICK_OR_STRIKE_PAY - ], - [ - HEALTH_INSURANCE, - ALIMONY_AND_CHILD_SUPPORT_PAID, - TRADE_UNIONS_ETC, - FRIENDLY_SOCIETIES, - WORK_EXPENSES, - AVCS, - OTHER_DEDUCTIONS, - LOAN_REPAYMENTS, - PENSION_CONTRIBUTIONS_EMPLOYEE, - PENSION_CONTRIBUTIONS_EMPLOYER, - SPARE_DEDUCT_1, - SPARE_DEDUCT_2, - SPARE_DEDUCT_3, - SPARE_DEDUCT_4, - SPARE_DEDUCT_5, - ] - ) - - const LEGACY_HB_INCOME = union( LEGACY_MT_INCOME.included, - # since these are passported this should only - # ever matter if we have a 'passporting' switch - # and it's turned off, but anyway .... - [ - INCOME_SUPPORT, - CONTRIB_JOBSEEKERS_ALLOWANCE, - NON_CONTRIB_JOBSEEKERS_ALLOWANCE, - NON_CONTRIB_EMPLOYMENT_AND_SUPPORT_ALLOWANCE, - CONTRIB_EMPLOYMENT_AND_SUPPORT_ALLOWANCE, - CHILD_TAX_CREDIT ] ) - - const LEGACY_PC_INCOME = setdiff(LEGACY_MT_INCOME.included, [WORKING_TAX_CREDIT] ) - - const LEGACY_SAVINGS_CREDIT_INCOME = setdiff( LEGACY_MT_INCOME.included, - [ WORKING_TAX_CREDIT, - INCAPACITY_BENEFIT, - CONTRIB_EMPLOYMENT_AND_SUPPORT_ALLOWANCE, - # TODO CHECK BOTH? - CONTRIB_JOBSEEKERS_ALLOWANCE, - MATERNITY_ALLOWANCE, - ALIMONY_AND_CHILD_SUPPORT_RECEIVED ]) - - const DIVIDEND_INCOME = [ STOCKS_SHARES ] - - const EXEMPT_INCOME = [ - CARERS_ALLOWANCE, - FREE_SCHOOL_MEALS, - DLA_SELF_CARE, - DLA_MOBILITY, - CHILD_BENEFIT, - PENSION_CREDIT, - BEREAVEMENT_ALLOWANCE, - ARMED_FORCES_COMPENSATION_SCHEME, - WAR_WIDOWS_PENSION, - SEVERE_DISABILITY_ALLOWANCE, - ATTENDANCE_ALLOWANCE, - INDUSTRIAL_INJURY_BENEFIT, - CONTRIB_EMPLOYMENT_AND_SUPPORT_ALLOWANCE, - INCAPACITY_BENEFIT, - INCOME_SUPPORT, - MATERNITY_ALLOWANCE, - MATERNITY_GRANT, - FUNERAL_GRANT, - GUARDIANS_ALLOWANCE, - WINTER_FUEL_PAYMENTS, - WORKING_TAX_CREDIT, - CHILD_TAX_CREDIT, - HOUSING_BENEFIT, - PERSONAL_INDEPENDENCE_PAYMENT_DAILY_LIVING, - PERSONAL_INDEPENDENCE_PAYMENT_MOBILITY - ] - - const ALL_TAXABLE_INCOME = setdiff( ALL_INCOMES, EXEMPT_INCOME ) - const NON_SAVINGS_INCOME = setdiff( ALL_TAXABLE_INCOME, DIVIDEND_INCOME, SAVINGS_INCOME ) - - const DEFAULT_PASSPORTED_BENS = [ - INCOME_SUPPORT, - NON_CONTRIB_EMPLOYMENT_AND_SUPPORT_ALLOWANCE, - NON_CONTRIB_JOBSEEKERS_ALLOWANCE, - PENSION_CREDIT ] - - export inctostr, isettostr - - function inctostr( incs :: AbstractVector ) :: String - n = size( incs )[1] - @assert n == INC_ARRAY_SIZE - s = - """ - - - | | | - |:-----------|-------------:| - """ - - for i in 1:n - if incs[i] != 0 - s *= "|**$(iname(i))**|$(incs[i])|\n" - end - end - s *= "\n\n" - return s - end - - function isettostr( iset ) - s = "" - for i in iset - s *= "$(iname(i))\n" - end - return s - end - - # FIXME move all exports close to the module top - export - create_incomes_dataframe, - fill_inc_frame_row! - - - function create_incomes_dataframe( RT :: DataType, n :: Int ) :: DataFrame - return DataFrame( - pid = zeros( BigInt, n ), - hid = zeros( BigInt, n ), - weight = zeros( RT, n ), - # buno = zeros( Int, n ), - - wages = zeros( RT, n ), - self_employment_income = zeros( RT, n ), - odd_jobs = zeros( RT, n ), - private_pensions = zeros( RT, n ), - national_savings = zeros( RT, n ), - bank_interest = zeros( RT, n ), - stocks_shares = zeros( RT, n ), - individual_savings_account = zeros( RT, n ), - property = zeros( RT, n ), - royalties = zeros( RT, n ), - bonds_and_gilts = zeros( RT, n ), - other_investment_income = zeros( RT, n ), - other_income = zeros( RT, n ), - alimony_and_child_support_received = zeros( RT, n ), - private_sickness_scheme_benefits = zeros( RT, n ), - accident_insurance_scheme_benefits = zeros( RT, n ), - hospital_savings_scheme_benefits = zeros( RT, n ), - unemployment_or_redundancy_insurance = zeros( RT, n ), - permanent_health_insurance = zeros( RT, n ), - any_other_sickness_insurance = zeros( RT, n ), - critical_illness_cover = zeros( RT, n ), - trade_union_sick_or_strike_pay = zeros( RT, n ), - spare_inc_1 = zeros( RT, n ), - spare_inc_2 = zeros( RT, n ), - spare_inc_3 = zeros( RT, n ), - spare_inc_4 = zeros( RT, n ), - spare_inc_5 = zeros( RT, n ), - - - health_insurance = zeros( RT, n ), - alimony_and_child_support_paid = zeros( RT, n ), - trade_unions_etc = zeros( RT, n ), - friendly_societies = zeros( RT, n ), - work_expenses = zeros( RT, n ), - avcs = zeros( RT, n ), - other_deductions = zeros( RT, n ), - loan_repayments = zeros( RT, n ), - pension_contributions_employee = zeros( RT, n ), - pension_contributions_employer = zeros( RT, n ), - spare_deduct_1 = zeros( RT, n ), - spare_deduct_2 = zeros( RT, n ), - spare_deduct_3 = zeros( RT, n ), - spare_deduct_4 = zeros( RT, n ), - spare_deduct_5 = zeros( RT, n ), - - income_tax = zeros( RT, n ), - national_insurance = zeros( RT, n ), - local_taxes = zeros( RT, n ), - social_fund_loan_repayment = zeros( RT, n ), - student_loan_repayments = zeros( RT, n ), - care_insurance = zeros( RT, n ), - spare_tax_1 = zeros( RT, n ), - spare_tax_2 = zeros( RT, n ), - spare_tax_3 = zeros( RT, n ), - spare_tax_4 = zeros( RT, n ), - spare_tax_5 = zeros( RT, n ), - - child_benefit = zeros( RT, n ), - state_pension = zeros( RT, n ), - bereavement_allowance = zeros( RT, n ), - armed_forces_compensation_scheme = zeros( RT, n ), - war_widows_pension = zeros( RT, n ), - severe_disability_allowance = zeros( RT, n ), - attendance_allowance = zeros( RT, n ), - carers_allowance = zeros( RT, n ), - industrial_injury_benefit = zeros( RT, n ), - incapacity_benefit = zeros( RT, n ), - personal_independence_payment_daily_living = zeros( RT, n ), - personal_independence_payment_mobility = zeros( RT, n ), - dla_self_care = zeros( RT, n ), - dla_mobility = zeros( RT, n ), - education_allowances = zeros( RT, n ), - foster_care_payments = zeros( RT, n ), - maternity_allowance = zeros( RT, n ), - maternity_grant = zeros( RT, n ), - funeral_grant = zeros( RT, n ), - any_other_ni_or_state_benefit = zeros( RT, n ), - friendly_society_benefits = zeros( RT, n ), - government_training_allowances = zeros( RT, n ), - contrib_jobseekers_allowance = zeros( RT, n ), - guardians_allowance = zeros( RT, n ), - widows_payment = zeros( RT, n ), - winter_fuel_payments = zeros( RT, n ), - working_tax_credit = zeros( RT, n ), - child_tax_credit = zeros( RT, n ), - non_contrib_employment_and_support_allowance = zeros( RT, n ), - income_support = zeros( RT, n ), - pension_credit = zeros( RT, n ), - savings_credit = zeros( RT, n ), - non_contrib_jobseekers_allowance = zeros( RT, n ), - housing_benefit = zeros( RT, n ), - free_school_meals = zeros( RT, n ), - universal_credit = zeros( RT, n ), - other_benefits = zeros( RT, n ), - student_grants = zeros( RT, n ), - student_loans = zeros( RT, n ), - council_tax_benefit = zeros( RT, n ), - contrib_employment_and_support_allowance = zeros( RT, n ), - scottish_carers_supplement = zeros( RT, n ), - spare_ben_1 = zeros( RT, n ), - spare_ben_2 = zeros( RT, n ), - spare_ben_3 = zeros( RT, n ), - spare_ben_4 = zeros( RT, n ), - spare_ben_5 = zeros( RT, n ), - spare_ben_6 = zeros( RT, n ), - spare_ben_7 = zeros( RT, n ), - spare_ben_8 = zeros( RT, n ), - spare_ben_9 = zeros( RT, n )) - end - - function fill_inc_frame_row!( - incd :: DataFrameRow, - pid :: BigInt, - hid :: BigInt, - weight :: Real, - income :: AbstractArray ) - @assert size( income )[1] == INC_ARRAY_SIZE - incd.pid = pid - incd.hid = hid - incd.weight = weight - col = 3 # FIXME make sure this is incremented if anything added above - for i in 1:INC_ARRAY_SIZE - col += 1 - incd[col] = income[i] - end - end # add_to_inc_frame - -end # module \ No newline at end of file diff --git a/scripts/create-scottish-la-weights.jl b/scripts/create-scottish-la-weights.jl index 876c48d3..b2121a3a 100644 --- a/scripts/create-scottish-la-weights.jl +++ b/scripts/create-scottish-la-weights.jl @@ -263,8 +263,8 @@ function initialise_target_dataframe_scotland_la( n :: Integer ) :: DataFrame # 25 if INCLUDE_EMPLOYMENT # d.working = zeros(n) - # d.economically_active_employee = zeros(n) - # d.economically_active_self_employed = zeros(n) + d.economically_active_employee = zeros(n) + d.economically_active_self_employed = zeros(n) d.economically_active_unemployed = zeros(n) end # 29 @@ -405,13 +405,13 @@ function make_target_row_scotland_la!( if INCLUDE_EMPLOYMENT # drop inactive if pers.employment_status in [Full_time_Employee, Part_time_Employee] - # row.economically_active_employee += 1 + row.economically_active_employee += 1 # row.working += 1 elseif pers.employment_status in [ Full_time_Self_Employed, Part_time_Self_Employed ] # row.working += 1 - # row.economically_active_self_employed += 1 + row.economically_active_self_employed += 1 elseif pers.employment_status in [Unemployed] row.economically_active_unemployed += 1 end @@ -567,8 +567,8 @@ function make_target_list( alldata::DataFrame, council::Symbol )::Vector v.m_65plus = data.m_65plus if INCLUDE_EMPLOYMENT # v.working = data.working - # v.economically_active_employee = data.economically_active_employee - # v.economically_active_self_employed = data.economically_active_self_employed + v.economically_active_employee = data.economically_active_employee + v.economically_active_self_employed = data.economically_active_self_employed v.economically_active_unemployed = data.economically_active_unemployed end if INCLUDE_OCCUP