artifacts for FRS

Artifacts.toml

@@ -1,9 +1,18 @@
 [augdata]
-git-tree-sha1 = "288471b266e4ec933d1c6190c8dd7fc3e82d3f4c"
+git-tree-sha1 = "b1ad31b8fcd1165a57e02d7de274b7140254b480"
+lazy = true
 
     [[augdata.download]]
-    sha256 = "774759a0726fe9d904016569c643650ad1cb0bcf650784c52f60bdcb35a0f2db"
-    url = "https://virtual-worlds.biz/artifacts//aux-data.tar.gz"
+    sha256 = "ff09ec25734af6aecd831a4fb89ba3c402945812f5b34a29986aed8252dadf02"
+    url = "https://virtual-worlds.biz/artifacts//augdata.tar.gz"
+
+[disability]
+git-tree-sha1 = "288ef84eac1aad34af423852e64001e75be3b822"
+lazy = true
+
+    [[disability.download]]
+    sha256 = "7ed1e796fc2e09a4c48f772ea3cd06b7c6aab721c183f6905b2fa83890d0184b"
+    url = "https://virtual-worlds.biz/artifacts//disability.tar.gz"
 
 [exampledata]
 git-tree-sha1 = "7853136d3530ea531e4f45a29f8fcef81d08fc1c"

Project.toml

@@ -6,6 +6,7 @@ version = "0.1.2"
 [deps]
 ArgCheck = "dce04be8-c92d-5529-be00-80e4d2c0e197"
 ArtifactUtils = "8b73e784-e7d8-4ea5-973d-377fed4e3bce"
+Artifacts = "56f22d72-fd6d-98f1-02f0-08ddc0907c33"
 BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
 BudgetConstraints = "51aacf4b-43e8-45f9-b960-ce5126c2a956"
 CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b"
@@ -63,6 +64,7 @@ UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
 [compat]
 ArgCheck = "2.3.0, 4"
 ArtifactUtils = "0.2.4"
+Artifacts = "1.11.0"
 BenchmarkTools = "1.3.2, 4"
 BudgetConstraints = "1.0.1, 3"
 CSV = "0.10.10, 1"
@@ -115,22 +117,19 @@ artifact_server_upload = "[email protected]:public_html/artifacts/"
 artifact_server_url = "https://virtual-worlds.biz/artifacts/"
 auto_weight = true
 default_run_name = "default_run"
-use_weighting = true
-target_nation = "N_Scotland"
+do_marginal_rates = false
+do_replacement_rates = false
 dump_frames = false
-to_y = 2024
-to_q = 3
+dump_frames2 = true
+ineq_income_measure = "eq_bhc_net_income"
 means_tested_routing = "modelled_phase_in"
 poverty_line_source = "pl_first_sys"
-ineq_income_measure = "eq_bhc_net_income"
-do_marginal_rates = false
-do_replacement_rates = false
+prem2 = "enhanced_disability_child"
 replacement_rate_hours = 30
+target_nation = "N_Scotland"
+to_q = 3
+to_y = 2024
+use_weighting = true
 
-prem2 = "enhanced_disability_child"
-dump_frames2 = true
-
 [targets]
 test = ["Test"]
-
-

docs/ima-pluto.jl

@@ -57,7 +57,6 @@ begin
 	const DEFAULT_NUM_TYPE = Float64
 	settings = RunSettings.Settings()
 	settings.requested_threads = 4
-	data_dir( settings ) = "/home/graham_s/julia/vw/ScottishTaxBenefitModel/data"
 
 	function init_data(; reset :: Bool = false )
 	   nhh = FRSHouseholdGetter.get_num_households()
@@ -71,14 +70,6 @@ begin
 
 	function load_system()::TaxBenefitSystem
 		sys = get_default_system_for_cal_year( 2021 )
-		# load_file( joinpath( Definitions.MODEL_PARAMS_DIR, "sys_2021_22.jl" ))
-		#
-		# Note that as of Budget21 removing these doesn't actually happen till May 2022.
-		#
-		# load_file!( sys, joinpath( Definitions.MODEL_PARAMS_DIR, "sys_2021-uplift-removed.jl"))
-		# uc taper to 55
-		# load_file!( sys, joinpath( Definitions.MODEL_PARAMS_DIR, "budget_2021_uc_changes.jl"))
-		# weeklyise!( sys )
 		return sys
 	end
 

params/sys_2023_24_ruk.jl

@@ -359,7 +359,7 @@ function load_sys_2023_24_ruk!( sys :: TaxBenefitSystem{T} ) where T
       :NIRELAND => -99999.99
     ] )
 
-  brmapath = joinpath(MODEL_DATA_DIR, "local", "brma-2023-2024-country-averages.csv")
+  brmapath = joinpath( artifact"augdata", "brma-2023-2024-country-averages.csv" )
 
   sys.hr.brmas = loadBRMAs( 4, Float64, brmapath )
 

params/sys_2023_24_scotland.jl

@@ -10,7 +10,7 @@ function load_sys_2023_24_scotland!( sys :: TaxBenefitSystem )
     sys.it.non_savings_thresholds = [2_162, 13_118, 31_092, 125_120.0]
     sys.it.non_savings_basic_rate = 2 # above this counts as higher rate rate FIXME 3???
     sys.nmt_bens.carers.scottish_supplement = 0.0 # FROM APRIL 2021
-    brmapath = joinpath(MODEL_DATA_DIR, "local", "lha_rates_scotland_2023_24.csv")
+    brmapath = joinpath( artifact"augdata", "lha_rates_scotland_2023_24.csv")
     sys.hr.brmas = loadBRMAs( 4, Float64, brmapath )
     sys.loctax.ct.band_d = Dict(
       [

regressions/disability_regressions.jl

@@ -10,22 +10,11 @@ using CSV,DataFrames,GLM,RegressionTables
 
 
 const settings = Settings()
+# settings.data_source = SyntheticSource # !!! changeme!!! 
+daf = get_data_artifact( settings )
 
-const LOCAL_DATA_DIR = data_dir( settings )
-#
-# !!! For Actual Dataset, UK wide with Scottish dummy
-# For synth data, Scotland only for now!!! 
-#
-settings.data_source = SyntheticSource # !!! changeme!!! 
-if settings.data_source == FRS
-  settings.household_name = "model_households-2015-2021.tab"
-  settings.people_name = "model_people-2015-2021.tab"
-end
-
-const DATASETS = main_datasets( settings )
-
-frshh = CSV.File( DATASETS.hhlds ) |> DataFrame
-frspeople = CSV.File( DATASETS.people ) |> DataFrame
+frshh = CSV.File( joinpath( daf, "households.tab"  )) |> DataFrame
+frspeople = CSV.File( joinpath( daf, "households.tab" )) |> DataFrame
 
 fm = innerjoin( frshh, frspeople, on=[:data_year, :hid ], makeunique=true )
 

src/ExampleHouseholdGetter.jl

@@ -8,6 +8,7 @@ using DataFrames
 using CSV
 using ArgCheck
 using Pkg.Artifacts
+using LazyArtifacts
 
 using ScottishTaxBenefitModel
 using .Definitions
@@ -56,12 +57,10 @@ function initialise(
 
     global KEYMAP 
     global EXAMPLE_HOUSEHOLDS
-    # tmpsource = settings.data_source # hack to work round datasource being wired in to settings
-    # settings.data_source = ExampleSource
-    # println( "DEF_MODEL_DATA_DIR=|$(Definitions.DEF_MODEL_DATA_DIR)| MODEL_DATA_DIR=|$MODEL_DATA_DIR|")
     # lazy load cons data if needs be
     tmp_data_source = settings.data_source 
     settings.data_source = ExampleSource
+    # tmpsource = settings.data_source # hack to work round datasource being wired in to settings
     if settings.indirect_method == matching
         ConsumptionData.init( settings ) 
     end

src/HistoricBenefits.jl

@@ -9,7 +9,7 @@ module HistoricBenefits
 # with a series of complete parameter files, once we have 
 # everything defined fully.
 # 
-using CSV, DataFrames, Dates
+using CSV, DataFrames, Dates, Artifacts, LazyArtifacts
 using ScottishTaxBenefitModel
 using .Definitions 
 using .ModelHousehold: Person
@@ -41,20 +41,23 @@ function load_historic( file ) :: Dict
 end
 
 function load_pip()
-    pip=CSV.File( "$(MODEL_DATA_DIR)/receipts/pip_2002-2023_from_stat_explore.csv",
+    pip=CSV.File( joinpath(artifact"augdata", "pip_2002-2023_from_stat_explore.csv"),
         missingstring="..",
         types=Dict([:Date=>String]))|>DataFrame
     pip.Date = Date.( pip.Date, dateformat"yyyymm" )
     return pip
 end
 
 function load_dla()
-    dla=CSV.File( "$(MODEL_DATA_DIR)/receipts/dla_2002-2023_from_stat_explore.csv" )|> DataFrame
+    dla=CSV.File( joinpath(artifact"augdata","dla_2002-2023_from_stat_explore.csv" ))|> DataFrame
     dla.Date = Date.( dla.Date, dateformat"u-yy" ) .+Year(2000)
     return dla
 end
 
-const HISTORIC_BENEFITS = load_historic( "$(MODEL_PARAMS_DIR)/historic_benefits.csv" ) 
+const HISTORIC_BENEFITS = load_historic( 
+    joinpath( 
+        artifact"augdata", "historic_benefits.csv" ))
+
 const DLA_RECEIPTS = load_dla()
 const PIP_RECEIPTS =  load_pip()
 

src/LocalLevelCalculations.jl

@@ -9,6 +9,8 @@ module LocalLevelCalculations
 
 using StaticArrays
 using CSV,DataFrames
+using Pkg, Pkg.Artifacts
+using LazyArtifacts
 
 using ScottishTaxBenefitModel
 using .Definitions
@@ -118,7 +120,7 @@ export
 
     # FIXME hard code this in
     function make_la_to_brma_map()
-        lacsv = CSV.File( "$(MODEL_DATA_DIR)/local/la_to_brma_approx_mappings.csv" ) |> DataFrame
+        lacsv = CSV.File( joinpath( artifact"augdata", "la_to_brma_approx_mappings.csv" )) |> DataFrame
         out = Dict{Symbol,Symbol}()
         for r in eachrow( lacsv )
             out[Symbol(r.ccode)] = Symbol(r.bcode)

src/RunSettings.jl

@@ -4,6 +4,7 @@ module RunSettings
     #
     using Pkg
     using Pkg.Artifacts
+    using LazyArtifacts
     using Parameters
     using Preferences 
     using UUIDs
@@ -45,7 +46,6 @@ module RunSettings
         # DatasetType,
         # actual_data,
         # synthetic_data,
-        data_dir,
         get_skiplist,
 
         get_all_uk_settings_2023,
@@ -90,7 +90,6 @@ module RunSettings
         scotland_full :: Bool = true
         weighted = @load_preference( "use_weighting", true )
         auto_weight = @load_preference( "auto_weight", true )
-        data_dir :: String = MODEL_DATA_DIR # DELETE
         household_name = "model_households_scotland-2015-2021-w-enums-2"
         people_name  = "model_people_scotland-2015-2021-w-enums-2"
         target_nation :: Nation = eval(Symbol(@load_preference("target_nation", "N_Scotland"))) #  N_Scotland
@@ -178,6 +177,7 @@ module RunSettings
         settings.replacement_rate_hours = @load_preference( "replacement_rate_hours" )
     end
     =#
+
 
     function get_data_artifact( settings::Settings )::AbstractString
         return if settings.data_source == FRSSource
@@ -197,6 +197,16 @@ module RunSettings
         end
     end
 
+    function main_datasets( settings :: Settings ) :: NamedTuple
+        artd = get_data_artifact( settings )
+        return ( 
+            hhlds = joinpath( artd, "households.tab" ),
+            people = joinpath( artd, "people.tab" )
+        )
+    end
+
+    #=
+
     function data_dir( settings :: Settings ) :: String
         ds = if settings.data_source == FRSSource
             "actual_data"
@@ -243,6 +253,8 @@ module RunSettings
         )
     end
 
+    =#
+
     """
     Hacky prebuilt settings for the Northumbria model.
     """

src/Runner.jl

@@ -6,6 +6,8 @@ module Runner
     #
     using Base.Threads
 
+    using Pkg, Pkg.Artifacts
+    using LazyArtifacts
     using Parameters: @with_kw
     using DataFrames: DataFrame, DataFrameRow, Not, select!
     using CSV
@@ -75,7 +77,7 @@ module Runner
             @time settings.num_households, settings.num_people, nhh2 = 
                 FRSHouseholdGetter.initialise( settings )
             if settings.benefit_generosity_estimates_available
-                BenefitGenerosity.initialise( MODEL_DATA_DIR*"/disability/" )  
+                BenefitGenerosity.initialise( artifact"disability" )  
             end     
         end
         full_results = Array{HouseholdResult}(undef,0,0)

src/STBParameters.jl

@@ -13,6 +13,8 @@ using Dates: Date, now, TimeType, Year
 using TimeSeries
 using StaticArrays
 using Parameters
+using Pkg, Pkg.Artifacts
+using LazyArtifacts
 using DataFrames,CSV
 
 using ScottishTaxBenefitModel
@@ -822,7 +824,7 @@ end
 
     function weeklyise!( hb :: HousingBenefits; wpm=WEEKS_PER_MONTH, wpy=WEEKS_PER_YEARs )
     hb.taper /= 100.0
-    end
+end
 
 @with_kw mutable struct LegacyMeansTestedBenefitSystem{RT<:Real}
     # CPAG 2019/bur.pers[pid].20 p335
@@ -863,7 +865,7 @@ function loadBRMAs( N :: Int, T :: Type, file :: String  ) :: Dict{Symbol,BRMA{N
     dict
 end
 
-const DEFAULT_BRMA_2021 = joinpath(MODEL_DATA_DIR,"local", "lha_rates_scotland_2020_21.csv")
+const DEFAULT_BRMA_2021 = joinpath( artifact"augdata", "lha_rates_scotland_2020_21.csv")
 
 @with_kw mutable struct HousingRestrictions{RT<:Real}
     abolished :: Bool = false

src/Utils.jl

@@ -94,15 +94,18 @@ file should contain: `people.tab` `households.tab` `README.md`, all top-level
 """
 function make_artifact(;
    artifact_name :: AbstractString,
-   gzip_file_name:: AbstractString, 
    toml_file = "Artifacts.toml" )::Int 
-   art_server_upload = @load_preference( "artifact_server_upload" )
-   art_server_url = @load_preference( "artifact_server_url" )
-   dest = "$(art_server_upload)/$(gzip_file_name)"
-   println( "copying |tmp/$gzip_file_name| to |$dest| ")
-   upload = `scp tmp/$(gzip_file_name) $(dest)`
+   gzip_file_name = "$(artifact_name).tar.gz"
+   dir = "/mnt/data/ScotBen/artifacts/"
+   artifact_server_upload = @load_preference( "artifact_server_upload" )
+   artifact_server_url = @load_preference( "artifact_server_url" )
+   tarcmd = `tar zcvf $(dir)/tmp/$(gzip_file_name) -C $(dir)/$(artifact_name)/ .`
+   run( tarcmd )
+   dest = "$(artifact_server_upload)/$(gzip_file_name)"
+   println( "copying |$(dir)/tmp/$gzip_file_name| to |$dest| ")
+   upload = `scp $(dir)/tmp/$(gzip_file_name) $(dest)`
    println( "upload cmd |$upload|")
-   url = "$(art_server_url)/$gzip_file_name"
+   url = "$(artifact_server_url)/$gzip_file_name"
    try
       run( upload )
       add_artifact!( toml_file, artifact_name, url; force=true, lazy=true )