update FPs, benchmarking tests and add law school causal dataset

src/Fairness.jl

@@ -68,7 +68,7 @@ export @load_toydata, @load_toyfairtensor
 export @load_compas, @load_adult, @load_german,
        @load_bank_marketing, @load_communities_crime, @load_student_performance
 
-export genZafarData, genSubgroupData, genZafarData2, genBiasedSampleData
+export genZafarData, genSubgroupData, genZafarData2, genBiasedSampleData, law_school
 # -------------------------------------------------------------------
 # re-export From CategoricalArrays
 export categorical, levels, levels!

src/benchmark/evaluate.jl

@@ -28,7 +28,7 @@ function get_df(fp::FairnessProblem, models_fn)
 	for i in 1:runs
 		cv = StratifiedCV(nfolds=nfolds, shuffle=true, rng=seeds[i])
 		tr_tt_pairs = MLJBase.train_test_pairs(cv, 1:length(y),
-			categorical(X[!, protected_attr] .* "-" .* string(y)))
+			categorical(string.(X[!, protected_attr]) .* "-" .* string(y)))
 		for i_model in 1:length(models)
 			model = models[i_model]
 			for j in 1:nfolds
@@ -37,8 +37,8 @@ function get_df(fp::FairnessProblem, models_fn)
 				@suppress fit!(mach, rows=tr_tt_pairs[j][1], verbosity=0)
 
 				# Add out-sample performance measures
-				ŷ = predict(mach, rows=tr_tt_pairs[j][2])
-				if typeof(ŷ[1])<:MLJ.UnivariateFinite ŷ = mode.(ŷ) end
+				ŷ = MMI.predict(mach, rows=tr_tt_pairs[j][2])
+				if typeof(ŷ[1])<:MLJBase.UnivariateFinite ŷ = StatsBase.mode.(ŷ) end
 				ft = fair_tensor(ŷ, y[tr_tt_pairs[j][2]],
 										X[tr_tt_pairs[j][2], protected_attr])
 				accVal = accuracy(ft)
@@ -51,8 +51,8 @@ function get_df(fp::FairnessProblem, models_fn)
 
 
 				# Add in-sample performance measures
-				ŷ = predict(mach, rows=tr_tt_pairs[j][1])
-				if typeof(ŷ[1])<:MLJ.UnivariateFinite ŷ = mode.(ŷ) end
+				ŷ = MMI.predict(mach, rows=tr_tt_pairs[j][1])
+				if typeof(ŷ[1])<:MLJBase.UnivariateFinite ŷ = StatsBase.mode.(ŷ) end
 				ft = fair_tensor(ŷ, y[tr_tt_pairs[j][1]],
 										X[tr_tt_pairs[j][1], protected_attr])
 				accVal = accuracy(ft)
@@ -66,7 +66,7 @@ function get_df(fp::FairnessProblem, models_fn)
 			end
 		end
 	end
-	CSV.write(joinpath(dirname(@__FILE__), "results", fp.name*".csv"), df)
+	CSV.write(joinpath(pwd(), fp.name*"-results.csv"), df)
 	return df
 end
 
@@ -103,16 +103,16 @@ function get_pareto_df(fp::FairnessProblem, models_fn, alphas=0:0.1:1)
 		for i in 1:runs
 			cv = StratifiedCV(nfolds=nfolds, shuffle=true, rng=seeds[i])
 			tr_tt_pairs = MLJBase.train_test_pairs(cv, 1:length(y),
-				categorical(X[!, protected_attr] .* "-" .* string(y)))
+				categorical(string.(X[!, protected_attr]) .* "-" .* string(y)))
 			for i_model in 1:length(models)
 				model = models[i_model]
 				for j in 1:nfolds
 					Random.seed!(seeds[i])
 					mach = machine(model, X, y)
 					@suppress fit!(mach, rows=tr_tt_pairs[j][1])
 					# Add out-sample performance measures
-					ŷ = predict(mach, rows=tr_tt_pairs[j][2])
-					if typeof(ŷ[1])<:MLJ.UnivariateFinite ŷ = mode.(ŷ) end
+					ŷ = MMI.predict(mach, rows=tr_tt_pairs[j][2])
+					if typeof(ŷ[1])<:MLJBase.UnivariateFinite ŷ = StatsBase.mode.(ŷ) end
 					ft = fair_tensor(ŷ, y[tr_tt_pairs[j][2]],
 											X[tr_tt_pairs[j][2], protected_attr])
 					accVal = accuracy(ft)
@@ -125,8 +125,8 @@ function get_pareto_df(fp::FairnessProblem, models_fn, alphas=0:0.1:1)
 
 
 					# Add in-sample performance measures
-					ŷ = predict(mach, rows=tr_tt_pairs[j][1])
-					if typeof(ŷ[1])<:MLJ.UnivariateFinite ŷ = mode.(ŷ) end
+					ŷ = MMI.predict(mach, rows=tr_tt_pairs[j][1])
+					if typeof(ŷ[1])<:MLJBase.UnivariateFinite ŷ = StatsBase.mode.(ŷ) end
 					ft = fair_tensor(ŷ, y[tr_tt_pairs[j][1]],
 											X[tr_tt_pairs[j][1], protected_attr])
 					accVal = accuracy(ft)
@@ -141,7 +141,7 @@ function get_pareto_df(fp::FairnessProblem, models_fn, alphas=0:0.1:1)
 			end
 		end
 	end
-	CSV.write(joinpath(dirname(@__FILE__), "results", fp.name*"-pareto.csv"), df)
+	CSV.write(joinpath(pwd(), fp.name*"-pareto-results.csv"), df)
 	return df
 end
 

src/benchmark/fairness_problems.jl

@@ -1,4 +1,4 @@
-function communities_crime()
+function communities_crime(repls=1, nfolds=6)
 	communities_data = @load_communities_crime
 	communities_data[1].Column8 = convert(Array{Float64},
 										communities_data[1].Column8) .<= 0.18
@@ -10,19 +10,19 @@ function communities_crime()
 	FairnessProblem(
 		communities_crime_task,
 		measures=[fnr, fpr],
-		repls=10, nfolds=10, name="communities_crime")
+		repls=repls, nfolds=nfolds, name="communities_crime")
 end
 
-function student()
+function student(repls=1, nfolds=6)
    student_task = Task((@load_student_performance)...,
 					   grp=:sex, debiasmeasures=[fnr])
    FairnessProblem(
 	   student_task,
 	   measures=[fnr, fpr],
-	   repls=10, nfolds=10, name="StudentPerformance")
+	   repls=repls, nfolds=nfolds, name="StudentPerformance")
 end
 
-function hmda()
+function hmda(repls=1, nfolds=6)
 	hmda_df = CSV.read(joinpath(@__DIR__, "..", "data", "HMDA_2018_California.csv"))
 	filter!(:derived_sex => x -> x != "Sex Not Available", hmda_df)
 	categorical!(hmda_df, [:conforming_loan_limit, :derived_loan_product_type ,
@@ -37,71 +37,71 @@ function hmda()
 
 	FairnessProblem(hmda_task,
 		measures=[false_omission_rate, fpr],
-		repls=10, nfolds=10, name="HMDA_Mortgage")
+		repls=repls, nfolds=nfolds, name="HMDA_Mortgage")
 end
 
-function zafar()
+function zafar(repls=1, nfolds=6)
    zafar_task = Task((genZafarData())...,
 					   grp=:z, debiasmeasures=[])
    FairnessProblem(
 	   zafar_task,
 	   measures=[],
-	   repls=10, nfolds=10, name="ZafarData")
+	   repls=repls, nfolds=nfolds, name="ZafarData")
 end
 
-function zafar2()
+function zafar2(repls=1, nfolds=6)
    zafar2_task = Task((genZafarData2())...,
 					   grp=:z, debiasmeasures=[])
    FairnessProblem(
 	   zafar2_task,
 	   measures=[],
-	   repls=10, nfolds=10, name="StudentPerformance")
+	   repls=repls, nfolds=nfolds, name="StudentPerformance")
 end
 
-function subgroup()
+function subgroup(repls=1, nfolds=6)
    subgroup_task = Task((genSubgroupData(setting="B01"))...,
 					   grp=:z, debiasmeasures=[])
    FairnessProblem(
 	   subgroup_task,
 	   measures=[],
-	   repls=10, nfolds=10, name="SubGroupData")
+	   repls=repls, nfolds=nfolds, name="SubGroupData")
 end
 
-function biased()
+function biased(repls=1, nfolds=6)
    biased_task = Task((genBiasedSampleData())...,
 					   grp=:z, debiasmeasures=[])
    biased = FairnessProblem(
 	   biased_task,
 	   measures=[],
-	   repls=10, nfolds=10, name="BiasedSampleData")
+	   repls=repls, nfolds=nfolds, name="BiasedSampleData")
 end
 
-function adult()
+function adult(repls=1, nfolds=6)
    adult_task = Task((@load_adult)..., grp=:sex, debiasmeasures=[ppr]) # could keep :sex also as protected attribute
    FairnessProblem(
 	   adult_task,
 	   refGrp="Male", measures=[ppr, fnr, fpr],
-	   repls=10, nfolds=10, name="Adult")
+	   repls=repls, nfolds=nfolds, name="Adult")
 end
 
-function german()
+function german(repls=1, nfolds=6)
    german_task = Task((@load_german)..., grp=:gender_status, debiasmeasures=[false_omission_rate])
    FairnessProblem(
 	   german_task,
 	   refGrp="male_single", measures=[foar, fnr, fpr],
-	   repls=10, nfolds=10, name="German")
+	   repls=repls, nfolds=nfolds, name="German")
 end
 
-function portuguese()
+function portuguese(repls=1, nfolds=6)
    portuguese_task = Task((@load_bank_marketing)...,
 					   grp=:marital, debiasmeasures=[false_omission_rate, ppr])
    FairnessProblem(
 	   portuguese_task,
 	   measures=[false_omission_rate, ppr, fpr],
-	   repls=10, nfolds=10, name="Portuguese")
+	   repls=repls, nfolds=nfolds, name="Portuguese")
 end
 
-function framingham()
+function framingham(repls=1, nfolds=6)
    framingham_df = CSV.read(joinpath(@__DIR__, "..", "data", "framingham.csv"))
    categorical!(framingham_df, [:male, :education, :currentSmoker, :BPMeds,
 			   :prevalentStroke, :prevalentHyp, :diabetes, :TenYearCHD])
@@ -112,21 +112,21 @@ function framingham()
 
    FairnessProblem(framingham_task,
 				   measures=[false_omission_rate],
-				   repls=10, nfolds=10, name="Framingham")
+				   repls=repls, nfolds=nfolds, name="Framingham")
 end
 
-function loan()
+function loan(repls=1, nfolds=6)
    loan_df = CSV.read(joinpath(@__DIR__, "..", "data", "loan_default.csv"))
    categorical!(loan_df, [:SEX, :EDUCATION, :MARRIAGE,
 		   Symbol("default payment next month")])
    loan_task = Task(deletecols(loan_df, Symbol("default payment next month")),
 			   loan_df[!, Symbol("default payment next month")],
 			   grp=:SEX, debiasmeasures=[false_omission_rate])
    FairnessProblem(loan_task, measures=[false_omission_rate, fpr],
-					   repls=10, nfolds=10, name="LoanDefault")
+					   repls=repls, nfolds=nfolds, name="LoanDefault")
 end
 
-function medical()
+function medical(repls=1, nfolds=6)
    medical_df = CSV.read(joinpath(@__DIR__, "..", "data", "meps.csv"))
 
    categorical!(medical_df, ["REGION","SEX","MARRY","FTSTU","ACTDTY","HONRDC",
@@ -140,7 +140,7 @@ function medical()
 			   medical_df[!, Symbol("UTILIZATION")],
 			   grp=:RACE, debiasmeasures=[false_omission_rate])
    FairnessProblem(medical_task, measures=[false_omission_rate, fnr],
-					   repls=10, nfolds=10, name="MedicalExpenditure")
+					   repls=repls, nfolds=nfolds, name="MedicalExpenditure")
 end
 
 fairness_problems() = [compas(), adult(), german(), portuguese(),

src/benchmark/task.jl

@@ -38,14 +38,14 @@ function FairnessProblem(task::Task; refGrp=nothing,
 								   seed=5, name=nothing)
    name = name==nothing ? randstring(['A':'Z'; '0':'9'], 12) : name
    measures = measures==nothing ? debiasmeasures : measures
-   refGrp = refGrp==nothing ? mode(task.X[!, task.grp]) : refGrp
+   refGrp = refGrp==nothing ? StatsBase.mode(task.X[!, task.grp]) : refGrp
    task.X[!, task.grp] = convert(Array{Any}, task.X[!, task.grp])
    task.X[!, task.grp] = string.(task.X[!, task.grp])
    task.X[task.X[!, task.grp].!=refGrp, task.grp] .= "0"
    task.X[task.X[!, task.grp].==refGrp, task.grp] .= "1"
    task.X[!, task.grp] = categorical(convert(Array{String},
 												task.X[!, task.grp]))
-   categorical!(task.X, task.grp)
+   transform!(task.X, task.grp => categorical, renamecols=false)
    return FairnessProblem(task, refGrp, measures, repls, nfolds, seed, name)
 end
 

src/datasets/datasets.jl

@@ -173,7 +173,7 @@ macro load_bank_marketing()
             download(url, "tempdataset.zip")
             zarchive = ZipFile.Reader("tempdataset.zip")
             zipfile = filter(x->x.name=="bank-additional/bank-additional-full.csv", zarchive.files)[1]
-            df = DataFrame!(CSV.File(read(zipfile)))
+            df = DataFrame(CSV.File(read(zipfile)); copycols = false)
             CSV.write(fpath, df)
             close(zarchive)
             Base.Filesystem.rm("tempdataset.zip", recursive=true)
@@ -223,7 +223,7 @@ macro load_student_performance()
             download(url, "tempdataset.zip")
             zarchive = ZipFile.Reader("tempdataset.zip")
             zipfile = filter(x->x.name=="student-mat.csv", zarchive.files)[1]
-            df = DataFrame!(CSV.File(read(zipfile)))
+            df = DataFrame(CSV.File(read(zipfile)); copycols = false)
             CSV.write(fpath, df)
             close(zarchive)
             Base.Filesystem.rm("tempdataset.zip", recursive=true)

src/datasets/synthetic.jl

@@ -237,3 +237,26 @@ function genBiasedSampleData(n = 10000; sampling_bias = 0.8)
   y = categorical(ifelse.(yprob .> u, 1, 0))
   return X, y
 end
+
+"""
+returns R, S, A, G, L, F
+see https://github.com/apedawi-cs/Causal-inference-discussion/blob/master/law_school.ipynb
+"""
+function law_school(nb_obs, R_pct=0.75, S_pct=0.6)
+  function simulate_exogenous_vars(nb_obs, R_pct=0.5, S_pct=0.5)
+    R = rand(Distributions.Uniform(0, 1), nb_obs, 1) .< R_pct
+    S = rand(Distributions.Uniform(0, 1), nb_obs, 1) .< S_pct
+    A = randn(nb_obs, 1)
+    return R, S, A
+  end
+  function simulate_endogenous_vars(A, R, S)
+    nb_obs = length(A)
+    G = A + 2.1 * R + 3.3 * S + 0.5 * randn(nb_obs, 1)
+    L = A + 5.8 * R + 0.7 * S + 0.1 * randn(nb_obs, 1)
+    F = A + 2.3 * R + 1.0 * S + 0.3 * randn(nb_obs, 1)
+    return G, L, F
+  end
+  R, S, A = simulate_exogenous_vars(nb_obs, R_pct, S_pct)
+  G, L, F = simulate_endogenous_vars(A, R, S)
+  R, S, A, G, L, F
+end

test/benchmark/benchmark.jl

@@ -9,3 +9,37 @@
 	@test size(results["results"])==(1, 2, 6)
 	@test size(results["tstats"])==(1, 2, 2)
 end
+
+function models_fn(fp::FairnessProblem, alpha=1.0)
+	########## You can change code segment below to add more models
+
+	rf = @pipeline ContinuousEncoder ConstantClassifier
+
+	models = MLJBase.Model[rf]
+	#  Just simply add the base classifiers above and to this list
+
+	model_names = ["RF"] # Add names of models. Preferably keep names short
+
+	################################################################
+	model_count = length(models)
+	protected_attr = fp.task.grp
+	debiasmeasures = fp.task.debiasmeasures
+	for i in 1:model_count
+    rw = ReweighingSamplingWrapper(classifier=models[i], grp=protected_attr, alpha=alpha)
+		lp = LinProgWrapper(classifier=models[i], grp=protected_attr,
+								measures=debiasmeasures, alpha=alpha)
+		eo = EqOddsWrapper(classifier=models[i], grp=protected_attr, alpha=alpha)
+		append!(models, [rw, lp, eo])
+		append!(model_names, model_names[i]*"-".*["Reweighing",
+		"LinProg-".*join(debiasmeasures, "-"), "Equalized Odds"])
+	end
+	return models, model_names
+end
+
+@testset "Get df" begin
+	@test size(get_df(student(), models_fn))==(48, 7)
+end
+
+@testset "Get Pareto df" begin
+	@test size(get_pareto_df(student(), models_fn, 0:1:0.5))==(48, 8)
+end

test/datasets/synthetic.jl

@@ -21,3 +21,8 @@ end
   @test scitype(y) == AbstractArray{Multiclass{2}, 1}
   @test string.(names(X)) == ["x1", "x2", "x3", "x4", "z"]
 end
+
+@testset "Law School" begin
+  A = law_school(10)
+  @test length(A)==6
+end