diff --git a/Project.toml b/Project.toml index 230c05a9..4dc9b7f8 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "MLJBase" uuid = "a7f614a8-145f-11e9-1d2a-a57a1082229d" authors = ["Anthony D. Blaom "] -version = "0.20.7" +version = "0.20.8" [deps] CategoricalArrays = "324d7699-5711-5eae-9e2f-1d82baa6b597" diff --git a/src/MLJBase.jl b/src/MLJBase.jl index efc65dec..2499eb5b 100644 --- a/src/MLJBase.jl +++ b/src/MLJBase.jl @@ -53,7 +53,7 @@ import MLJModelInterface: fit, update, update_data, transform, predict_mean, predict_median, predict_joint, evaluate, clean!, is_same_except, save, restore, is_same_except, istransparent, - params, training_losses + params, training_losses, feature_importances # Macros using Parameters @@ -258,7 +258,7 @@ export @mlj_model, metadata_pkg, metadata_model export fit, update, update_data, transform, inverse_transform, fitted_params, predict, predict_mode, predict_mean, predict_median, predict_joint, - evaluate, clean!, training_losses + evaluate, clean!, training_losses, feature_importances # data operations export matrix, int, classes, decoder, table, diff --git a/src/composition/learning_networks/machines.jl b/src/composition/learning_networks/machines.jl index 8e15d86d..266af003 100644 --- a/src/composition/learning_networks/machines.jl +++ b/src/composition/learning_networks/machines.jl @@ -347,10 +347,7 @@ the following: - Calls `fit!(mach, verbosity=verbosity, acceleration=acceleration)`. -- Moves any data in source nodes of the learning network into `cache` - (for data-anonymization purposes). - -- Records a copy of `model` in `cache`. +- Records a copy of `model` in a variable called `cache`. - Returns `cache` and outcomes of training in an appropriate form (specifically, `(mach.fitresult, cache, mach.report)`; see [Adding @@ -396,17 +393,10 @@ function return!(mach::Machine{<:Surrogate}, verbosity isa Nothing || fit!(mach, verbosity=verbosity, acceleration=acceleration) setfield!(mach.fitresult, :network_model_names, network_model_names_) - # anonymize the data - sources = MLJBase.sources(glb(mach)) - data = Tuple(s.data for s in sources) - [MLJBase.rebind!(s, nothing) for s in sources] - # record the current hyper-parameter values: old_model = deepcopy(model) - cache = (sources = sources, - data=data, - old_model=old_model) + cache = (; old_model) setfield!(mach.fitresult, :network_model_names, @@ -424,9 +414,11 @@ network_model_names(model::Nothing, mach::Machine{<:Surrogate}) = """ copy_or_replace_machine(N::AbstractNode, newmodel_given_old, newnode_given_old) -For now, two top functions will lead to a call of this function: `Base.replace(::Machine, ...)` and -`save(::Machine, ...)`. A call from `Base.replace` with given `newmodel_given_old` will dispatch to this method. -A new Machine is built with training data from node N. +For now, two top functions will lead to a call of this function: +`Base.replace(::Machine, ...)` and `save(::Machine, ...)`. A call from +`Base.replace` with given `newmodel_given_old` will dispatch to this +method. A new Machine is built with training data from node N. + """ function copy_or_replace_machine(N::AbstractNode, newmodel_given_old, newnode_given_old) train_args = [newnode_given_old[arg] for arg in N.machine.args] @@ -437,13 +429,19 @@ end """ copy_or_replace_machine(N::AbstractNode, newmodel_given_old::Nothing, newnode_given_old) -For now, two top functions will lead to a call of this function: `Base.replace(::Machine, ...)` and -`save(::Machine, ...)`. A call from `save` will set `newmodel_given_old` to `nothing` which will -then dispatch to this method. -In this circumstance, the purpose is to make the machine attached to node N serializable (see `serializable(::Machine)`). +For now, two top functions will lead to a call of this function: +`Base.replace(::Machine, ...)` and `save(::Machine, ...)`. A call from +`save` will set `newmodel_given_old` to `nothing` which will then +dispatch to this method. In this circumstance, the purpose is to make +the machine attached to node N serializable (see +`serializable(::Machine)`). """ -function copy_or_replace_machine(N::AbstractNode, newmodel_given_old::Nothing, newnode_given_old) +function copy_or_replace_machine( + N::AbstractNode, + newmodel_given_old::Nothing, + newnode_given_old +) m = serializable(N.machine) m.args = Tuple(newnode_given_old[s] for s in N.machine.args) return m diff --git a/src/composition/learning_networks/nodes.jl b/src/composition/learning_networks/nodes.jl index ab4d4601..5199480c 100644 --- a/src/composition/learning_networks/nodes.jl +++ b/src/composition/learning_networks/nodes.jl @@ -21,8 +21,7 @@ The key components of a Node are: When a node `N` is called, as in `N()`, it applies the operation on the machine (if there is one) together with the outcome of calls to its node arguments, to compute the return value. For details on a -node's calling behavior, see the [`node`](ref), which is used to -construct `Node` objects. +node's calling behavior, see [`node`](@ref). See also [`node`](@ref), [`Source`](@ref), [`origins`](@ref), [`sources`](@ref), [`fit!`](@ref). diff --git a/src/composition/models/methods.jl b/src/composition/models/methods.jl index 0544523f..880b377c 100644 --- a/src/composition/models/methods.jl +++ b/src/composition/models/methods.jl @@ -28,8 +28,7 @@ function update(model::M, # Otherwise, a "smart" fit is carried out by calling `fit!` on a # greatest lower bound node for nodes in the signature of the - # underlying learning network machine. For this it is necessary to - # temporarily "de-anonymize" the source nodes. + # underlying learning network machine. network_model_names = getfield(fitresult, :network_model_names) old_model = cache.old_model @@ -40,26 +39,13 @@ function update(model::M, return fit(model, verbosity, args...) end - # return data to source nodes for fitting: - sources, data = cache.sources, cache.data - for k in eachindex(sources) - rebind!(sources[k], data[k]) - end - fit!(glb_node; verbosity=verbosity) # Retrieve additional report values report_additions_ = _call(_report_part(signature(fitresult))) - # anonymize data again: - for s in sources - rebind!(s, nothing) - end - # record current model state: - cache = (sources=cache.sources, - data=cache.data, - old_model = deepcopy(model)) - + cache = (; old_model = deepcopy(model)) + return (fitresult, cache, merge(report(glb_node), report_additions_)) diff --git a/src/machines.jl b/src/machines.jl index 93397b7f..391ecd81 100644 --- a/src/machines.jl +++ b/src/machines.jl @@ -222,7 +222,7 @@ is computed, and this is compared with the scitypes expected by the model, unless `args` contains `Unknown` scitypes and `scitype_check_level < 4`, in which case no further action is taken. Whether warnings are issued or errors thrown depends the -level. For details, see `default_scitype_check_level`](@ref), a method +level. For details, see [`default_scitype_check_level`](@ref), a method to inspect or change the default level (`1` at startup). ### Learning network machines @@ -821,7 +821,30 @@ function training_losses(mach::Machine) end end +""" + feature_importances(mach::Machine) + +Return a list of `feature => importance` pairs for a fitted machine, +`mach`, if supported by the underlying model, i.e., if +`reports_feature_importances(mach.model) == true`. Otherwise return +`nothing`. + +""" +function feature_importances(mach::Machine) + if isdefined(mach, :report) && isdefined(mach, :fitresult) + return _feature_importances(mach.model, mach.fitresult, mach.report) + else + throw(NotTrainedError(mach, :feature_importances)) + end +end +function _feature_importances(model, fitresult, report) + if reports_feature_importances(model) + return MMI.feature_importances(mach.model, fitresult, report) + else + return nothing + end +end ############################################################################### ##### SERIALIZABLE, RESTORE!, SAVE AND A FEW UTILITY FUNCTIONS ##### ############################################################################### diff --git a/src/sources.jl b/src/sources.jl index e1d056ef..f947e3f2 100644 --- a/src/sources.jl +++ b/src/sources.jl @@ -45,7 +45,7 @@ The calling behaviour of a `Source` object is this: Xs(rows=r) = selectrows(X, r) # eg, X[r,:] for a DataFrame Xs(Xnew) = Xnew -See also: [`@from_network`](@ref], [`sources`](@ref), +See also: [`@from_network`](@ref), [`sources`](@ref), [`origins`](@ref), [`node`](@ref). """ diff --git a/test/composition/models/from_network.jl b/test/composition/models/from_network.jl index 8138d107..2dcad50d 100644 --- a/test/composition/models/from_network.jl +++ b/test/composition/models/from_network.jl @@ -295,11 +295,6 @@ knn = model_.knn_rgs @test MLJBase.tree(mach.fitresult.predict).arg1.arg1.arg1.arg1.model.K == 55 -# check data anonymity: -@test all(x->(x===nothing), - [s.data for s in sources(mach.fitresult.predict)]) - - multistand = Standardizer() multistandM = machine(multistand, W) W2 = transform(multistandM, W) @@ -328,10 +323,6 @@ FP = MLJBase.fitted_params(mach) @test keys(FP) == (:one_hot, :machines, :fitted_params_given_machine) @test Set(FP.one_hot.fitresult.all_features) == Set(keys(X)) -# check data anomynity: -@test all(x->(x===nothing), - [s.data for s in sources(mach.fitresult.transform)]) - transform(mach, X); diff --git a/test/composition/models/methods.jl b/test/composition/models/methods.jl index 3959c618..33aee82b 100644 --- a/test/composition/models/methods.jl +++ b/test/composition/models/methods.jl @@ -128,10 +128,6 @@ selector_model = FeatureSelector() fitresult, cache, rep = MLJBase.fit(composite, 0, Xtrain, ytrain); - # test data anonymity: - ss = sources(glb(values(MLJBase.signature(fitresult))...)) - @test all(isempty, ss) - # to check internals: ridge = MLJBase.machines(fitresult.predict)[1] selector = MLJBase.machines(fitresult.predict)[2] diff --git a/test/machines.jl b/test/machines.jl index fb8ba5a6..f4373757 100644 --- a/test/machines.jl +++ b/test/machines.jl @@ -9,6 +9,7 @@ using Serialization using ..TestUtilities const MLJModelInterface = MLJBase.MLJModelInterface +const MMI = MLJModelInterface N=50 X = (a=rand(N), b=rand(N), c=rand(N)); @@ -32,40 +33,49 @@ pca = PCA() @test !MLJBase._contains_unknown(Union{Tuple{Int}, Tuple{Int,Char}}) end -t = machine(tree, X, y) -@test_throws MLJBase.NotTrainedError(t, :fitted_params) fitted_params(t) -@test_throws MLJBase.NotTrainedError(t, :report) report(t) -@test_throws MLJBase.NotTrainedError(t, :training_losses) training_losses(t) -@test_logs (:info, r"Training") fit!(t) -@test_logs (:info, r"Training") fit!(t, rows=train) -@test_logs (:info, r"Not retraining") fit!(t, rows=train) -@test_logs (:info, r"Training") fit!(t) -t.model.max_depth = 1 -@test_logs (:info, r"Updating") fit!(t) - -@test training_losses(t) === nothing - -predict(t, selectrows(X,test)); -@test rms(predict(t, selectrows(X, test)), y[test]) < std(y) - -mach = machine(ConstantRegressor(), X, y) -@test_logs (:info, r"Training") fit!(mach) -yhat = predict_mean(mach, X); - -n = nrows(X) -@test rms(yhat, y) ≈ std(y)*sqrt(1 - 1/n) - -# test an unsupervised univariate case: -mach = machine(UnivariateStandardizer(), float.(1:5)) -@test_logs (:info, r"Training") fit!(mach) -@test isempty(params(mach)) - -# test a frozen Machine -stand = machine(Standardizer(), source((x1=rand(10),))) -freeze!(stand) -@test_logs (:warn, r"not trained as it is frozen\.$") fit!(stand) +@testset "machine training and inpection" begin + t = machine(tree, X, y) + + @test_throws MLJBase.NotTrainedError(t, :fitted_params) fitted_params(t) + @test_throws MLJBase.NotTrainedError(t, :report) report(t) + @test_throws MLJBase.NotTrainedError(t, :training_losses) training_losses(t) + @test_throws MLJBase.NotTrainedError(t, :feature_importances) feature_importances(t) + + @test_logs (:info, r"Training") fit!(t) + @test_logs (:info, r"Training") fit!(t, rows=train) + @test_logs (:info, r"Not retraining") fit!(t, rows=train) + @test_logs (:info, r"Training") fit!(t) + t.model.max_depth = 1 + @test_logs (:info, r"Updating") fit!(t) + + # The following tests only pass when machine `t` has been fitted + @test fitted_params(t) == MMI.fitted_params(t.model, t.fitresult) + @test report(t) == t.report + @test training_losses(t) === nothing + @test feature_importances(t) === nothing + + predict(t, selectrows(X,test)); + @test rms(predict(t, selectrows(X, test)), y[test]) < std(y) + + mach = machine(ConstantRegressor(), X, y) + @test_logs (:info, r"Training") fit!(mach) + yhat = predict_mean(mach, X); + + n = nrows(X) + @test rms(yhat, y) ≈ std(y)*sqrt(1 - 1/n) + + # test an unsupervised univariate case: + mach = machine(UnivariateStandardizer(), float.(1:5)) + @test_logs (:info, r"Training") fit!(mach) + @test isempty(params(mach)) + + # test a frozen Machine + stand = machine(Standardizer(), source((x1=rand(10),))) + freeze!(stand) + @test_logs (:warn, r"not trained as it is frozen\.$") fit!(stand) +end -@testset "warnings" begin +@testset "machine instantiation warnings" begin @test_throws DimensionMismatch machine(tree, X, y[1:end-1]) # supervised model with bad target: