Mlj.jl: Issues with some regression models [tbc]
using MLJ, RDatasets, DataFrames;
#Load & make model list.
@inline function load_m(model_list)
@inbounds for (i, model) in enumerate(model_list)
load(model.name, pkg=model.package_name, verbosity=0) #
end
end
# Load all models ONCE!
drop_pkg = ["NaiveBayes", "ScikitLearn"];
models(m -> m.package_name β drop_pkg) |> load_m
#
@inline function one_hot_encode(d::DataFrame)
encoded = DataFrame()
@inbounds for col in names(d), val in unique(d[!, col])
lab = string(col) * "_" * string(val)
encoded[!, Symbol(lab) ] = ifelse.(d[!, col] .== val, 1, 0)
end
return encoded
end
#AZ: convert Strings & Count to OHE.
@inline function AZ(X)
sch = schema(X);
#ty = [CategoricalString{UInt8}, CategoricalString{UInt32}, CategoricalValue{Int64,UInt32}]
tn = [Int, Float16, Float32, Float64]
vs = [];
@inbounds for (name, type) in zip(sch.names, sch.types)
if type β tn #β ty #β [Int32, Int64, Float64]
#println(:($name) , " ", type)
push!(vs, :($name) )
#global X = coerce(X, :($name) =>Continuous);
end
end
#
Xd= DataFrame(X);
X_ohe = one_hot_encode( Xd[:, vs] )
Xd = hcat( X_ohe, select(Xd, Not( vs )) )
Xd = coerce(Xd, autotype(Xd, :discrete_to_continuous))
#sch= schema(Xd);
#@show sch.scitypes;
#
X=Xd
return X
end
#Train & Score.
#NOTE: if we do target engineering we need to transform Y back to compare score.
@inline function train_m(m, X, y, train, test, pr, meas; invtrans=identity)
t1 = time_ns()
m = m.name
println(m)
if m =="XGBoostRegressor"
mdl = eval(Meta.parse("$(m)(num_round=500)"))
elseif m=="LGBMRegressor"
mdl = eval(Meta.parse("$(m)(num_iterations = 1_000, min_data_in_leaf=10)"))
elseif m=="EvoTreeRegressor"
mdl = eval(Meta.parse("$(m)(nrounds = 1500)"))
else
mdl = eval(Meta.parse("$(m)()"))
end
#
mach = machine(mdl, X, y)
fit!(mach, rows=train, verbosity=0) #, verbosity=0
#yΜ = MLJ.pr(mach, rows=test)
yΜ = pr(mach, rows=test)
yΜ = invtrans.(yΜ)
y = invtrans.(y)
#AZ Custom oos-R2
if meas==rmsl
s = meas(abs.(yΜ), abs.(y[test]) ) #abs.() for rmsl AMES.
else
s = meas(yΜ, y[test])
end
t2 = time_ns()
return [m,round(s, sigdigits=5), round((t2-t1)/1.0e9, sigdigits=5)]
end
#
@inline function f(X, y, train, test, pr, meas; pr_type = [:deterministic, :probabilistic])
X = AZ(X)
dropm = ["ARDRegressor"]; drop_pkg = ["NaiveBayes", "ScikitLearn"];
m_match = models(matching(X, y),
x -> x.prediction_type β pr_type,
x -> x.package_name β drop_pkg,
x -> x.name β dropm);
sc = Array{Any}(undef, size(m_match, 1), 3)
@inbounds for (i,m) in enumerate(m_match)
sc[i,:] .= try
train_m(m, X, y, train, test, pr, meas)
catch
m.name, 10_000,10_000
end
end
df= DataFrame(Model = sc[sortperm(sc[:,2]), 1],
SCORE = sc[sortperm(sc[:,2]), 2],
Time = sc[sortperm(sc[:,2]), 3])
#showtable(df)
#
return df
end
Now apply to Boston data (eg):
X, y = @load_boston;
train, test = partition(eachindex(y), .7, rng=333);
df = f(X, y, train, test, predict, rmsp, pr_type = [:deterministic])
The model LinearRegressor doesn't work.
Note: I dropped all modes in ["NaiveBayes", "ScikitLearn"]
azev77
All 14 comments
The model LinearRegressor doesn't work
Please report the error, stack trace, and ideally a minimum working example.
ablaom
on 21 Jun 2020
When I run my code to train all (deterministic regression) models the only two (out of 50) that give errors are:
MLJLinearModels.LinearRegressor() & ScikitLearn.LinearRegressor()
When I run the code separately (restarting Julia):
using MLJ
X, y = @load_boston
train, test = partition(eachindex(y), .7, rng=333);
```julia
julia> @load LinearRegressor
ERROR: LoadError: ArgumentError: Ambiguous model name. Use pkg=... .
The model LinearRegressor is provided by these packages:
["MLJLinearModels", "GLM", "ScikitLearn"].
Stacktrace:
[1] info(::String; pkg::Nothing) at /Users/AZevelev/.julia/packages/MLJModels/uSKTW/src/model_search.jl:80
[2] load(::String; pkg::Nothing, kwargs::Base.Iterators.Pairs{Symbol,Any,Tuple{Symbol,Symbol},NamedTuple{(:modl, :verbosity),Tuple{Module,Int64}}}) at /Users/AZevelev/.julia/packages/MLJModels/uSKTW/src/loading.jl:81
[3] @load(::LineNumberNode, ::Module, ::Any, ::Vararg{Any,N} where N) at /Users/AZevelev/.julia/packages/MLJModels/uSKTW/src/loading.jl:125
in expression starting at none:1
It appears there are 3 pkg w/ `LinearRegressor()`, `["MLJLinearModels", "GLM", "ScikitLearn"]`, w/ GLM `probabilistic` the other two `deterministic`.
```julia
#
@load LinearRegressor pkg = MLJLinearModels
mdl = LinearRegressor()
mach = machine(mdl, X, y)
fit!(mach, rows=train, verbosity=0)
yΜ = predict(mach, rows=test)
rmsp(yΜ, y[test])
#
@load LinearRegressor pkg = ScikitLearn
mdl = LinearRegressor()
#mdl = ScikitLearn.LinearRegressor()
mach = machine(mdl, X, y)
fit!(mach, rows=train, verbosity=0)
yΜ = predict(mach, rows=test)
rmsp(yΜ, y[test])
For reasons I can't understand: LinearRegressor works separately, but not through my framework to train all models. (before it worked fine though...)
I'm gonna have to think more about this.
azev77
on 21 Jun 2020
A few other points:
- this was discussed before: multiple models w/ the same name creates problems.
Perhaps:MLJLinearModels.LinearRegressor(),ScikitLearn.LinearRegressor(),GLM.LinearRegressor() - yesterday @OkonSamuel made
ScikitLearn.jlwork on mac again. We are lucky to have him onboard.
azev77
on 21 Jun 2020
Wait now they're working (I think you have to restart Juno a couple times...)
It's no longer working, this may be specific to my stuff, I'll have to work on it some more.
Btw, the exercise above provides a sanity check:
- the top-5 models are the boosted trees
β Row β Model β Pkg β SCORE β Time β
β β Any β Any β Any β Any β
βββββββΌβββββββββββββββββββββββββββββββββββββββΌββββββββββββββββββββΌββββββββββββΌββββββββββββ€
β 1 β EvoTreeRegressor β EvoTrees β 0.13985 β 6.662 β
β 2 β GradientBoostingRegressor β ScikitLearn β 0.14363 β 0.65507 β
β 3 β XGBoostRegressor β XGBoost β 0.14375 β 1.7388 β
β 4 β ExtraTreesRegressor β ScikitLearn β 0.15092 β 0.68245 β
β 5 β LGBMRegressor β LightGBM β 0.15601 β 3.2093 β
β 6 β RandomForestRegressor β ScikitLearn β 0.16037 β 0.0062503 β
β 7 β RandomForestRegressor β DecisionTree β 0.16325 β 0.63957 β
- Huber models give the same score
β 11 β HuberRegressor β MLJLinearModels β 0.22567 β 3.2882 β
β 12 β HuberRegressor β ScikitLearn β 0.22567 β 0.011471 β
- (Deterministic) Linear models give the same score (GLM is probabilistic & omitted here)
β 18 β LinearRegressor β MLJLinearModels β 0.25235 β 0.17905 β
β 19 β LinearRegressor β ScikitLearn β 0.25235 β 0.0013115 β
- three Ridge models give the same score
β 28 β RidgeRegressor β MLJLinearModels β 0.26502 β 2.423 β
β 29 β RidgeRegressor β MultivariateStats β 0.26502 β 0.0025984 β
β 30 β RidgeRegressor β ScikitLearn β 0.26502 β 0.0054935 β
- Two Lasso & two EN:
β 36 β LassoRegressor β MLJLinearModels β 0.32835 β 0.78851 β
β 37 β LassoRegressor β ScikitLearn β 0.32835 β 0.054286 β
β 38 β ElasticNetRegressor β MLJLinearModels β 0.32837 β 1.9358 β
β 39 β ElasticNetRegressor β ScikitLearn β 0.32837 β 0.008472 β
- Two constant regressors give the same score:
β 43 β DeterministicConstantRegressor β MLJModels β 0.59494 β 0.3227 β
β 44 β DummyRegressor β ScikitLearn β 0.59494 β 0.48613 β
- some models have suboptimal default HP:
β 45 β LassoLarsRegressor β ScikitLearn β 0.59494 β 0.42588 β
β 46 β PassiveAggressiveRegressor β ScikitLearn β 0.6607 β 0.45705 β
β 47 β GaussianProcessRegressor β ScikitLearn β 0.98654 β 0.48546 β
β 48 β SVMLRegressor β ScikitLearn β 1.0 β 0.46353 β
β 49 β SGDRegressor β ScikitLearn β 3.3938e12 β 0.47502 β
@tlienart I'm ready to start working on a tutorial.
azev77
on 22 Jun 2020
@azev77 Cool
OkonSamuel
on 22 Jun 2020
Nice!
Re multiple models with same name, please add any wishes to the discussion here: https://github.com/alan-turing-institute/MLJModels.jl/issues/242#issuecomment-647755073 . May get to this soon.
ablaom
on 23 Jun 2020
@azev77 So can we close this?
ablaom
on 24 Jun 2020
@ablaom can you give me a bit more time to get to the bottom of this?
azev77
on 24 Jun 2020
Sure - no worries! Appreciate the investigation.
ablaom
on 24 Jun 2020
@azev77 Still unresolved?
ablaom
on 23 Jul 2020
@ablaom
can we close this issue once we finish creating the tutorial: https://github.com/alan-turing-institute/DataScienceTutorials.jl/issues/47
azev77
on 23 Jul 2020
@azev77 this is unrelated, if you can run the script please kindly close this issue as the name is fairly ominous! Thanks :)
tlienart
on 24 Jul 2020
@tlienart
- I have a program to automatically train all relevant models on a given dataset
- Previously it worked for all models
- One day a few of the models didnβt work, in my script, so I opened this issue.
- Those models still work individually, outside my script.
Not sure why. - Once we get to bottom of this & publish tutorial, Iβd feel easier closing this issue.
Youβre right, the title of this issue is ominous/misleading/unfair to MLJ.
I learned this trick from our leader (potus)...
azev77
on 24 Jul 2020
Yes that analysis is fine however
- the tutorial will take some time to get integrated
- the present issue is currently not super helpful because it doesn't point to a clear problem that we could open a PR for
So I suggest
- closing this for now
- working on your suggestion in DST (which I'm very grateful for)
- flagging specific issues from there
but it's fine; I'll just change the title here.
tlienart
on 24 Jul 2020
Related issues
mllg
Β·
6Comments
CameronBieganek
Β·
6Comments
aviatesk
Β·
7Comments
juliohm
Β·
7Comments
caseykneale
Β·
6Comments
Most helpful comment
Wait now they're working (I think you have to restart Juno a couple times...)
It's no longer working, this may be specific to my stuff, I'll have to work on it some more.
Btw, the exercise above provides a sanity check:
@tlienart I'm ready to start working on a tutorial.