Super Learner Benchmarks

Legacy SuperLearner

The legacy SuperLearner package serves as a suitable baseline. We can fit it sequentially (no parallelization):

time_SuperLearner_sequential <- system.time({
  SuperLearner(task$Y, as.data.frame(task$X), newX = NULL, family = gaussian(), 
               SL.library = c("SL.glmnet", "SL.randomForest", "SL.speedglm"),
               method = "method.NNLS", id = NULL, verbose = FALSE,
               control = list(), cvControl = list(), obsWeights = NULL,
               env = parent.frame())
})

We can also fit it using multicore parallelization, using the mcSuperLearner function.

options(mc.cores = cpus_physical)
time_SuperLearner_multicore <- system.time({
  mcSuperLearner(task$Y, as.data.frame(task$X), newX = NULL,
                 family = gaussian(),
                 SL.library = c("SL.glmnet", "SL.randomForest", "SL.speedglm"),
                 method = "method.NNLS", id = NULL, verbose = FALSE,
                 control = list(), cvControl = list(), obsWeights = NULL,
                 env = parent.frame())
})

The SuperLearner package supports a number of other parallelization schemes, although these weren’t tested here.

sl3 with Legacy SuperLearner Wrappers

To maximize comparability with the legacy implementation, we can use sl3 with the SuperLearner wrappers, so that the actual computation used to train the learners is identical:

sl_glmnet <- Lrnr_pkg_SuperLearner$new("SL.glmnet")
sl_random_forest <- Lrnr_pkg_SuperLearner$new("SL.randomForest")
sl_speedglm <- Lrnr_pkg_SuperLearner$new("SL.speedglm")
nnls_lrnr <- Lrnr_nnls$new()

sl3_legacy <- Lrnr_sl$new(list(sl_random_forest, sl_glmnet, sl_speedglm),
                          nnls_lrnr)

sl3 with Native Learners

We can also use native sl3 learners, which have been rewritten to be performant on large sample sizes:

lrnr_glmnet <- Lrnr_glmnet$new()
random_forest <- Lrnr_randomForest$new()
glm_fast <- Lrnr_glm_fast$new()
nnls_lrnr <- Lrnr_nnls$new()

sl3_native <- Lrnr_sl$new(list(random_forest, lrnr_glmnet, glm_fast), nnls_lrnr)

sl3 Parallelization Options

sl3 uses the delayed package to parallelize training tasks. Delayed, in turn, uses the future package to support a range of parallel back-ends. We test several of these, for both the legacy wrappers and native learners.

First, sequential evaluation (no parallelization):

plan(sequential)
test <- delayed_learner_train(sl3_legacy, task)
time_sl3_legacy_sequential <- system.time({
  sched <- Scheduler$new(test, SequentialJob)
  cv_fit <- sched$compute()
})

test <- delayed_learner_train(sl3_native, task)
time_sl3_native_sequential <- system.time({
  sched <- Scheduler$new(test, SequentialJob)
  cv_fit <- sched$compute()
})

Next, multicore parallelization:

plan(multicore, workers = cpus_physical)
test <- delayed_learner_train(sl3_legacy, task)
time_sl3_legacy_multicore <- system.time({
  sched <- Scheduler$new(test, FutureJob, nworkers = cpus_physical,
                         verbose = FALSE)
  cv_fit <- sched$compute()
})

test <- delayed_learner_train(sl3_native, task)
time_sl3_native_multicore <- system.time({
  sched <- Scheduler$new(test, FutureJob, nworkers = cpus_physical,
                         verbose = FALSE)
  cv_fit <- sched$compute()
})

We also test multicore parallelization with hyper-threading – we use a number of workers equal to the number of logical, not physical, cores:

plan(multicore, workers = cpus_logical)
test <- delayed_learner_train(sl3_legacy, task)
time_sl3_legacy_multicore_ht <- system.time({
  sched <- Scheduler$new(test, FutureJob, nworkers = cpus_logical,
                         verbose = FALSE)
  cv_fit <- sched$compute()
})

test <- delayed_learner_train(sl3_native, task)
time_sl3_native_multicore_ht <- system.time({
  sched <- Scheduler$new(test, FutureJob, nworkers = cpus_logical,
                         verbose = FALSE)
  cv_fit <- sched$compute()
})

Finally, we test parallelization using multisession:

plan(multisession, workers = cpus_physical)
test <- delayed_learner_train(sl3_legacy, task)
time_sl3_legacy_multisession <- system.time({
  sched <- Scheduler$new(test, FutureJob, nworkers = cpus_physical,
                         verbose = FALSE)
  cv_fit <- sched$compute()
})

test <- delayed_learner_train(sl3_native, task)
time_sl3_native_multisession <- system.time({
  sched <- Scheduler$new(test, FutureJob, nworkers = cpus_physical,
                         verbose = FALSE)
  cv_fit <- sched$compute()
})