I have a simple question concerning the extraction of predictions when we do a nested resampling. I don't know how to extract in data table the results from the validation and test sets :
Here is my code :
set.seed(1234)
rr_xgboost = tune_nested(tuner = tnr("mbo"), task = task_imp12, learner = learner_xgboost, inner_resampling = resampling_inner, outer_resampling = resampling_outer, measure = msr("classif.auc"), term_evals = 10, store_models = TRUE, terminator = trm("none"))
xgboost_results_validation <- extract_inner_tuning_results(rr_xgboost)[, .SD, .SDcols = !c("learner_param_vals", "x_domain")]
AUC_validation = xgboost_results_validation$classif.auc
AUC_test = rr_xgboost$score(msr("classif.auc"))
predictions <- rr_xgboost$predictions()
View(predictions)
predictions
[[1]]
<PredictionClassif> for 234 observations:
row_ids truth response prob.0 prob.1
7 0 0 0.5522561 0.4477439
8 0 0 0.6626639 0.3373361
9 0 0 0.5247233 0.4752767
---
1025 0 0 0.8272567 0.1727433
1026 0 0 0.6095791 0.3904209
1027 1 0 0.7445868 0.2554132
Thanks to Marc :
Predictions_validation_xgboost <-irr$resample_result[[1]]$predictions()[[1]]
Predictions_test_xgboost <- predictions$prediction[[1]]
Predictions_validation_xgboost <- as.data.table(Predictions_validation_xgboost)
Predictions_test_xgboost <- as.data.table(Predictions_test_xgboost)
Next time please post a reproducible example.
library(mlr3verse)
tsk_sonar = tsk("sonar")
tnr_grid_search = tnr("grid_search", resolution = 5, batch_size = 5)
lrn_svm = lrn("classif.svm",
cost = to_tune(1e-5, 1e5, logscale = TRUE),
gamma = to_tune(1e-5, 1e5, logscale = TRUE),
kernel = "radial",
type = "C-classification"
)
rsmp_cv3 = rsmp("cv", folds = 3)
msr_ce = msr("classif.ce")
at = auto_tuner(
tuner = tnr_grid_search,
learner = lrn_svm,
resampling = rsmp("cv", folds = 4),
measure = msr_ce,
)
rr = resample(tsk_sonar, at, rsmp_cv3, store_models = TRUE)
rr
irr = extract_inner_tuning_archives(rr)
contains all resample results / predictions of the inner loop. You can access the first one with irr$resample_result[[1]]$predictions()
. rr$predictions()
contains the predictions of the outer loop.