我想在 mlr3 中创建一个自定义的 Precision-Recall AUC 度量。
我觉得我快到了，但是 R 抛出了一个我不知道如何解释的恼人的错误。
让我们定义度量：
PRAUC = R6::R6Class("PRAUC",
inherit = mlr3::MeasureClassif,
public = list(
initialize = function() {
super$initialize(
# custom id for the measure
id = "classif.prauc",
# additional packages required to calculate this measure
packages = c('PRROC'),
# properties, see below
properties = character(),
# required predict type of the learner
predict_type = "prob",
# feasible range of values
range = c(0, 1),
# minimize during tuning?
minimize = FALSE
)
}
),
private = list(
# custom scoring function operating on the prediction object
.score = function(prediction, ...) {
truth1 <- ifelse(prediction$truth == levels(prediction$truth)[1], 1, 0) # Function PRROC::pr.curve assumes binary response is numeric, positive class is 1, negative class is 0
PRROC::pr.curve(scores.class0 = prediction$prob, weights.class0 = truth1)
}
)
)
mlr3::mlr_measures$add("classif.prauc", PRAUC)
让我们看看它是否有效：
task_sonar <- tsk('sonar')
learner <- lrn('classif.rpart', predict_type = 'prob')
learner$train(task_sonar)
pred <- learner$predict(task_sonar)
pred$score(msr('classif.prauc'))
# Error in if (sum(weights < 0) != 0) { :
# missing value where TRUE/FALSE needed
这是回溯：
11.
check(length(sorted.scores.class0), weights.class0)
10.
compute.pr(scores.class0, scores.class1, weights.class0, weights.class1,
curve, minStepSize, max.compute, min.compute, rand.compute,
dg.compute)
9.
PRROC::pr.curve(scores.class0 = prediction$prob, weights.class0 = truth1)
8.
measure$.__enclos_env__$private$.score(prediction = prediction,
task = task, learner = learner, train_set = train_set)
7.
measure_score(self, prediction, task, learner, train_set)
6.
m$score(prediction = self, task = task, learner = learner, train_set = train_set)
5.
FUN(X[[i]], ...)
4.
vapply(.x, .f, FUN.VALUE = .value, USE.NAMES = FALSE, ...)
3.
map_mold(.x, .f, NA_real_, ...)
2.
map_dbl(measures, function(m) m$score(prediction = self, task = task,
learner = learner, train_set = train_set))
1.
pred$score(msr("classif.prauc"))
似乎故障来自PRROC::pr.curve. 但是，在实际预测对象上尝试此功能时pred，它工作得很好：
PRROC::pr.curve(
scores.class0 = pred$prob[, 1],
weights.class0 = ifelse(pred$truth == levels(pred$truth)[1], 1, 0)
)
# Precision-recall curve
#
# Area under curve (Integral):
# 0.9081261
#
# Area under curve (Davis & Goadrich):
# 0.9081837
#
# Curve not computed ( can be done by using curve=TRUE )
发生错误的一种可能情况是，在内部PRAUC，PRROC::pr.curve的参数weights.class0是NA。我无法确认这一点，但我怀疑weights.class0是接收NA而不是数字，导致PRROC::pr.curve内部出现故障PRAUC。如果是这样的话，我不知道为什么会这样。
可能还有其他我没有想到的场景。任何帮助都感激不尽。
编辑
misuse的回答帮助我意识到为什么我的措施不起作用。第一的，
PRROC::pr.curve(scores.class0 = prediction$prob, weights.class0 = truth1)
应该
PRROC::pr.curve(scores.class0 = prediction$prob[, 1], weights.class0 = truth1).
其次，函数pr.curve返回一个类对象PRROC，而mlr3我定义的度量实际上是期望numeric的。所以应该是
PRROC::pr.curve(scores.class0 = prediction$prob[, 1], weights.class0 = truth1)[[2]]
或者
PRROC::pr.curve(scores.class0 = prediction$prob[, 1], weights.class0 = truth1)[[3]],
取决于用于计算 AUC 的方法（请参阅参考资料?PRROC::pr.curve）。
请注意，虽然MLmetrics::PRAUC远没有那么令人困惑PRROC::pr.curve，但似乎前者实施得不好。
PRROC::pr.curve这是实际有效的措施的实现：
PRAUC = R6::R6Class("PRAUC",
inherit = mlr3::MeasureClassif,
public = list(
initialize = function() {
super$initialize(
# custom id for the measure
id = "classif.prauc",
# additional packages required to calculate this measure
packages = c('PRROC'),
# properties, see below
properties = character(),
# required predict type of the learner
predict_type = "prob",
# feasible range of values
range = c(0, 1),
# minimize during tuning?
minimize = FALSE
)
}
),
private = list(
# custom scoring function operating on the prediction object
.score = function(prediction, ...) {
truth1 <- ifelse(prediction$truth == levels(prediction$truth)[1], 1, 0) # Looks like in mlr3 the positive class in binary classification is always the first factor level
PRROC::pr.curve(
scores.class0 = prediction$prob[, 1], # Looks like in mlr3 the positive class in binary classification is always the first of two columns
weights.class0 = truth1
)[[2]]
}
)
)
mlr3::mlr_measures$add("classif.prauc", PRAUC)
例子：
task_sonar <- tsk('sonar')
learner <- lrn('classif.rpart', predict_type = 'prob')
learner$train(task_sonar)
pred <- learner$predict(task_sonar)
pred$score(msr('classif.prauc'))
#classif.prauc
# 0.923816
然而，现在的问题是改变正类会导致不同的分数：
task_sonar <- tsk('sonar')
task_sonar$positive <- 'R' # Now R is the positive class
learner <- lrn('classif.rpart', predict_type = 'prob')
learner$train(task_sonar)
pred <- learner$predict(task_sonar)
pred$score(msr('classif.prauc'))
#classif.prauc
# 0.9081261