3

我正在尝试使用 ddply 清理数据，但它在 1.3M 行上运行非常缓慢。

示例代码：

#Create Sample Data Frame
num_rows <- 10000
df <- data.frame(id=sample(1:20, num_rows, replace=T), 
                Consumption=sample(-20:20, num_rows, replace=T), 
                StartDate=as.Date(sample(15000:15020, num_rows, replace=T), origin = "1970-01-01"))
df$EndDate <- df$StartDate + 90
#df <- df[order(df$id, df$StartDate, df$Consumption),]
#Are values negative? 
# Needed for subsetting in ddply rows with same positive and negative values
df$Neg <- ifelse(df$Consumption < 0, -1, 1)
df$Consumption <- abs(df$Consumption)

我编写了一个函数来删除一行中的消耗值与另一行中的消耗值相同但为负的行（对于相同的 id）。

#Remove rows from a data frame where there is an equal but opposite consumption value
#Should ensure only one negative value is removed for each positive one. 
clean_negatives <- function(x3){
  copies <- abs(sum(x3$Neg))
  sgn <- ifelse(sum(x3$Neg) <0, -1, 1) 
  x3 <- x3[0:copies,]
  x3$Consumption <- sgn*x3$Consumption
  x3$Neg <- NULL
  x3}

然后我使用 ddply 应用该函数来删除数据中的这些错误行

ptm <- proc.time()
df_cleaned <- ddply(df, .(id,StartDate, EndDate, Consumption),
                    function(x){clean_negatives(x)})
proc.time() - ptm

我希望我可以使用 data.table 来加快速度，但我不知道如何使用 data.table 来提供帮助。

有 130 万行，到目前为止，我的桌面要花一整天的时间来计算，但还没有完成。

4

1 回答 1

6

您的问题询问data.table实施。所以，我在这里展示了它。您的功能也可以大大简化。可以先sign求和Neg，然后过滤表，再乘以Consumption（sign如下图）。

require(data.table)
# get the data.table in dt
dt <- data.table(df, key = c("id", "StartDate", "EndDate", "Consumption"))
# first obtain the sign directly
dt <- dt[, sign := sign(sum(Neg)), by = c("id", "StartDate", "EndDate", "Consumption")]
# then filter by abs(sum(Neg))
dt.fil <- dt[, .SD[seq_len(abs(sum(Neg)))], by = c("id", "StartDate", "EndDate", "Consumption")]
# modifying for final output (line commented after Statquant's comment
# dt.fil$Consumption <- dt.fil$Consumption * dt.fil$sign
dt.fil[, Consumption := (Consumption*sign)]
dt.fil <- subset(dt.fil, select=-c(Neg, sign))

基准测试

  • 百万行数据：

    #Create Sample Data Frame
num_rows <- 1e6
df <- data.frame(id=sample(1:20, num_rows, replace=T), 
                Consumption=sample(-20:20, num_rows, replace=T), 
                StartDate=as.Date(sample(15000:15020, num_rows, replace=T), origin = "1970-01-01"))
df$EndDate <- df$StartDate + 90
df$Neg <- ifelse(df$Consumption < 0, -1, 1)
df$Consumption <- abs(df$Consumption)

  • data.table功能：

    FUN.DT <- function() {
    require(data.table)
    dt <- data.table(df, key=c("id", "StartDate", "EndDate", "Consumption"))
    dt <- dt[, sign := sign(sum(Neg)), 
               by = c("id", "StartDate", "EndDate", "Consumption")]
    dt.fil <- dt[, .SD[seq_len(abs(sum(Neg)))], 
               by=c("id", "StartDate", "EndDate", "Consumption")]
    dt.fil[, Consumption := (Consumption*sign)]
    dt.fil <- subset(dt.fil, select=-c(Neg, sign))
}

  • 你的功能与ddply

    FUN.PLYR <- function() {
    require(plyr)
    clean_negatives <- function(x3) {
        copies <- abs(sum(x3$Neg))
        sgn <- ifelse(sum(x3$Neg) <0, -1, 1) 
        x3 <- x3[0:copies,]
        x3$Consumption <- sgn*x3$Consumption
        x3$Neg <- NULL
        x3
    }
    df_cleaned <- ddply(df, .(id, StartDate, EndDate, Consumption), 
                           function(x) clean_negatives(x))
}

  • 基准测试rbenchmark（仅运行 1 次）

    require(rbenchmark)
benchmark(FUN.DT(), FUN.PLYR(), replications = 1, order = "elapsed")

        test replications elapsed relative user.self sys.self user.child sys.child
1   FUN.DT()            1   6.137    1.000     5.926    0.211          0         0
2 FUN.PLYR()            1 242.268   39.477   152.855    82.881         0         0

我的data.table实现比您当前的plyr实现快大约 39 倍（我将我的实现与您的实现进行比较，因为功能不同）。

Note:我在函数内加载包是为了获取完整的时间来获取结果。此外，出于同样的原因，我在基准测试功能中将其转换data.frame为data.tablewith 键。因此，这是最小的加速。

于 2013-01-22T08:52:28.600 回答