根据 NeronLeVelu 和 Clayton Stanlay 的答案，我使用自定义函数、示例数据和一些 system.time() 来完成答案以进行比较。这些测试是在 Mac os 10.9 和 R 3.0.2 上进行的。但是，我在 linux 机器上进行了相同的测试，与预先计算了 nrows 和 colClasses 的 read.table() 相比，sed 命令的执行速度非常慢。fread 部分非常快，两个系统上的 5e6 行大约需要 5 秒。
library(data.table)
# create path to new temporary file
origData <- tempfile(pattern="origData",fileext=".txt")
# write table with irregular blank spaces separators.
write(paste0(" YYYY MM DD HH mm 19490 40790","\n",
paste(rep(" 1991 10 1 1 0 1.046465E+00 1.568405E+00", 5e6),
collapse="\n"),"\n"),
file=origData
)
# define column classes for read.table() optimization
colClasses <- c(rep('integer',5),rep('numeric',2))
# Function to count rows with command wc for read.table() optimization.
fileRowsCount <- function(file){
if(file.exists(file)){
sysCmd <- paste("wc -l", file)
rowCount <- system(sysCmd, intern=T)
rowCount <- sub('^\\s', '', rowCount)
as.numeric(
strsplit(rowCount, '\\s')[[1]][1]
)
}
}
# Function to sed data into temp file before importing with sed
sedFread<-function(file, sedCmd=NULL, ...){
require(data.table)
if(is.null(sedCmd)){
#default : sed for convert blank separated table to csv. Thanks NeronLevelu !
sedCmd <- "'s/^[[:blank:]]*//;s/[[:blank:]]\\{1,\\}/,/g'"
}
#sed into temp file
tmpPath<-tempfile(pattern='tmp',fileext='.txt')
sysCmd<-paste('sed',sedCmd, file, '>',tmpPath)
try(system(sysCmd))
DT<-fread(tmpPath,...)
try(system(paste('rm',tmpPath)))
return(DT)
}
Mac 操作系统结果：
# First sed into temp file and then fread.
system.time(
DT<-sedFread(origData, header=TRUE)
)
> user system elapsed
> 23.847 0.628 24.514
# Sed directly in fread command :
system.time(
DT <- fread(paste("sed 's/^[[:blank:]]*//;s/[[:blank:]]\\{1,\\}/,/g'", origData),
header=T)
)
> user system elapsed
> 23.606 0.515 24.219
# read.table without nrows and colclasses
system.time(
DF<-read.table(origData, header=TRUE)
)
> user system elapsed
> 38.053 0.512 38.565
# read.table with nrows an colclasses
system.time(
DF<-read.table(origData, header=TRUE, nrows=fileRowsCount(origData), colClasses=colClasses)
)
> user system elapsed
> 33.813 0.309 34.125
Linux结果：
# First sed into temp file and then fread.
system.time(
DT<-sedFread(origData, header=TRUE)
)
> Read 5000000 rows and 7 (of 7) columns from 0.186 GB file in 00:00:05
> user system elapsed
> 47.055 0.724 47.789
# Sed directly in fread command :
system.time(
DT <- fread(paste("sed 's/^[[:blank:]]*//;s/[[:blank:]]\\{1,\\}/,/g'", origData),
header=T)
)
> Read 5000000 rows and 7 (of 7) columns from 0.186 GB file in 00:00:05
> user system elapsed
> 46.088 0.532 46.623
# read.table without nrows and colclasses
system.time(
DF<-read.table(origData, header=TRUE)
)
> user system elapsed
> 32.478 0.436 32.912
# read.table with nrows an colclasses
system.time(
DF<-read.table(origData,
header=TRUE,
nrows=fileRowsCount(origData),
colClasses=colClasses)
)
> user system elapsed
> 21.665 0.524 22.192
# Control if DT and DF are identical :
setnames(DT, old=names(DT), new=names(DF))
identical(as.data.frame(DT), DF)
>[1] TRUE
很好：在这种情况下，我首先使用的方法是最有效的。
感谢 NeronLeVelu、Matt Dowle 和 Clayton Stanley！