0

我在 30 分钟包中有大约 15 GB 的压缩数据。使用 unzip 和 readr 或 fread 解压缩和读取它们都可以，但是 ram 要求不允许我读取任意数量的文件。所以我尝试使用 disk.frame 包。原则上这也可以正常工作，但我注意到在文件中每次读取的第 4000 行左右，列变得混乱。

这是我使用的代码：

library(stringr)
library(tidyverse)
library(data.table)
library(compare)
library(disk.frame)

setup_disk.frame(future_backend=future::sequential) # tried to set sequential to avoid the problem
options(future.globals.maxSize = Inf)
   
dat_list<-list.files(pattern = ".*1936.data")

name_vec<-name_vec[c(1:6, 9:48, 51, 53:63)] # columns to drop

# the dat variable works perfectly but is memory constrained

#dat<-dat_list %>% map_df(fread, skip=7, drop=name_vec, #data.table=getOption("datatable.fread.datatable", T))
#Date_Time<-paste(dat$Date, str_sub(dat$Time, 1,8))
#dat<-dat[,-c(1:2)]
#dat<-cbind.data.frame(date=lubridate::ymd_hms(Date_Time, tz="UTC"), dat)

# disk frame trial

outdir="D:/***/Test"
test<-csv_to_disk.frame(infile=dat_list, outdir = outdir, 
                        skip=7, .progress=T, drop = name_vec, header=T, overwrite = T,
                        inmapfn = function(chunk){
                          chunk[, Date := lubridate::ymd_hms(paste(Date, str_sub(Time, 1,8)))] 
                          # trying to create a combined date_time variable from date analogue to the Date_Time variable above
                        }, 
                        data.table=getOption("datatable.fread.datatable", T))

test<-data.frame(test[,-2])
#dat<-data.frame(dat)
#compare(dat, test)

读取数据后， dat的 df 看起来像这样：

structure(list(date = structure(c(1554727203, 1554727203, 1554727203, 
1554727203, 1554727203, 1554727203, 1554727203, 1554727204, 1554727204, 
1554727204, 1554727204, 1554727204, 1554727204, 1554727204, 1554727204, 
1554727204, 1554727204, 1554727205, 1554727205, 1554727205, 1554727205
), class = c("POSIXct", "POSIXt"), tzone = "UTC"), `U (m/s)` = c(-3.59775, 
-3.89427, -3.78592, -3.93987, -4.14395, -4.22348, -4.27332, -4.34219, 
-4.46859, -4.71244, -4.39688, -4.39266, -4.04464, -4.23887, -4.43878, 
-4.46269, -4.55271, -4.45263, -4.50232, -4.35592, -4.07062), 
    `V (m/s)` = c(-1.49433, -1.79746, -1.69747, -1.41175, -1.80788, 
    -1.84414, -1.67488, -1.48056, -1.49211, -1.51781, -1.80034, 
    -1.86993, -1.82314, -1.54926, -1.37781, -1.51184, -1.41061, 
    -1.43523, -0.683048, -0.559152, -0.420025), `T (C)` = c(21.1527, 
    21.214, 21.195, 21.1651, 21.1972, 21.0915, 20.7849, 20.3886, 
    20.4152, 20.8369, 20.9407, 21.1197, 21.033, 20.7123, 20.8921, 
    21.0232, 21.1044, 21.157, 21.1208, 21.1468, 21.1597)), row.names = 3980:4000, class = "data.frame")

像这样进行测试：

structure(list(Date = structure(c(1554727203, 1554727203, 1554727203, 
1554727203, 1554727203, 1554727203, 1554727203, 1554727204, 1554728400, 
1554728400, 1554728400, 1554728400, 1554728400, 1554728400, 1554728400, 
1554728400, 1554728400, 1554728400, 1554728401, 1554728401, 1554728401
), class = c("POSIXct", "POSIXt"), tzone = "UTC"), U..m.s. = c(-3.59775, 
-3.89427, -3.78592, -3.93987, -4.14395, -4.22348, -4.27332, -4.34219, 
-3.72044, -2.68918, -3.11362, -3.84935, -3.80292, -3.54106, -3.77755, 
-3.2498, -3.14659, -2.9482, -2.90917, -2.70361, -2.5597), V..m.s. = c(-1.49433, 
-1.79746, -1.69747, -1.41175, -1.80788, -1.84414, -1.67488, -1.48056, 
0.779225, 0.753698, 1.43587, 0.452789, 0.228636, -1.49971, -0.840048, 
-0.723638, -0.49741, -0.27166, -0.118487, -0.0760538, -0.107277
), T..C. = c(21.1527, 21.214, 21.195, 21.1651, 21.1972, 21.0915, 
20.7849, 20.3886, 21.8011, 21.7274, 21.7481, 21.7349, 21.7759, 
21.7998, 21.5799, 21.5692, 21.5885, 21.5234, 21.4854, 21.4857, 
21.5471)), row.names = 3980:4000, class = "data.frame")

这些分别是第 3980:4000 行，您可以看到，如果您对其进行测试，它们与第 3888 行不同。这些仍然是来自相同原始 30 分钟包的值。我不知道为什么会这样。我认为它可能是“工人”或者它可能是“时间”列，但改变任何一个似乎都没有多大作用。任何帮助将不胜感激。

PS：Windows 10、8GB 内存、R 4.0.2

4

1 回答 1

1

我看不出有什么不对。一件事是您不能假设行在disk.frame 中的顺序相同。

是否能够为每一行添加一个唯一的 ID？那你可以通过id比较吗？

于 2020-10-03T13:14:40.863 回答