1

我正在尝试在 R 中使用 bigmemory 来计算超过 100,00,000 行（粗略估计）和 16 列的距离矩阵

一小部分数据看起来像这样

list1 <- data.frame(longitude = c(80.15998, 72.89125, 77.65032, 77.60599, 
                                  72.88120, 76.65460, 72.88232, 77.49186, 
                                  72.82228, 72.88871), 
                    latitude = c(12.90524, 19.08120, 12.97238, 12.90927, 
                                 19.08225, 12.81447, 19.08241, 13.00984,
                                 18.99347, 19.07990))
list2 <- data.frame(longitude = c(72.89537, 77.65094, 73.95325, 72.96746, 
                                  77.65058, 77.66715, 77.64214, 77.58415,
                                  77.76180, 76.65460), 
                    latitude = c(19.07726, 13.03902, 18.50330, 19.16764, 
                                 12.90871, 13.01693, 13.00954, 12.92079,
                                 13.02212, 12.81447), 
                    locality = c("A", "A", "B", "B", "C", "C", "C", "D", "D", "E"))


library(geosphere)

# create distance matrix
mat <- distm(list1[,c('longitude','latitude')], list2[,c('longitude','latitude')], fun=distHaversine)

# assign the name to the point in list1 based on shortest distance in the matrix
list1$locality <- list2$locality[max.col(-mat)]

如何使用 bigmemory 构建海量 dist 矩阵？

4

1 回答 1

3

像这样的东西对我有用：

library(bigmemory)
library(foreach)

CutBySize <- function(m, block.size, nb = ceiling(m / block.size)) {
  int <- m / nb
  upper <- round(1:nb * int)
  lower <- c(1, upper[-nb] + 1)
  size <- c(upper[1], diff(upper))
  cbind(lower, upper, size)
}

seq2 <- function(lims) {
  seq(lims[1], lims[2])
}

n <- nrow(list1)
a <- big.matrix(n, n, backingfile = "my_dist.bk",
                descriptorfile = "my_dist.desc")

intervals <- CutBySize(n, block.size = 1000)
K <- nrow(intervals)

doParallel::registerDoParallel(parallel::detectCores() / 2)
foreach(j = 1:K) %dopar% {
  ind_j <- seq2(intervals[j, ])
  foreach(i = j:K) %do% {
    ind_i <- seq2(intervals[i, ])
    tmp <- distm(list1[ind_i, c('longitude', 'latitude')], 
                 list2[ind_j, c('longitude', 'latitude')], 
                 fun = distHaversine)
    a[ind_i, ind_j] <- tmp
    a[ind_j, ind_i] <- t(tmp)
    NULL
  }
}
doParallel::stopImplicitCluster()

我重复了您的列表 1000 次以测试 10K 行。

于 2017-06-02T06:08:07.707 回答