为什么“rbind”比“set”更快地增长数据表？

Question

我正在开发一个需要读取和组合大量数据表的模型。虽然数据表都有相同的列和相同的标题，但每个表都有不同的行数。我当前执行此操作的方法效率低下，并且根据要组合的数据表的数量，可能需要数小时。

我目前的做法是使用rbind来组合数据表；下面是一个具有更小数据集的可重现示例：

library(data.table)

old.way <- function() {
wildfire_data <- data.table()

for(tile in 1:3) {
# Normally this data would be read in from an external file, but we'll make some dummy data for this example
new_wildfire_data <- data.table(x = sample(1:1e6,1000), y = sample(1:1e6,1000), total_PM10 = sample(1:1e6,1000),
total_PM2.5 = sample(1:1e6,1000), total_CH4 = sample(1:1e6,1000), total_CO = sample(1:1e6,1000), total_CO2 = sample(1:1e6,1000), total_NOx = sample(1:1e6,1000), total_SO2 = sample(1:1e6,1000), total_VOC = sample(1:1e6,1000), total_char = sample(1:1e6,1000))

wildfire_data <- rbind(wildfire_data,new_wildfire_data)
}
return(wildfire_data)
}

看看其他问题，这看起来是一种低效的方法 (Growing a data.frame in a memory-efficient manner)，我应该改为预先分配大小并在 for 循环中使用数据表的“set”函数，填充空数据表。我也试过了：

new.way <- function() {
num.needed.rows <- 3000

# Create a data table of a pre-allocated size    
wildfire_data <- data.table(x = integer(num.needed.rows), y = integer(num.needed.rows), total_PM10 = integer(num.needed.rows), total_PM2.5 = integer(num.needed.rows), total_CH4 = integer(num.needed.rows), total_CO = integer(num.needed.rows), total_CO2 = integer(num.needed.rows), total_NOx = integer(num.needed.rows), total_SO2 = integer(num.needed.rows), total_VOC = integer(num.needed.rows), total_char = integer(num.needed.rows))

start.row <- as.integer(0)

for(tile in 1:3) {
# Again, this data would normally be read in from an external file
new_wildfire_data <- data.table(x = sample(1:1e6,1000), y = sample(1:1e6,1000), total_PM10 = sample(1:1e6,1000),
total_PM2.5 = sample(1:1e6,1000), total_CH4 = sample(1:1e6,1000), total_CO = sample(1:1e6,1000), total_CO2 = sample(1:1e6,1000), total_NOx = sample(1:1e6,1000), total_SO2 = sample(1:1e6,1000), total_VOC = sample(1:1e6,1000), total_char = sample(1:1e6,1000))

for(raw.data.row.i in 1:nrow(new_wildfire_data)) {
set(wildfire_data,start.row + raw.data.row.i,"x", new_wildfire_data[raw.data.row.i,x])
set(wildfire_data,start.row + raw.data.row.i,"y", new_wildfire_data[raw.data.row.i,y])
set(wildfire_data,start.row + raw.data.row.i,"total_PM10", new_wildfire_data[raw.data.row.i,total_PM10])
set(wildfire_data,start.row + raw.data.row.i,"total_PM2.5", new_wildfire_data[raw.data.row.i,total_PM2.5])
set(wildfire_data,start.row + raw.data.row.i,"total_PM2.5", new_wildfire_data[raw.data.row.i,total_PM2.5])
set(wildfire_data,start.row + raw.data.row.i,"total_CH4", new_wildfire_data[raw.data.row.i,total_CH4])
set(wildfire_data,start.row + raw.data.row.i,"total_CO", new_wildfire_data[raw.data.row.i,total_CO])
set(wildfire_data,start.row + raw.data.row.i,"total_CO2", new_wildfire_data[raw.data.row.i,total_CO2])
set(wildfire_data,start.row + raw.data.row.i,"total_NOx", new_wildfire_data[raw.data.row.i,total_NOx])
set(wildfire_data,start.row + raw.data.row.i,"total_SO2", new_wildfire_data[raw.data.row.i,total_SO2])
set(wildfire_data,start.row + raw.data.row.i,"total_VOC", new_wildfire_data[raw.data.row.i,total_VOC])
set(wildfire_data,start.row + raw.data.row.i,"total_char", new_wildfire_data[raw.data.row.i,total_char])
}
start.row <- start.row + nrow(new_wildfire_data)
}
return(wildfire_data)
}

但新方法要慢得多。这是我的基准测试结果：

library(microbenchmark)
microbenchmark(old.way(),new.way(),times=2

Unit: milliseconds
      expr         min          lq        mean      median          uq         max neval
 old.way()    24.29792    24.29792    25.06512    25.06512    25.83233    25.83233     2
 new.way() 12961.41358 12961.41358 13070.96187 13070.96187 13180.51016 13180.51016     2

是否有正确的方法来使用“set”，从而比使用“rbind”效率更高？

Answer 1

set 通常是 := 的替代品，用于快速分配给 data.table 的元素。 This 是其正常使用方式之一。

正如 chinsoon12 指出的那样，rbindlist(lapply(filepaths, fread)) 在这里应该是一个更快的解决方案。就给出的示例而言，一种选择是定义正确尺寸的列表并使用rbindlist：

list.way <- function() {
wildfire_data_list <- vector("list", length = 3)
for(tile in 1:3) {
    # Normally this data would be read in from an external file, but we'll make some dummy data for this example
    new_wildfire_data <- data.table(x = sample(1:1e6,1000), y = sample(1:1e6,1000), total_PM10 = sample(1:1e6,1000),
                                    total_PM2.5 = sample(1:1e6,1000), total_CH4 = sample(1:1e6,1000), total_CO = sample(1:1e6,1000), total_CO2 = sample(1:1e6,1000), total_NOx = sample(1:1e6,1000), total_SO2 = sample(1:1e6,1000), total_VOC = sample(1:1e6,1000), total_char = sample(1:1e6,1000))

    wildfire_data_list[[tile]] <- new_wildfire_data
}
wildfire_data <- rbindlist(wildfire_data_list)
return(wildfire_data)
}