如何展示 dplyr 连接动词的威力？

Question

我一直想向朋友展示在基本 R 和简单子集上使用 dplyr 的连接动词（例如 inner_join()）的优雅和速度。拿了一个大数据库（来自nycflights13 包），从一个简单的任务开始，令我惊讶的是，基础 R 和简单的子集设置速度快了 10 倍！而且我只能真正展示优雅，而不是速度。

问题是：我错过了什么，dplyr 的连接动词何时在性能上超过基础 R 和简单子集？他们有吗？...

（P.S.：我知道data.table的出色表现，询问dplyr）

我的演示：

library(tidyverse)
library(nycflights13)
library(microbenchmark)

dim(flights)

[1] 336776 19

dim(airports)

[1] 1458 8

任务是：在destination airport tzone 为“America/New_York”的航班中获取所有飞机的唯一 tailnums：

base_no_join <- function() {
  unique(flights$tailnum[flights$dest %in% airports$faa[airports$tzone == "America/New_York"]])
}

dplyr_no_join <- function() {
  flights %>%
    filter(dest %in% (airports %>%
                           filter(tzone=="America/New_York") %>%
                           pull(faa))) %>%
    pull(tailnum) %>%
    unique()
}

dplyr_join <- function() {
  flights %>%
    inner_join(airports, by = c("dest" = "faa")) %>%
    filter(tzone == "America/New_York") %>%
    pull(tailnum) %>%
    unique()
}

看到它们给出相同的结果：

all.equal(dplyr_join(), dplyr_no_join())

[1] TRUE

all.equal(dplyr_join(), base_no_join())

[1] TRUE

现在基准测试：

microbenchmark(base_no_join(), dplyr_no_join(), dplyr_join(), times = 10)

Unit: milliseconds
            expr     min      lq     mean   median       uq      max neval
  base_no_join()  9.7198 10.1067 13.16934 11.19465  13.4736  24.2831    10
 dplyr_no_join() 21.2810 22.9710 36.04867 26.59595  34.4221 108.0677    10
    dplyr_join() 60.7753 64.5726 93.86220 91.10475 119.1546 137.1721    10

如果存在，请帮助找到一个显示此连接优势的示例。

Answer 1

为了证明@Gregor 的观点，你可以从这样的事情开始。

1. 生成两个 data.frames 和 nr = 10^3 行，我们将使用基于两个键列的左连接来合并它们。

   set.seed(2018)
   nr <- 10^3
   lst <- replicate(2, data.frame(
       key1 = sample(letters[1:5], nr, replace = T),
       key2 = sample(LETTERS[6:10], nr, replace = T),
       value = runif(nr)), simplify = F)
   

2. 比较他们在microbenchmark中的表现

   library(microbenchmark)
   res <- microbenchmark(
       base_R = merge(lst[[1]], lst[[2]], by = c("key1", "key2"), all.x = T),
       dplyr_join = left_join(lst[[1]], lst[[2]], by = c("key1", "key2")))
   #Unit: milliseconds
   #       expr        min         lq       mean     median         uq        max
   #     base_R 148.570172 151.377020 172.251324 153.904316 172.202578 493.431178
   # dplyr_join   2.397498   2.962557   3.539393   3.275512   3.751469   7.794915
   # neval
   #   100
   #   100
   
   library(ggplot2)
   autoplot(res)
   

Answer 2

我将在@Gregor 的 cmets 和 @Maurits 答案的基础上编写我自己的答案（希望这不是粗鲁，我会解释为什么我不接受 @Maurits 的答案）：

首先，我认为@Maurits 的答案通过比较基本 R 的merge() 使用与多列的连接来有点忽略了这一点。因为dplyr 的加入速度更快，即使在加入第二列时，如下所示。

其次，我知道在我最初的用例中，答案只是不使用连接（！）。但我希望表明更手动的方法会更慢，所以我实现了另一个手动连接来显示它：

library(tidyverse)
library(microbenchmark)

set.seed(2018)
nr <- 10^3
lst <- replicate(2, data.frame(
  key1 = sample(letters[1:5], nr, replace = T),
  key2 = sample(LETTERS[6:10], nr, replace = T),
  value = runif(nr)), simplify = F)

without_merge <- function() {
  expanded <- cbind(lst[[1]][rep(1:nrow(lst[[1]]), times = nrow(lst[[2]])), ],
                    lst[[2]][rep(1:nrow(lst[[2]]), each = nrow(lst[[1]])), ])
  colnames(expanded) <- c("key1.x", "key2.x", "value.x", "key1.y", "key2.y","value.y")
  joined <- expanded[expanded$key1.x == expanded$key1.y, -1]
  colnames(joined) <- c("key2.x", "value.x", "key1", "key2.y","value.y")
  joined
}

res <- microbenchmark(
  dplyr_join = inner_join(lst[[1]], lst[[2]], by = c("key1")),
  base_R_with_merge = merge(lst[[1]], lst[[2]], by = c("key1")),
  base_R_without_merge = without_merge(),
  times = 20
)
res
# Unit: milliseconds
#                 expr       min        lq       mean     median        uq       max neval
#           dplyr_join    5.7251    6.1108   22.35497    6.32195    7.3038  239.7486    20
#    base_R_with_merge  716.2633  743.8472  813.63495  812.86165  862.3884 1024.9836    20
# base_R_without_merge 1848.9711 2009.4264 2100.98077 2097.19790 2174.8663 2365.7716    20

autoplot(res)

现在回到我的数据。 现在是时候加入多个列了，为此我将使用weather 数据集，它在 5 个键上连接flights：origin、year、@987654332 @、day 和 hour。

任务是：在温度高于 90 华氏度的飞行中获取所有飞机的唯一尾翼。

现在dplyr 真正展现了它的实力，无论是优雅还是速度。

首先，有一种简单且更智能的方法，即在加入之前过滤和选择必要的列：

library(nycflights13)

join_naive <- function() {
  flights %>%
    inner_join(weather, by = c("origin", "year", "month", "day", "hour")) %>%
    filter(temp > 90) %>%
    select(tailnum) %>%
    drop_na() %>%
    pull(tailnum) %>%
    unique()
}

join_smarter <- function() {
  flights %>%
    select(origin, year, month, day, hour, tailnum) %>%
    inner_join(weather %>%
                 select(origin, year, month, day, hour, temp) %>%
                 filter(temp > 90),
               by = c("origin", "year", "month", "day", "hour")) %>%
    select(tailnum) %>%
    drop_na() %>%
    pull(tailnum) %>%
    unique()
}

all.equal(join_naive(), join_smarter())

# TRUE

接下来是 R 基础 merge 方式，假设我们事先过滤了数据以赋予它一些优势，只保留必要的最小值：

weather_over_90 <- weather[weather$temp > 90, c("origin", "year", "month", "day", "hour")]
weather_over_90 <- weather_over_90[complete.cases(weather_over_90), ]
flights_minimum <- flights[
  flights$origin %in% weather_over_90$origin &
    flights$year %in% weather_over_90$year &
    flights$month %in% weather_over_90$month &
    flights$day %in% weather_over_90$day &
    flights$hour %in% weather_over_90$hour,
  c("origin", "year", "month", "day", "hour", "tailnum")
  ]
flights_minimum <- flights_minimum[complete.cases(flights_minimum), ]

with_merge <- function() {
  unique(merge(weather_over_90, flights_minimum, by = c("origin", "year", "month", "day", "hour"))$tailnum)
}

all.equal(sort(with_merge()), sort(join_smarter()))

# TRUE

最后是我能想到的最手动的方式，没有实际的 for 循环，类似于我用上面的单个键实现连接的手动方式：

without_merge <- function(df1, df2) {
  colnames(df1) <- c("origin.x", "year.x", "month.x", "day.x", "hour.x")
  colnames(df2) <- c("origin.y", "year.y", "month.y", "day.y", "hour.y", "tailnum")
  expanded <- cbind(df1[rep(1:nrow(df1), times = nrow(df2)), ],
                    df2[rep(1:nrow(df2), each = nrow(df1)), ])
  joined <- expanded[
    expanded$origin.x == expanded$origin.y &
      expanded$year.x == expanded$year.y &
      expanded$month.x == expanded$month.y &
      expanded$day.x == expanded$day.y &
      expanded$hour.x == expanded$hour.y,
    -c(1:5)
    ]
  unique(joined$tailnum)
}

all.equal(sort(without_merge(weather_over_90, flights_minimum)),
          sort(dplyr_join_smarter()))

# TRUE

比较：

res <- microbenchmark(
  dplyr_join_smart = join_smarter(),
  dplyr_join_naive = join_naive(),
  base_R_with_merge = with_merge(),
  base_R_without_merge = without_merge(weather_over_90, flights_minimum),
  times = 20
)
res
# Unit: milliseconds
#                 expr       min         lq       mean    median         uq       max neval
#     dplyr_join_smart   19.6140   20.08890   21.71460   20.6103   21.79740   30.3105    20
#     dplyr_join_naive   65.6180   69.08685   71.14589   71.1451   72.47165   81.9732    20
#    base_R_with_merge  189.1192  193.81325  201.46174  197.7632  207.40575  231.0595    20
# base_R_without_merge 1763.1307 1814.44825 1871.61599 1840.4509 1884.59005 2162.0949    20
autoplot(res)

可以看出，即使是 dplyr 的连接的幼稚方法也比基本 merge 快，但它可以提高 3 倍。请记住dplyrs 运行时包含过滤！