提高发现事件首次发生的效率

Question

我正在处理一个大型时间序列 data.table，60 *B*illion 行 X 50 列

对于三个特定的列，我想添加一个相应的 T/F 列，通过idCol 指示每个事件第一次发生的时间

换句话说，对于 ColumnA，新列将是

DT[, flag.ColumnA :=  dateCol==min(dateCol)
   , by=list(idCol, ColumnA)]

但是：min(dateCol) 经常有关联，而关联的解决方法是只标记一个元素 TRUE，其余的 FALSE。这导致了以下方法

## Set key to {idCol, dateCol} so that the first row in each group
##   is the unique element in that group that should be set to TRUE
setkey(DT, idCol, dateCol)
DT[, flag.ColumnA := FALSE]
DT[, { DT[ .I[[1L]], flag.ColumnA := TRUE] }  # braces here are just for easier reading
   , by=list(idCol, ColumnA)]

问题在于，第二种方法将运行时间增加了 3 倍以上，而第一种方法每列已经花费了一个多小时（在相对较快的盒子上）

我也考虑过手动解决方法 1 中的关系，但这比上述两种方法慢。

关于如何更有效地完成这项任务的任何建议？ 下面的示例数据

---

预期输出样本

DT["ID_01"] [ColumnA %in% c("BT", "CK", "MH")] [order(ColumnA, dateCol)]

    idCol    dateCol ColumnA ColumnB flag.ColumnA.M1 flag.ColumnA.M2
 1: ID_01 2013-06-01      BT     xxx            TRUE            TRUE <~~ M1 is WRONG, M2 is correct
 2: ID_01 2013-06-01      BT     www            TRUE           FALSE <~~ M1 is WRONG, M2 is correct
 3: ID_01 2013-06-01      BT     yyy            TRUE           FALSE <~~ M1 is WRONG, M2 is correct
 4: ID_01 2013-06-22      BT     xxx           FALSE           FALSE
 5: ID_01 2013-11-23      BT     yyy           FALSE           FALSE
 6: ID_01 2013-11-30      BT     zzz           FALSE           FALSE
 7: ID_01 2013-06-15      CK     www            TRUE            TRUE
 8: ID_01 2013-06-15      CK     uuu            TRUE           FALSE
 9: ID_01 2013-06-15      CK     www            TRUE           FALSE
10: ID_01 2013-06-29      CK     zzz           FALSE           FALSE
11: ID_01 2013-10-12      CK     vvv           FALSE           FALSE
12: ID_01 2013-11-02      CK     uuu           FALSE           FALSE
13: ID_01 2013-06-22      MH     uuu            TRUE            TRUE
14: ID_01 2013-06-22      MH     xxx            TRUE           FALSE
15: ID_01 2013-06-22      MH     zzz            TRUE           FALSE
16: ID_01 2013-08-24      MH     ttt           FALSE           FALSE
17: ID_01 2013-09-07      MH     xxx           FALSE           FALSE
18: ID_01 2013-09-14      MH     zzz           FALSE           FALSE
19: ID_01 2013-09-21      MH     vvv           FALSE           FALSE
20: ID_01 2013-11-30      MH     ttt           FALSE           FALSE

---

样本数据

# increase N for realistic test    
N <- 2e4  # N should be large, as certain methods will be seemingly fast but wont scale

ids   <- sprintf("ID_%02d", seq(5))
A     <- apply(expand.grid(LETTERS, LETTERS), 1, paste0, collapse="")
B     <- paste0(letters, letters, letters)[20:26]
dates <- seq.Date(as.Date("2013-06-01"), as.Date("2013-12-01"), by=7)

set.seed(1)
DT <- data.table( dateCol=sample(dates, N, TRUE)
                , idCol  =sample(ids,   N, TRUE)
                , ColumnA=sample(A,     N, TRUE)
                , ColumnB=sample(B,     N, TRUE)
                , key="idCol")


{
  cat("\n==========\nMETHOD ONE:\n")
  print(system.time({
       DT[, flag.ColumnA.M1 :=  dateCol==min(dateCol)
          , by=list(idCol, ColumnA)]}))
  cat("\n\n==========\nMETHOD TWO:\n")
  print(system.time({
      setkey(DT, idCol, dateCol)
      DT[, flag.ColumnA.M2 := FALSE]
      DT[, { DT[ .I[[1L]], flag.ColumnA.M2 := TRUE] }  # braces here are just for easier reading
         , by=list(idCol, ColumnA)]}))
}

## For Example, looking at ID_01, at a few select values of ColumnA: 
DT["ID_01"] [ColumnA %in% c("BT", "CK", "MH")] [order(ColumnA, dateCol)]

Answer 1

只需使用which.min 解决关系：

DT[, flag := FALSE]
DT[DT[, .I[which.min(dateCol)], by = list(idCol, ColumnA)]$V1, flag := TRUE]

对于您的小数据样本，这对我来说是瞬时的，正如 system.time 无法测量的那样，它在 N=1e7 上比您的方法 1 快 1.5 倍。我没有测试更大的 N。

Answer 2

我会在[.data.table 中使用set 而不是:=。当set 可以创建列时，这将再次更快。

类似（通过 id 和日期键入以确保排序正确）

system.time({
set.seed(1)
DT <- data.table( dateCol=sample(dates, N, TRUE)
                  , idCol  =sample(ids,   N, TRUE)
                  , ColumnA=sample(A,     N, TRUE)
                  , ColumnB=sample(B,     N, TRUE)
                  , key=c("idCol", "dateCol"))
ll <- lapply(c('ColumnA','ColumnB'), function(cc) DT[,.I[1],by = c('idCol',cc)][['V1']])

flags <- c('flagA','flagB')
DT[, (flags) := FALSE]

jflag <- match(flags, names(DT), nomatch=0)

for(jj in seq_along(jflag)){
  set(DT, i = ll[[jj]], j = jflag[jj], value = TRUE)

}
})


# See this is lightening fast (even incorporating the creation of the data.table)
##   user  system elapsed 
##   0.02    0.00    0.02 




DT["ID_01"] [ColumnA %in% c("BT", "CK", "MH")] [order(ColumnA, dateCol)]
    idCol    dateCol ColumnA ColumnB flagA flagB
 1: ID_01 2013-06-01      BT     xxx  TRUE  TRUE
 2: ID_01 2013-06-01      BT     www FALSE FALSE
 3: ID_01 2013-06-01      BT     yyy FALSE FALSE
 4: ID_01 2013-06-22      BT     xxx FALSE FALSE
 5: ID_01 2013-11-23      BT     yyy FALSE FALSE
 6: ID_01 2013-11-30      BT     zzz FALSE FALSE
 7: ID_01 2013-06-15      CK     www  TRUE FALSE
 8: ID_01 2013-06-15      CK     uuu FALSE FALSE
 9: ID_01 2013-06-15      CK     www FALSE FALSE
10: ID_01 2013-06-29      CK     zzz FALSE FALSE
11: ID_01 2013-10-12      CK     vvv FALSE FALSE
12: ID_01 2013-11-02      CK     uuu FALSE FALSE
13: ID_01 2013-06-22      MH     uuu  TRUE FALSE
14: ID_01 2013-06-22      MH     xxx FALSE FALSE
15: ID_01 2013-06-22      MH     zzz FALSE FALSE
16: ID_01 2013-08-24      MH     ttt FALSE FALSE
17: ID_01 2013-09-07      MH     xxx FALSE FALSE
18: ID_01 2013-09-14      MH     zzz FALSE FALSE
19: ID_01 2013-09-21      MH     vvv FALSE FALSE
20: ID_01 2013-11-30      MH     ttt FALSE FALSE

其他可能性包括一次性完成所有操作

例如（如果您每次都知道添加列的列号，这会更快（您应该在您的情况下，没有必要 match 标记列到 data.table 的名称） hten)

lapply(c('A','B'), function(LL){
    cn <- sprintf('Column%s',LL)
    fl <- sprintf('flag%s',LL)
    DT[, (fl) :=FALSE]
    is <- DT[,.I[1],by =c('idCol',cn)][['V1']]
    jm <- match(fl, names(DT), nomatch=0)
    set(DT, i=is, j=jm, value=TRUE)
    invisible()
  })

或

for(ff in seq_along(flags)){

  is <- DT2[,.I[1], by =c('idCol',cols[1])][['V1']]
  set(DT2, i = is, j = jflag[ff], value = TRUE)
}