更快地聚合时间序列数据答案

【问题标题】：aggregate time series data faster更快地聚合时间序列数据
【发布时间】：2020-07-14 22:47:00
【问题描述】：

我需要通过随时间分布它们然后聚合来计算一些特征，如下所示。该代码产生了正确的结果，但我的实际数据集中大约有 100 万行数据，运行时间类似于下面的代码需要我的机器几天。我正在寻找更有效的代码。我不确定xts 或tidyverse 包在这里对加速有用。我与data.table 合作过，我认为这有助于提高速度——也许这是错误的选择。有什么想法吗？

library(data.table)
library(lubridate)

#toy example
rows=1000
set.seed(1)
data=data.table(
  customer.arv = as.POSIXct("2020-01-01 00:00")+dminutes(sample(1:(60*24*7),rows,replace = T)),
  location = sample(1:4,rows,replace = T),
  customer.type = sample(LETTERS[1:5],rows,replace = T),
  charge = sample(seq(50,200,10),rows,replace = T)
  )
data[,':='(customer.dep = customer.arv+dminutes(sample(1:500,rows,replace = T)),
           arv.time.floor = floor_date(customer.arv,"hours"),
           arv.hour = hour(customer.arv))]

#distribute the charge over the length of stay (departure-arrival) and calculate the hourly charge
tot.hourly.charge = function(pass.location,pass.arv.time.floor,pass.customer.type) {
  full.hr.cust = data[customer.arv<=pass.arv.time.floor&customer.dep>=pass.arv.time.floor+dhours(1)&location==pass.location&customer.type==pass.customer.type,sum(charge)]
  partial.hr.cust = data[customer.arv<=pass.arv.time.floor&customer.dep<pass.arv.time.floor+dhours(1)&customer.dep>pass.arv.time.floor&location==pass.location&customer.type==pass.customer.type,sum(charge*minute(customer.dep)/60)]
  return(full.hr.cust+partial.hr.cust)
}

#aggregate
res = data[,.(hourly.charge = tot.hourly.charge(location,arv.time.floor,customer.type)), by=.(location,arv.time.floor,customer.type)]

#sample output
res[order(location,customer.type,arv.time.floor)][1:10,]
    location      arv.time.floor customer.type hourly.charge
 1:        1 2020-01-01 00:00:00             A       0.00000
 2:        1 2020-01-01 03:00:00             A     190.00000
 3:        1 2020-01-01 06:00:00             A     216.66667
 4:        1 2020-01-01 09:00:00             A     100.00000
 5:        1 2020-01-01 12:00:00             A     100.00000
 6:        1 2020-01-01 14:00:00             A      16.66667
 7:        1 2020-01-01 15:00:00             A      50.00000
 8:        1 2020-01-01 18:00:00             A      62.50000
 9:        1 2020-01-01 20:00:00             A       0.00000
10:        1 2020-01-01 22:00:00             A     190.00000

【问题讨论】：

你能用一个简单的例子更详细地解释一下聚合应该做什么吗？
例如，对于 arv.time.floor "2020-01-01 03:00",location==1,type==A：将所有在 2020 年之前到达的客户的费用相加- 01-01 2020-01-01 03:00 及 2020-01-01 04:00 之后。对于在 2020 年 1 月 1 日 03:00 之前到达并在 03:00 到 04:00 之间离开的客户，将他们的费用乘以他们在 03:00 到 04:00 之间在业务中花费的小时数的分数。因此，如果某个客户在 03:20（20 分钟）之后花费了 300 美元，那么他们对每小时费用的贡献是 300 美元*20/60=100 美元
你推荐过data[,.(hourly.charge吗？我试过replicate(n = 100, data[,.(hourly.charge,，应该指出问题，[，tot.hourly.charge，但你更清楚你的问题。
您能否分享data[, .N, .(location, arv.time.floor, customer.type)][, unique(N)] 以获取您的实际数据集？还有data[, range(arv.time.floor)]
第一个：[1] 1 5 4 2 3 8 6 7 10 9 12 13 11 15 14 16 18 17。日期范围：[1] "2016-01-22 22:00:00 CST" "2020-06-30 23:00:00 CDT"

标签： r data.table

【解决方案1】：

您可以先尝试以下方法：

data[, arv.time.floor.1h := arv.time.floor + 60*60]

full <- data[data, on=.(location=location, customer.type=customer.type, 
    customer.arv<=arv.time.floor, customer.dep>=arv.time.floor.1h),
    .(charge=x.charge, location, arv.time.floor=i.arv.time.floor, customer.type=i.customer.type)][,
        .(full.hr.cust=sum(charge)), keyby=.(location, customer.type, arv.time.floor)][
            is.na(full.hr.cust), full.hr.cust := 0]

partial <- data[data, on=.(location=location, customer.type=customer.type, 
    customer.arv<=arv.time.floor, customer.dep>arv.time.floor, customer.dep<arv.time.floor.1h),
    .(charge=x.charge, m=minute(x.customer.dep), location, arv.time.floor=i.arv.time.floor, customer.type=i.customer.type)][,
        .(partial.hr.cust=sum(charge * m / 60)), keyby=.(location, customer.type, arv.time.floor)][
            is.na(partial.hr.cust), partial.hr.cust := 0]

ans <- full[partial][, charge := full.hr.cust + partial.hr.cust]

数据：

library(data.table)

#toy example
rows=1000
set.seed(1)
data=data.table(
    customer.arv = as.POSIXct("2020-01-01 00:00") + 60 * (sample(1:(60*24*7), rows, replace = TRUE)),
    location = sample(1:4,rows,replace = TRUE),
    customer.type = sample(LETTERS[1:5],rows,replace = TRUE),
    charge = sample(seq(50,200,10),rows,replace = TRUE)
)
data[, `:=`(customer.dep = customer.arv + 60 * sample(1:500,rows,replace = TRUE),
    arv.time.floor = as.POSIXct(round.POSIXt(customer.arv, units="hours")))]
setorder(data, location, customer.type, arv.time.floor)

【讨论】：