【问题标题】:Execute many simulations avoiding for-loop执行许多模拟以避免for循环
【发布时间】:2020-08-13 12:17:14
【问题描述】:

我正在尝试为我的数据的不同子组的更改模型模拟拟合值,这些模型再次基于我的原始数据框的另一个子集的随机抽样(我为这个问题编写的最小示例忽略了随机抽样等.,导致所有模拟的拟合值相同,但这并不重要)。我编写了一个 dplyr 代码来存储每个组的模型,生成新的 x 值来预测拟合值,预测它们等等。它会产生一列拟合值,完全符合我的要求。但是,我想将整个过程进行 1000 倍。我当然可以使用 for 循环来执行此操作(如下面的示例中所做的那样),但是是否有可能在 dplyr-pipe 行中执行此操作?也许会加快整个过程(我的原始数据集相当大,for-loop 需要很长时间)?

# making up data
dat <- data.frame("species" =  seq(1:20), "col_A" = runif(20, min=1000, max=2500), "col_B" = runif(20, min = 0, max = 1500), 
                  "maximum" = rep(2500, 20), "minimum" = rep(1000, 20), groups = rep(LETTERS[1:5], each = 4))

# functions to use with purrr
linear_mod <- function(dat) {
  lm(col_A ~ col_B, data = dat)
}

# define parametres and an empty data frame to use in the for loop
runs <- 10
fitted_sim <- data.frame(matrix(data=NA,nrow=20,ncol=runs+1,byrow=FALSE)) #empty dataframe to contain fitted values for each alt
names(fitted_sim) <- as.factor(seq(1:runs+1))

# for-loop around my dplyr-code
for (j in 1:runs){
  simul <- dat %>%
    group_by(groups) %>%
    nest(data = c(col_A, col_B, species)) %>%
    mutate(model = map(data, linear_mod), # add model for every group
           sim_data = list(seq(minimum, maximum, by = 10))) %>% # define new x-values for later predictions
    unnest(sim_data) %>%
    nest(sim_data = sim_data) %>%
    mutate(fitted = map2(model, sim_data, ~predict(.x, col_B = sim_data, type = "response")), # predict values
    unnest(fitted) # unnest predicted values to save in data frame
  
  # save newly fitted values in fitted_sim data frame
  fitted_sim[,1] <- simul$groups
  fitted_sim[,1+j] <- simul$fitted
}

感谢您的每一个提示!

编辑: 这是一个扩展示例代码,包括上述随机抽样,但在我的第一个示例中省略:

# for-loop around my dplyr-code
for (j in 1:runs) {
  simul <- dat %>%
    group_by(groups) %>%
    rowwise() %>%
    mutate(vector_column = case_when(abs(col_A) == (maximum-minimum) ~ list(col_B),
                                     sign(col_A) == 1 ~ list(dat$col_B[dat$col_B <= maximum - col_A]), # using list function to store vectors in a data.frame
                                     sign(col_A) != 1 ~ list(dat$col_B[dat$col_B >= minimum + col_A])),
           helper = !is_empty(vector_column), # in case some of the vectors are empty so it is not possible to use sample
           col_B_new = ifelse(helper, sample(vector_column, 1), NA),
           helper = NULL,
           sim_data = list(seq(minimum, maximum, by = 10))) %>% # define new x-values for later predictions
    ungroup() %>% # get rid of rowwise()
    group_by(groups) %>%
    unnest(sim_data) %>%
    nest(sim_data = sim_data) %>%
    nest(data = c(col_A, col_B, col_B_new, species)) %>%
    mutate(model = map(data, linear_mod),
           fitted = map2(model, sim_data, ~predict(.x, col_B_new = sim_data, type = "response"))) %>% # predict values
    unnest(fitted) # unnest predicted values to save in data frame
           
           # save newly fitted values in fitted_sim data frame
           fitted_sim[,1] <- simul$groups
           fitted_sim[,1+j] <- simul$fitted
}

【问题讨论】:

    标签: r for-loop dplyr


    【解决方案1】:

    不确定您将来需要从 simul data.frame 中获得什么,但各个循环元素似乎彼此独立,因此您可以使用 parallel:mclappy 并行运行它们。在我的示例中,我只使用了lapply,但这已经相当快了——如果它对你有任何帮助的话......

    library(tidyverse)
    library(data.table)
    library(microbenchmark)
    
    # making up data
    dat <- data.frame("species" =  seq(1:20), "col_A" = runif(20, min=1000, max=2500), "col_B" = runif(20, min = 0, max = 1500), 
                      "maximum" = rep(2500, 20), "minimum" = rep(1000, 20), groups = rep(LETTERS[1:5], each = 4))
    
    # functions to use with purrr
    linear_mod <- function(dat) {
        lm(col_A ~ col_B, data = dat)
    }
    
    # define parametres and an empty data frame to use in the for loop
    runs <- 10
    fitted_sim <- data.frame(matrix(data=NA,nrow=20,ncol=runs+1,byrow=FALSE)) #empty dataframe to contain fitted values for each alt
    names(fitted_sim) <- as.factor(seq(1:runs+1))
    
    # I just wrapped your code in a function for benchmarking
    run.simul <- function(){
        for (j in 1:runs){
            simul <- dat %>%
                group_by(groups) %>%
                nest(data = c(col_A, col_B, species)) %>%
                mutate(model = map(data, linear_mod), # add model for every group
                       sim_data = list(seq(minimum, maximum, by = 10))) %>% # define new x-values for later predictions
                unnest(sim_data) %>%
                nest(sim_data = sim_data) %>%
                mutate(fitted = map2(model, sim_data, ~predict(.x, col_B = sim_data, type = "response"))) %>% # predict values
                unnest(fitted) # unnest predicted values to save in data frame
            # save newly fitted values in fitted_sim data frame
            fitted_sim[,1] <- simul$groups
            fitted_sim[,1+j] <- simul$fitted
        }
        fitted_sim
    }
    
    
    # my version (sorry for using `data.table`...)
    Dat <- data.table(dat, key="groups")
    getFits <- function(x){
        x[, fitted := predict(lm(col_A ~ col_B), .(list(seq(minimum[1], maximum[1], by = 10)))), by = groups]
        x[, .(species, groups, fitted)]
    }
    
    # get the results into a data.frame; use mclapply instead of lapply if you like
    dcast(rbindlist(lapply(seq_len(runs), function(z) getFits(data.table(dat, key="groups"))), idcol = "run"), ... ~ run, value.var = "fitted")
    #>     species groups        1        2        3        4        5        6
    #>  1:       1      A 1767.621 1767.621 1767.621 1767.621 1767.621 1767.621
    #>  2:       2      A 1376.679 1376.679 1376.679 1376.679 1376.679 1376.679
    #>  3:       3      A 1523.100 1523.100 1523.100 1523.100 1523.100 1523.100
    #>  4:       4      A 1794.593 1794.593 1794.593 1794.593 1794.593 1794.593
    #>  5:       5      B 1272.408 1272.408 1272.408 1272.408 1272.408 1272.408
    #>  6:       6      B 1967.709 1967.709 1967.709 1967.709 1967.709 1967.709
    #>  7:       7      B 1792.934 1792.934 1792.934 1792.934 1792.934 1792.934
    #>  8:       8      B 2318.699 2318.699 2318.699 2318.699 2318.699 2318.699
    #>  9:       9      C 2017.187 2017.187 2017.187 2017.187 2017.187 2017.187
    #> 10:      10      C 1899.827 1899.827 1899.827 1899.827 1899.827 1899.827
    #> 11:      11      C 1734.953 1734.953 1734.953 1734.953 1734.953 1734.953
    #> 12:      12      C 1834.046 1834.046 1834.046 1834.046 1834.046 1834.046
    #> 13:      13      D 1797.615 1797.615 1797.615 1797.615 1797.615 1797.615
    #> 14:      14      D 1915.832 1915.832 1915.832 1915.832 1915.832 1915.832
    #> 15:      15      D 1841.489 1841.489 1841.489 1841.489 1841.489 1841.489
    #> 16:      16      D 1798.442 1798.442 1798.442 1798.442 1798.442 1798.442
    #> 17:      17      E 1641.359 1641.359 1641.359 1641.359 1641.359 1641.359
    #> 18:      18      E 1631.253 1631.253 1631.253 1631.253 1631.253 1631.253
    #> 19:      19      E 1634.197 1634.197 1634.197 1634.197 1634.197 1634.197
    #> 20:      20      E 1634.991 1634.991 1634.991 1634.991 1634.991 1634.991
    #>            7        8        9       10
    #>  1: 1767.621 1767.621 1767.621 1767.621
    #>  2: 1376.679 1376.679 1376.679 1376.679
    #>  3: 1523.100 1523.100 1523.100 1523.100
    #>  4: 1794.593 1794.593 1794.593 1794.593
    #>  5: 1272.408 1272.408 1272.408 1272.408
    #>  6: 1967.709 1967.709 1967.709 1967.709
    #>  7: 1792.934 1792.934 1792.934 1792.934
    #>  8: 2318.699 2318.699 2318.699 2318.699
    #>  9: 2017.187 2017.187 2017.187 2017.187
    #> 10: 1899.827 1899.827 1899.827 1899.827
    #> 11: 1734.953 1734.953 1734.953 1734.953
    #> 12: 1834.046 1834.046 1834.046 1834.046
    #> 13: 1797.615 1797.615 1797.615 1797.615
    #> 14: 1915.832 1915.832 1915.832 1915.832
    #> 15: 1841.489 1841.489 1841.489 1841.489
    #> 16: 1798.442 1798.442 1798.442 1798.442
    #> 17: 1641.359 1641.359 1641.359 1641.359
    #> 18: 1631.253 1631.253 1631.253 1631.253
    #> 19: 1634.197 1634.197 1634.197 1634.197
    #> 20: 1634.991 1634.991 1634.991 1634.991
    
    run.simul()
    #>    1        2        3        4        5        6        7        8        9
    #> 1  A 1767.621 1767.621 1767.621 1767.621 1767.621 1767.621 1767.621 1767.621
    #> 2  A 1376.679 1376.679 1376.679 1376.679 1376.679 1376.679 1376.679 1376.679
    #> 3  A 1523.100 1523.100 1523.100 1523.100 1523.100 1523.100 1523.100 1523.100
    #> 4  A 1794.593 1794.593 1794.593 1794.593 1794.593 1794.593 1794.593 1794.593
    #> 5  B 1272.408 1272.408 1272.408 1272.408 1272.408 1272.408 1272.408 1272.408
    #> 6  B 1967.709 1967.709 1967.709 1967.709 1967.709 1967.709 1967.709 1967.709
    #> 7  B 1792.934 1792.934 1792.934 1792.934 1792.934 1792.934 1792.934 1792.934
    #> 8  B 2318.699 2318.699 2318.699 2318.699 2318.699 2318.699 2318.699 2318.699
    #> 9  C 2017.187 2017.187 2017.187 2017.187 2017.187 2017.187 2017.187 2017.187
    #> 10 C 1899.827 1899.827 1899.827 1899.827 1899.827 1899.827 1899.827 1899.827
    #> 11 C 1734.953 1734.953 1734.953 1734.953 1734.953 1734.953 1734.953 1734.953
    #> 12 C 1834.046 1834.046 1834.046 1834.046 1834.046 1834.046 1834.046 1834.046
    #> 13 D 1797.615 1797.615 1797.615 1797.615 1797.615 1797.615 1797.615 1797.615
    #> 14 D 1915.832 1915.832 1915.832 1915.832 1915.832 1915.832 1915.832 1915.832
    #> 15 D 1841.489 1841.489 1841.489 1841.489 1841.489 1841.489 1841.489 1841.489
    #> 16 D 1798.442 1798.442 1798.442 1798.442 1798.442 1798.442 1798.442 1798.442
    #> 17 E 1641.359 1641.359 1641.359 1641.359 1641.359 1641.359 1641.359 1641.359
    #> 18 E 1631.253 1631.253 1631.253 1631.253 1631.253 1631.253 1631.253 1631.253
    #> 19 E 1634.197 1634.197 1634.197 1634.197 1634.197 1634.197 1634.197 1634.197
    #> 20 E 1634.991 1634.991 1634.991 1634.991 1634.991 1634.991 1634.991 1634.991
    #>          10       NA
    #> 1  1767.621 1767.621
    #> 2  1376.679 1376.679
    #> 3  1523.100 1523.100
    #> 4  1794.593 1794.593
    #> 5  1272.408 1272.408
    #> 6  1967.709 1967.709
    #> 7  1792.934 1792.934
    #> 8  2318.699 2318.699
    #> 9  2017.187 2017.187
    #> 10 1899.827 1899.827
    #> 11 1734.953 1734.953
    #> 12 1834.046 1834.046
    #> 13 1797.615 1797.615
    #> 14 1915.832 1915.832
    #> 15 1841.489 1841.489
    #> 16 1798.442 1798.442
    #> 17 1641.359 1641.359
    #> 18 1631.253 1631.253
    #> 19 1634.197 1634.197
    #> 20 1634.991 1634.991
    
    # benchmarking
    microbenchmark(
        a=dcast(rbindlist(lapply(seq_len(runs), function(z) getFits(data.table(dat, key="groups"))), idcol = "run"), ... ~ run, value.var = "fitted"),
        b=run.simul(), times= 10L
        )
    #> Unit: milliseconds
    #>  expr       min        lq      mean    median       uq      max neval cld
    #>     a  46.46851  48.65027  51.09618  49.88775  54.7177  56.2888    10  a 
    #>     b 389.76893 410.57588 418.16993 415.74493 424.2042 448.4604    10   b
    

    reprex package (v0.3.0) 于 2020-08-13 创建

    【讨论】:

    • 这太好了,谢谢!我对数据表不是很熟悉,但我绝对应该看看它。但是,对于我的真实数据,对于每一轮 for 循环,我从数据框中的列表列中为每个物种随机采样一个新的 col_B 值,这可以集成到您的代码中吗?数据表是否适用于列表列?
    • 我添加了示例脚本的扩展版本来说明我的意思。不过,请随意忽略它,您很好地回答了我的第一个问题。
    • 除非您更改了 linear_mod 函数,否则您的编辑不会更改模型。你想做lm(col_A ~ col_B_new)吗?此外,您的抽样似乎是针对abs(col_A) == (maximum-minimum) 的分组,但除此之外,所有组都是这样的吗?是的,data.table 处理列表列,如在我的getFits 函数中所见,请参见带有.(list(seq(minimum[1], maximum[1], by = 10)) 的行,其中.() 将列表包装到另一个列表中以保留列中的内部列表。跨度>
    • 不,我没有更改模型 - 我只是不确定如何集成附加条件,但我现在想通了。非常感谢您的帮助!
    猜你喜欢
    • 1970-01-01
    • 1970-01-01
    • 2019-12-26
    • 1970-01-01
    • 1970-01-01
    • 2023-03-27
    • 2012-06-25
    • 2020-03-10
    • 1970-01-01
    相关资源
    最近更新 更多