【问题标题】:Loop or batch process through a list of CSV files in R通过 R 中的 CSV 文件列表循环或批处理
【发布时间】:2017-02-05 01:33:26
【问题描述】:

我想循环浏览一个 CSV 文件列表:

Macro <- read.csv("P:/R/R_Input/JWN_Input.csv")
Macro <- read.csv("P:/R/R_Input/BBY_Input.csv")
...

这也输出到相应的 CSV 文件:

write.csv(a, "P:/Model_Output/JWN.csv", row.names = F, na="")
write.csv(a, "P:/Model_Output/BBY.csv", row.names = F, na="")
...

以上两项是唯一唯一的输入/输出。代码主体如下。我正在尝试使用下面的代码主体批量处理输入/输出 CSV 文件。

Macro <- read.csv("P:/R/R_Input/JWN_Input.csv")

# train set up
ctrl <- caret::trainControl(method = "timeslice", initialWindow = 8, horizon = 1, 
                            fixedWindow = FALSE, savePredictions = TRUE)

# Loads all variable names from Macro and Macro2
vars_macro = names(Macro)[!names(Macro) %in% c("qtrs", "y", "s1", "s2", "s3")] 
vars_macro2 = names(Macro2)[!names(Macro2) %in% c("y", "s1", "s2", "s3")]
vars_macro3 = names(Macro3)[!names(Macro3) %in% c("y", "s1", "s2", "s3")]
vars = c(vars_macro, vars_macro2, vars_macro3)

# run lm
lst = foreach(var = vars) %dopar% {

  if (var %in% vars_macro)  
    foo <- function(start, mod_formula) {
      myfit <- caret::train(mod_formula, data = Macro[start:14, ,drop = FALSE],
                            method = "lm", trControl = ctrl)
      c(myfit$pred)  ## return; drop dimension as a vector
    }
  if (var %in% vars_macro2)
    foo <- function(start, mod_formula) {
      myfit <- caret::train(mod_formula, data = Macro2[start:14, ,drop = FALSE],
                            method = "lm", trControl = ctrl)
      c(myfit$pred)  ## return; drop dimension as a vector
    }
  if (var %in% vars_macro3)
    foo <- function(start, mod_formula) {
      myfit <- caret::train(mod_formula, data = Macro3[start:14, ,drop = FALSE],
                            method = "lm", trControl = ctrl)
      c(myfit$pred)  ## return; drop dimension as a vector
    } 
  f = formula(paste0("y ~ ", var,  "+ s1 + s2 + s3"))
  Forecast <- sapply(1:6, foo, mod_formula = f)  

  F9 <- c(Forecast[[1,1]][1])
  F10 <- c(Forecast[[1,1]][2], Forecast[[1,2]][1])
  F11 <- c(Forecast[[1,1]][3], Forecast[[1,2]][2], Forecast[[1,3]][1])
  F12 <- c(Forecast[[1,1]][4], Forecast[[1,2]][3], Forecast[[1,3]][2],
           Forecast[[1,4]][1])
  F13 <- c(Forecast[[1,1]][5], Forecast[[1,2]][4], Forecast[[1,3]][3],
           Forecast[[1,4]][2], Forecast[[1,5]][1])
  F14 <- c(Forecast[[1,1]][6], Forecast[[1,2]][5], Forecast[[1,3]][4],
           Forecast[[1,4]][3], Forecast[[1,5]][2], Forecast[[1,6]][1])

  A <-c((mean(F9)/Macro[9:9,2:2]-1), (mean(F10)/Macro[10:10,2:2]-1), 
        (mean(F11)/Macro[11:11,2:2]-1), (mean(F12)/Macro[12:12,2:2]-1), 
        (mean(F13)/Macro[13:13,2:2]-1),(mean(F14)/Macro[14:14,2:2]-1))
  Temp <- mean(abs(A[0:5]))
  P <-c((mean(F9)/Macro[9:9,2:2]-1), (mean(F10)/Macro[10:10,2:2]-1), 
        (mean(F11)/Macro[11:11,2:2]-1), (mean(F12)/Macro[12:12,2:2]-1), 
        (mean(F13)/Macro[13:13,2:2]-1),(mean(F14)/Macro[14:14,2:2]-1),
        Temp,(mean(F14)/(1+mean(A[3:5])))/Macro[14:14,2:2]-1)
  #E <- scales::percent(P)
  C <- c(mean(F9),mean(F10),mean(F11), mean(F12), mean(F13), mean(F14),
         "abs error",mean(F14)/(1+mean(P[3:5])))
  data.frame(C, P)
}

# Summary            
model_error = as.character(sapply(lst, function(elt) elt$P[7]))
forecasts = as.numeric(as.character(sapply(lst, function(elt) elt$C[8])))
delta = as.character(sapply(lst, function(elt) elt$P[8]))
df = data.frame(Card = vars, Model_Avg_Error = model_error, 
                Forecast = forecasts,  Delta = delta)
df$blankVar = NA

df_macro1 = df[df$Card %in% vars_macro,]
df_macro1$blankVar = NA
df_macro2 = df[df$Card %in% vars_macro2,]
df_macro2 = df_macro2[order(df_macro2$Model_Avg_Error),]
df_macro2$blankVar = NA
df_macro3 = df[df$Card %in% vars_macro3,]
df_macro3 = df_macro3[order(df_macro3$Model_Avg_Error),]
df_macro3$blankVar = NA
df_macro4 = df[df$Card %in% names(Macro4),]
df_macro4 = df_macro4[order(df_macro4$Model_Avg_Error),]

df = df[order(df$Model_Avg_Error),]
a = cbind.fill(df_macro1, df_macro2, df_macro3, df, df_macro4)
# save
write.csv(a, "P:/Model_Output/JWN.csv", row.names = F, na="")

【问题讨论】:

  • 你能写一个函数来输入文件名吗?
  • 我在想类似的事情,然后循环遍历文件名并输出不同的输出?你介意提供一个例子吗?谢谢。
  • myfunc &lt;- function(fname) { Macro &lt;- read.csv(fname, ...); ...; write.csv(a, ...); } 将有效地“返回”无数据,但其副作用将是创建输出文件。如果您希望每个文件中都可以使用a 变量,请以return(a) 结束您的函数;此时,使用alldat &lt;- lapply(filenames, myfunc) 是有意义的。 (请注意,保存文件返回a的副作用可能会产生不良后果......副作用可能是有问题的。)

标签: r loops lapply


【解决方案1】:

只需创建一个文件名列表并像这样遍历它们:

Files = c("JWN", "BBY")

for(f in Files) {
    InFile  = paste("P:/R/R_Input/", f, "_Input.csv",  sep="")
    OutFile = paste("P:/R/R_Input/", f, "_Output.csv", sep="")
    Macro <- read.csv(InFile)

      ## All of that other code 

      write.csv(a, OutFile, row.names = F, na="")
}

基于 cmets 的附录: 原始海报出现错误:

文件中的错误(con,“w”):所有连接都在使用中”

已通过在 write.csv 语句之后立即添加 closeAllConnections() 来解决此问题。

【讨论】:

  • 谢谢。使用此方法时收到错误“文件中的错误(con,“w”):所有连接都在使用中”。
  • 这很奇怪。您能确定哪个语句引发了该错误吗?
  • 我不明白您为什么会收到该错误,但可能值得尝试在 write.csv 语句之后立即添加语句 closeAllConnections()
猜你喜欢
  • 1970-01-01
  • 1970-01-01
  • 1970-01-01
  • 2019-11-13
  • 1970-01-01
  • 2020-05-08
  • 2020-10-18
  • 1970-01-01
  • 2022-10-21
相关资源
最近更新 更多