阅读R中的网页列表并将输出保存在csv中

Question

我在 csv 文件中有一个包含 40,000 个网页地址的列表。我想在一个新的 csv 文件中读取这些页面，以便 csv 中的每个单元格都是相关网页的内容。 我可以使用以下代码读取（解析）单个网页

library(XML)

# Read and parse HTML file
doc.html = htmlTreeParse('',useInternal = TRUE)

# Extract all the paragraphs (HTML tag is p, starting at
# the root of the document). Unlist flattens the list to
# create a character vector.
doc.text = unlist(xpathApply(doc.html, '//p', xmlValue))

# Replace all \n by spaces
doc.text = gsub('\\n', ' ', doc.text)

# Join all the elements of the character vector into a single
# character string, separated by spaces
doc.text = paste(doc.text, collapse = ' ')

是否可以使用以网页地址为输入的 csv 并获取包含上述所有内容的新文件？

Answer 1

你可以试试下面的代码。它应该可以满足您的目的，但未经测试，因为我不知道您要查看的网站：

library(XML)
library(rvest)


df <- read.csv("Webpage_urls.csv", stringsAsFactors = F)

webpage_parser <- function(x){
x <- read_html(x)

  doc.html = htmlTreeParse(x, useInternal = TRUE)
  # Extract all the paragraphs (HTML tag is p, starting at
  # the root of the document). Unlist flattens the list to
  # create a character vector.
  doc.text = unlist(xpathApply(doc.html, '//p', xmlValue))

  # Replace all \n by spaces
  doc.text = gsub('\\n', ' ', doc.text)
  # Join all the elements of the character vector into a single
  # character string, separated by spaces
  doc.text = paste(doc.text, collapse = ' ')
}

all_webpages <- lapply(df, function(x) webpage_parser(x))

Pages <- do.call(rbind, all_webpages)

Parsed_pages <- cbind(df, Pages)

write.csv(Parsed_pages, "All_parsed_pages.csv", row.names = F)

如果我们想同时，我们可以使用 R 中的 doParallel 库，这将设置多个集群（R 的实例）并且应该有助于加快您的进程。

library(doParallel)
# split your webpage list into n vectors and create a list called Split_df

Split_df <- list(df1, df2, df3,..., dfn)

# Here I initiate my cluster
cl <- makeCluster(detectCores()-1)
registerDoParallel(cl)

Parsed_pages  <- foreach(i = 1:length(Split_df), .combine = rbind) %dopar%
{
  library(rvest)
  library(XML)
  all_webpages <- lapply(Split_df[[i]], function(x) webpage_parser(x))

  Pages <- do.call(rbind, all_webpages)

  Parsed_pages <- cbind(Split_df[[i]], Pages)

  Parsed_pages
}
stopCluster(cl)

write.csv(Parsed_pages, "All_parsed_pages.csv", row.names = F)