node.js 网络爬虫图片/视频

Question

最近我一直在使用网络爬虫，我已经想出了如何搜索文本，但是有什么方法可以让我的网络爬虫搜索其他内容，例如视频和图像，然后下载并组织它们。

到目前为止，这是我的网络爬虫：

var request = require('request');
var cheerio = require('cheerio');
var URL = require('url-parse');
var fs = require('fs');

var START_URL = "https://www.google.com";
var SEARCH_WORD = "apples";
var MAX_PAGES_TO_VISIT = 200;

var pagesVisited = {};
var numPagesVisited = 0;
var pagesToVisit = [];
var url = new URL(START_URL);
var baseUrl = url.protocol + "//" + url.hostname;

pagesToVisit.push(START_URL);
 crawl();

 function crawl() {
  if(numPagesVisited >= MAX_PAGES_TO_VISIT) {
    console.log("Reached max limit of number of pages to visit.");
     return;
  }
  var nextPage = pagesToVisit.pop();
  if (nextPage in pagesVisited) {
    // We've already visited this page, so repeat the crawl
    crawl();
  } else {
    // New page we haven't visited
    visitPage(nextPage, crawl);
  }
}

function visitPage(url, callback) {
  // Add page to our set
  pagesVisited[url] = true;
 numPagesVisited++;

  // Make the request
 console.log("Visiting page " + url);
  request(url, function(error, response, body) {
     // Check status code (200 is HTTP OK)
     console.log("Status code: " + response.statusCode);
     if(response.statusCode !== 200) {
       callback();
       return;
    }
    // Parse the document body
      var $ = cheerio.load(body);
     var isWordFound = searchForWord($, SEARCH_WORD);
     if(isWordFound) {
      console.log('Word ' + SEARCH_WORD + ' found at page ' + url);
     } else {
       collectInternalLinks($);
       // In this short program, our callback is just calling crawl()
       callback();
     }
  });
 }

function searchForWord($, word) {
  var bodyText = $('html > body').text().toLowerCase();
  return(bodyText.indexOf(word.toLowerCase()) !== -1);
}

 function collectInternalLinks($) {
     var relativeLinks = $("a[href^='/']");
console.log("Found " + relativeLinks.length + " relative links on page");
relativeLinks.each(function() {
    pagesToVisit.push(baseUrl + $(this).attr('href'));
});
}

我从在线教程中获得了大部分代码来帮助我开始，但我需要更多帮助代码工作我只是想知道是否以及如何可以网络抓取图像和视频。

较新的代码：

var request = require('request');
var cheerio = require('cheerio');
var URL = require('url-parse');
var fs = require('fs');

var START_URL = "http://moetube.net";
//var SEARCH_WORD = "anime";
 var MAX_PAGES_TO_VISIT = 200;

 var pagesVisited = {};
 var numPagesVisited = 0;
 var pagesToVisit = [];
  var url = new URL(START_URL);
 var baseUrl = url.protocol + "//" + url.hostname;

 pagesToVisit.push(START_URL);
 crawl();

 function crawl() {
   if(numPagesVisited >= MAX_PAGES_TO_VISIT) {
     console.log("Reached max limit of number of pages to visit.");
     return;
   }
   var nextPage = pagesToVisit.pop();
   if (nextPage in pagesVisited) {
     // We've already visited this page, so repeat the crawl
     crawl();
    } else {
       // New page we haven't visited
       visitPage(nextPage, crawl);
     }
   }

   function visitPage(url, callback) {
    // Add page to our set
    pagesVisited[url] = true;
    numPagesVisited++;

     // Make the request
     console.log("Visiting page " + url);
    request(url, function(error, response, body) {
    var $ = cheerio.load(body);
      // Check status code (200 is HTTP OK)
      console.log("Status code: " + response.statusCode);
      collectImages($);
      if(response.statusCode !== 200) {
        callback();

          return;
       }
      // Parse the document body

     // var isWordFound = searchForWord($, SEARCH_WORD);

    // if(isWordFound) {
     //   console.log('Word ' + SEARCH_WORD + ' found at page ' + url);
    // } else {
       collectInternalLinks($);
       // In this short program, our callback is just calling crawl()
       callback();
   //  }
   });
}

   function searchForWord($, word) {
   var bodyText = $('html > body').text().toLowerCase();
  return(bodyText.indexOf(word.toLowerCase()) !== -1);
    }

function collectImages($) {

   return $("img").map(function() {
        return $(this).text();
         console.log((this).text() + "JHJHHHHHHHHHHHHHHHHHHHH");
     }).get();
      }

 function collectInternalLinks($) {

     var relativeLinks = $("a[href^='/']");
     console.log("Found " + relativeLinks.length + " relative links on page");
     relativeLinks.each(function() {
         pagesToVisit.push(baseUrl + $(this).attr('href'));
     });
 }

Answer 1

就像您使用cheerio 在正文中搜索链接一样，您也可以在正文中搜索<img> 或<video> 标签。当您找到这些标签时，您并没有确切说明您想要做什么，但是您可以创建一个类似于您的 collectInternalLinks() 的函数，该函数将收集媒体对象以进行进一步处理：

// return array of image URLs (these may be page-relative URLS)
function collectImages($) {
    return $("img").map(function() {
        return $(this).prop("src");
    }).get();
}

// return collection of video elements
function collectVideos($) {
    let videoUrls = [];
    $("video").each(function() {
        let src = $(this).prop("src");
        if (src) {
            videoUrls.push(src);
        } else {
            let subElements = $(this).find("track, source");
            subElements.each(function() {
                let src = $(this).prop("src");
                if (src) {
                    videoUrls.push(src);
                }
            });
        }
    });
    return videoUrls;
}

收集视频 URL 有点复杂，因为可以通过多种不同的方式指定这些 URL（.src 属性、嵌入的 <track> 标记、嵌入的 <source> 标记等），因此您可以为每个<video> 标签解析出每种可能的方式。