在 NodeJS 中下载大文件的最佳方法是什么？

Question

以下服务器代码适用于 5GB 文件，使用 wget http://localhost:11146/base/bigFile.zip 但不使用客户端代码。

服务器端代码。

var http = require("http");
var fs = require("fs");
var filename = "base/bigFile.zip";

var serv = http.createServer(function (req, res) {
    var stat = fs.statSync(filename);
    res.writeHeader(200, {"Content-Length": stat.size});
    var fReadStream = fs.createReadStream(filename);
    fReadStream.on('data', function (chunk) {
       if(!res.write(chunk)){
           fReadStream.pause();
       }
   });
   fReadStream.on('end', function () {
      res.end();
   });
   res.on("drain", function () {
      fReadStream.resume();
   });
});

serv.listen(1114);

使用请求模块的客户端代码。这段代码有什么问题？

var request = require('request')
request('http:/localhost:11146/base/bigFile.zip', function (error, response, body) {
  console.log('error:', error); // Print the error if one occurred
  console.log('statusCode:', response && response.statusCode); // Print the response status code if a response was received
  console.log('body:', body); // Print the HTML for the Google homepage.
});

上述客户端代码的错误如下 -

error: Error: Invalid protocol
    at Request.init (/Users/air/Projects/node_modules/request/request.js:338:51)
    at new Request (/Users/air/Projects//node_modules/request/request.js:105:8)
    at request (/Users/air/Projects/Vertico/Vertico-CLI/node_modules/request/index.js:53:11)
    at Object.<anonymous> (/Users/air/Projects/req.js:2:1)
    at Module._compile (module.js:569:30)
    at Object.Module._extensions..js (module.js:580:10)
    at Module.load (module.js:503:32)
    at tryModuleLoad (module.js:466:12)
    at Function.Module._load (module.js:458:3)
    at Function.Module.runMain (module.js:605:10)
statusCode: undefined
body: undefined

我将客户端修改为使用wget shell 进程而不是请求包，代码如下，问题是 - 我无法看到 wget 的良好下载进度，此代码的任何解决方法，这样我可以在子进程中看到进度条。

const fs = require('fs');
   const child_process = require('child_process');

   var workerProcess = child_process.spawn('wget', ['-O','fdsf.zip', 'http://localhost:11146/base/bigFile.zip']);

   workerProcess.stdout.on('data', function (data) {
     console.log('stdout: ' + data);
   });

   workerProcess.stderr.on('data', function (data) {
     //console.log('stderr: ' + data);
   });

   workerProcess.on('close', function (code) {
      console.log('Download Completed' + code);
   });

所以最后我想知道如何使用用 nodejs 编写的客户端代码下载文件？

Answer 1

更新：

---

request 模块已弃用

建议使用主动模块。我个人的偏好是got 有关示例，请参阅 got streams。
另一种选择是 node-fetch

---

最简单的方法是使用request模块

在这里，您尝试将整个结果存储在内存中并在控制台记录它。 5GB 相当大，要么您必须增加 Node.js 内存限制（不推荐），要么您必须使用流。请参阅下面来自request npm 文档的streaming example：

const fs = require('fs');
const request = require('request');
request('http://google.com/doodle.png').pipe(fs.createWriteStream('doodle.png'))

您必须通过管道传输响应，以便无论是 1MB、1GB 还是 1TB，只有一小部分文件会在内存中，并且会尽快写入磁盘。您可以使用 Node.js 内置函数使用相同的方法，但实现起来会很困难，并且当有 request 模块时就像重新发明轮子一样。

对于下载进度，您可以使用request-progress 模块和request 模块，请参见下面的示例（taken from their documentation）：

var fs = require('fs');
var request = require('request');
var progress = require('request-progress');
 
// The options argument is optional so you can omit it 
progress(request('https://az412801.vo.msecnd.net/vhd/VMBuild_20141027/VirtualBox/IE11/Windows/IE11.Win8.1.For.Windows.VirtualBox.zip'), {
    // throttle: 2000,                    // Throttle the progress event to 2000ms, defaults to 1000ms 
    // delay: 1000,                       // Only start to emit after 1000ms delay, defaults to 0ms 
    // lengthHeader: 'x-transfer-length'  // Length header to use, defaults to content-length 
})
.on('progress', function (state) {
    // The state is an object that looks like this: 
    // { 
    //     percent: 0.5,               // Overall percent (between 0 to 1) 
    //     speed: 554732,              // The download speed in bytes/sec 
    //     size: { 
    //         total: 90044871,        // The total payload size in bytes 
    //         transferred: 27610959   // The transferred payload size in bytes 
    //     }, 
    //     time: { 
    //         elapsed: 36.235,        // The total elapsed seconds since the start (3 decimals) 
    //         remaining: 81.403       // The remaining seconds to finish (3 decimals) 
    //     } 
    // } 
    console.log('progress', state);
})
.on('error', function (err) {
    // Do something with err 
})
.on('end', function () {
    // Do something after request finishes 
})
.pipe(fs.createWriteStream('IE11.Win8.1.For.Windows.VirtualBox.zip'));

Answer 2

var request = require('request')
request('http://localhost:11146/base/bigFile.zip', function (error, response, body) {
  console.log('error:', error); // Print the error if one occurred
  console.log('statusCode:', response && response.statusCode); // Print the response status code if a response was received
  console.log('body:', body); // Print the HTML for the Google homepage.
});

Answer 3

我的代码版本。

const fs = require( 'fs' );
const request = require( 'request' );
const progress = require( 'request-progress' );
const pre = '----';
const downloadManager = function ( url, filename ) {
    progress( request( url ), {
        throttle: 500
    } ).on( 'progress', function ( state ) {
        process.stdout.write( pre + '' + ( Math.round( state.percent * 100 ) ) + "%" );
    } ).on( 'error', function ( err ) {
        console.log( 'error :( ' + err );
    } ).on( 'end', function () {
        console.log( pre + '100% \n Download Completed' );
    } ).pipe( fs.createWriteStream( filename ) );
}
downloadManager( 'http://localhost:4181', 's.zip' );