谷歌云功能节点JS SFTP到云存储传输文件错误答案

【问题标题】：Google Cloud Function Node JS SFTP To Cloud Storage Transfer File Error谷歌云功能节点JS SFTP到云存储传输文件错误
【发布时间】：2019-01-29 01:01:45
【问题描述】：

我正在尝试使用节点库 ssh2-sftp 运行以下代码，但我没有从 GCP 控制台收到任何错误。我在这里要做的是将我从 SFTP 接收到的数据通过管道传输到 GCS 上的输出文件中。

代码如下：

  /**
 * Generic background Cloud Function to be triggered by Cloud Storage.
 *
 * @param {object} event The Cloud Functions event.
 * @param {function} callback The callback function.
 */

let request = require("request");
let Storage = require('@google-cloud/storage');
let Client = require('ssh2-sftp-client');
let sftp = new Client();
const fs = require('fs');
const path = require('path');
const storage = new Storage({projectId: process.env.PROJECT_ID});

exports.retrieveFilesFromSFTP = (event, callback) => {


    let pubsubMessage = event.data;
    let fileName = pubsubMessage.data
        ? Buffer.from(pubsubMessage.data, 'base64').toString()
        : 'No Files in pubsub';


    console.log(fileName, 'File name from Pub/Sub Message was received');


    fs.readdir(__dirname, (err, files) => {
        if (err) {
            console.error(err);
            res.sendStatus(500);
        } else {
            console.log('Files', files);
            res.sendStatus(200);
        }
    });

    getFilesFromSFTP(fileName);
    callback();
};

function getFilesFromSFTP(fileName) {

    const bucket = storage.bucket('dataflowdirectory');

    const outFile = bucket.file("relex_output/"+fileName);

    //TODO create a file on the bucket , get the correct file path and then pipe it to the file

    sftp.connect({
        host: process.env.SFTP_SERVER,
        port: process.env.SFTP_PORT,
        username: process.env.SFTP_USER,
        password: process.env.SFTP_PASSWORD
    }).then(() => {


        sftp.list(process.env.SFTP_OUTBOUND_PATH)
            .then((data) => {

                sftp.get(process.env.SFTP_OUTBOUND_PATH + "/" + fileName).then((data) => {

                    data.pipe((outFile.createWriteStream({gzip: true})
                        .on('error', function (err) {
                            console.error("error",err);
                        })
                        .on('finish', function () {
                            // The file upload is complete.
                            outFile.close();
                        })));

                });

            }).catch((err) => {
            console.log(err);

        });

    })

}

我从日志中看到的唯一内容如下：忽略已完成函数的异常

有什么办法解决这个问题吗？

谢谢，

【问题讨论】：

标签： node.js google-cloud-functions

【解决方案1】：

我可以用下面的代码解决它，我添加了文件的完整路径：relex_output 它仍然非常非常慢，因此任何性能改进都会受到赞赏：

/**
 * Generic background Cloud Function to be triggered by Cloud Storage.
 *
 * @param {object} event The Cloud Functions event.
 * @param {function} callback The callback function.
 */

let request = require("request");
let Storage = require('@google-cloud/storage');
let Client = require('ssh2-sftp-client');
let sftp = new Client();
const fs = require('fs');
const storage = new Storage({projectId: process.env.PROJECT_ID});

exports.retrieveFilesFromSFTP = (event, callback) => {


    let pubsubMessage = event.data;
    let fileName = pubsubMessage.data
        ? Buffer.from(pubsubMessage.data, 'base64').toString()
        : 'No Files in pubsub';


    console.log(fileName, 'File name from Pub/Sub Message was received');


    fs.readdir(__dirname, (err, files) => {
        if (err) {
            console.error(err);
            res.sendStatus(500);
        } else {
            console.log('Files', files);
            res.sendStatus(200);
        }
    });

    getFilesFromSFTP(fileName);
    callback();
};

function getFilesFromSFTP(fileName) {

    const bucket = storage.bucket('dataflowdirectory');

    const outFile = bucket.file("relex_output/"+fileName);



    sftp.connect({
        host: process.env.SFTP_SERVER,
        port: process.env.SFTP_PORT,
        username: process.env.SFTP_USER,
        password: process.env.SFTP_PASSWORD
    }).then(() => {


        sftp.list(process.env.SFTP_OUTBOUND_PATH)
            .then((data) => {

                sftp.get(process.env.SFTP_OUTBOUND_PATH + "/" + fileName).then((data) => {

                    data.pipe((outFile.createWriteStream({gzip: true})
                        .on('error', function (err) {
                            console.error("error",err);
                        })
                        .on('finish', function () {
                            // The file upload is complete.
                            sftp.end();
                        })));

                });

            }).catch((err) => {
            console.log(err);

        });

    })

【讨论】：