如何从 Lambda 函数解析 AWS S3 文件

Question

我需要一些帮助来正确构建代码以使用 S3 存储桶和 Lambda 函数处理一些文本文件。

我想使用由在 S3 存储桶中创建新对象触发的 Lambda 函数来读取文件并提取一些数据并将其写入放置在另一个 S3 存储桶中的文件。

到目前为止，我的函数工作正常，可以将文件从一个 S3 存储桶复制到另一个存储桶，但我不太清楚如何添加一个函数来处理文件并将结果写入最终的 S3 目标。

这些文件是简单的文本文件，我需要从文件的每一行中提取数据。

如果我当前使用的 Node.js 代码添加了一个附加函数来处理文件，请参见下面的带有 ?? 的 cmets我在哪里寻求帮助。

// dependencies
var async = require('async');
var AWS = require('aws-sdk');
var util = require('util');


// get reference to S3 client 
var s3 = new AWS.S3();

exports.handler = function(event, context) {
    // Read options from the event.
    console.log("Reading options from event:\n", util.inspect(event, {depth: 5}));
    var srcBucket = event.Records[0].s3.bucket.name;
    // Object key may have spaces or unicode non-ASCII characters.
    var srcKey    =
    decodeURIComponent(event.Records[0].s3.object.key.replace(/\+/g, " "));  
    var dstBucket = "inputBucket";
    var dstKey    = srcKey + ".txt";

    // Sanity check: validate that source and destination are different buckets.
    if (srcBucket == dstBucket) {
        console.error("Destination bucket must not match source bucket.");
        return;
    }

    // Infer the file type.
    var typeMatch = srcKey.match(/\.([^.]*)$/);
    if (!typeMatch) {
        console.error('unable to infer file type for key ' + srcKey);
        return;
    }
    var imageType = typeMatch[1];
    if (imageType != "txt") {
        console.log('skipping non-image ' + srcKey);
        return;
    }

    // Download the image from S3, transform, and upload to a different S3 bucket.
    async.waterfall([
        function download(next) {
            // Download the file from S3 into a buffer.
            s3.getObject({
                    Bucket: srcBucket,
                    Key: srcKey
                },
                next);
            },
        function transform(response, next) {
            // Read the file we have just downloaded 
            // ? response.Body ?
            var rl = require('readline').createInterface({
                input: require('fs').createReadStream('file.in')
            });

            // Process each line here writing the result to an output buffer?
            rl.on('line', function (line) {
                 console.log('Line from file:', line);
                //Do something with the line... 

                //Create some output string 'outputline'

                //Write 'outputline' to an output buffer 'outbuff'
                // ??

            });
            // Now pass the output buffer to the next function
            // so it can be uploaded to another S3 bucket 
            // ?? 
            next;
        }
        function upload(response, next) {
            // Stream the file to a different S3 bucket.
            s3.putObject({
                    Bucket: dstBucket,
                    Key: dstKey,
                    Body: response.Body,
                    ContentType: response.contentType
                },
                next);
            }
        ], function (err) {
            if (err) {
                console.error(
                    'Unable to process ' + srcBucket + '/' + srcKey +
                    ' and upload to ' + dstBucket + '/' + dstKey +
                    ' due to an error: ' + err
                );
            } else {
                console.log(
                    'Successfully processed ' + srcBucket + '/' + srcKey +
                    ' and uploaded to ' + dstBucket + '/' + dstKey
                );
            }

            context.done();
        }
    );
};

Answer 1

在 s3.getObject 的回调中

s3.getObject(params,function(err,data){}) 

如果您的文件是文本，那么您可以将文本提取为字符串

data.Body.toString("utf-8")