【发布时间】：2022-01-20 17:42:53
【问题描述】：
我正在开发一个应用程序，它使用 Node.js 作为 Firebase Functions 和 SendGrid 的后端，以便在准备好跟踪大量订单时一次性向客户发送大量电子邮件。我可以将包含 1-3 的列表的 CSV 上传到系统，也许还有更多，并且都可以正常工作，但是当我有多达 100 或 1000 个时，系统将不会发送并且我收到错误：
Error with sending tracking emails: Error: socket hang up 后跟 Error setting processed to true: Error: 4 DEADLINE_EXCEEDED: Deadline exceeded 在 Firebase Function 的 Node.js 服务器日志上。像这样批量发送电子邮件是否有上限？关于问题可能出在哪里的任何线索？
相关代码片段：
"use strict";
import functions = require('firebase-functions');
import * as uuid from 'uuid';
import admin = require("firebase-admin");
import { DocumentSnapshot } from '@google-cloud/firestore';
import { RESPONSE_TYPES } from './common';
admin.initializeApp(functions.config().firebase);
const request = require("request");
const Papa = require('papaparse');
const sgMail = require('@sendgrid/mail')
sgMail.setApiKey(functions.config().sendgrid_api.key)
export const onUploadCreated = functions.firestore.document('users/{userId}/uploads/{uploadId}')
.onCreate(async (snap: DocumentSnapshot, context: functions.EventContext) => {
const newValue = snap.data();
if (newValue === null || newValue === undefined) {
return;
}
try {
const allPromises: Array<Promise<any>> = [];
console.log("Upload type set to set shipment tracking info...")
const orders: any = [];
// Parse through uploaded CSV
const options = {
download: true,
header: true,
worker: true
};
const parseStream = await Papa.parse(Papa.NODE_STREAM_INPUT, options);
allPromises.push(parseStream);
const dataStream = await request.get(newValue.fileUrl).pipe(parseStream);
allPromises.push(dataStream);
let parseError = "";
allPromises.push(
await parseStream.on("data", async (chunk: any) => {
if(
chunk['Order ID'] && chunk['Order ID'] !== undefined && chunk['Order ID'] !== null &&
chunk['Order Number'] && chunk['Order Number'] !== undefined && chunk['Order Number'] !== null &&
chunk['First Name'] && chunk['First Name'] !== undefined && chunk['First Name'] !== null &&
chunk['Last Name'] && chunk['Last Name'] !== undefined && chunk['Last Name'] !== null &&
chunk['Email'] && chunk['Email'] !== undefined && chunk['Email'] !== null &&
chunk['Tracking'] && chunk['Tracking'] !== undefined && chunk['Tracking'] !== null
){
// If in e notation, throw error
if(/e\+|E\+/g.test(chunk['Tracking'])){
parseError = "Tracking is in E notation, please reformat as outlined in NOTE 3 and re-upload."
} else {
await orders.push({
id: chunk['Order ID'],
number: chunk['Order Number'],
productName: chunk['Product Name'] || '',
productSize: chunk['Product Size'] || '',
productVariant: chunk['Product Variant'] || '',
firstName: chunk['First Name'],
lastName: chunk['Last Name'],
email: chunk['Email'],
tracking: chunk['Tracking'],
carrier: chunk['Carrier'] || '',
method: chunk['Method'] || '',
})
}
} else {
parseError = "Required columns were not properly defined properly. Please check your file follows the NOTES and re-upload."
}
})
);
// TODO: if multiple products, we need to just create more rows for each product like BS does
allPromises.push(
await dataStream.on("finish", async () => {
if(orders.length > 1000){
parseError = "You uploaded over 1000 orders to this upload. Please retry by only submitting 1000 orders at a time so we don't overwhelm the servers!"
}
if(parseError.length === 0){
console.log("Parsing complete, no errors. Number of orders processed: " + orders.length + ". Starting to add orders to Firestore Order docs now... ")
const emailMessages: { to: any; from: string; subject: string; text: string; html: string; }[] = [];
let shopData: FirebaseFirestore.DocumentData | any = null;
await admin.firestore().collection('public').doc("shop").get().then((shopDoc) => {
if (shopDoc.exists) {
let docWithMore = Object.assign({}, shopDoc.data());
docWithMore.id = shopDoc.id;
shopData = docWithMore;
} else {
console.error("Shop doc doesn't exist!")
}
}).catch((error) => {
console.log("Error getting shop document:", error);
})
const batchArray: any = [];
batchArray.push(admin.firestore().batch());
console.log("batchArray.length: " + batchArray.length);
let operationCounter = 0;
let batchIndex = 0;
let ordersProcessed = 0;
allPromises.push(
await orders.forEach((order: any) => {
const orderRef = admin.firestore().collection('orders').doc(order.id)
batchArray[batchIndex].set(
orderRef,
{
shipment: {
tracking: order.tracking,
carrier: order.carrier || '',
method: order.method || ''
}
},
{ merge: true }
);
const htmlEmail =
`
<div style="width: 100%; font-family: Arial, Helvetica, sans-serif">
${shopData?.nav?.showLogo ?
`
<div style="text-align: center;">
<img
alt="company logo"
src="${shopData?.logoUrl}"
width="${shopData?.nav?.logoSize || "200"}" height="auto"
/>
</div>
`
: ""
}
${shopData?.nav?.showTitle ? `<h1 style="margin: 20px 0 0 0; text-align: center;">${shopData?.name}</h1>` : ""}
<div style="margin: auto; width: 70%; padding: 1%;">
<h2>Great news ${order.firstName}!</h2>
<p>
Your order #${order.number} has shipped with the tracking # <b>${order.tracking}</b>.
</p>
${order?.productName ? `<p><b>Product Name:</b> ${order.productName}</p>` : ""}
${order?.productSize ? `<p><b>Product Size:</b> ${order.productSize}</p>` : ""}
${order?.productVariant ? `<p><b>Product Variant:</b> ${order.productVariant}</p>` : ""}
${order?.method ? `<p><b>Shipping Method:</b> ${order.method}</p>` : ""}
${order?.carrier ? `<p><b>Shipping Carrier:</b> ${order.carrier}</p>` : ""}
${shopData?.emails?.order?.shipment ?? ''}
<p>
Feel free to reach out to <a href="mailto:${shopData?.emails?.support ?? `help@email.com`}">${shopData?.emails?.support ?? `help@email.com`}</a> if you have any questions!
</p>
</div>
</div>
`
const msg = {
to: order.email,
from: `noreply@email.com`,
subject: `${shopData?.name} Order #${order.number} Shipped!`,
text: `
Great news! We shipped out your order with the tracking #${order.tracking}!
${order?.method ? ` Shipping Method: ${order.method} ` : ""}
${order?.carrier ? ` Shipping Carrier: ${order.carrier} ` : ""}
`,
html: htmlEmail,
}
emailMessages.push(msg)
operationCounter++;
ordersProcessed++;
if (operationCounter === 499) {
console.log("operationCounter: " + operationCounter)
batchArray.push(admin.firestore().batch());
batchIndex++;
operationCounter = 0;
}
})
);
// Attempt to send out emails, if it fails, do not continue with adding changes to Firestore! Sending email is the main purpose of this process.
allPromises.push(
sgMail.send(emailMessages).then(async () => {
console.log(`${emailMessages.length} tracking emails sent successfully!`);
console.log("Starting to push batchArray.length: " + batchArray.length);
let batchesProcessed = 0;
allPromises.push(
await batchArray.forEach(async (batch: any) => {
console.log("batchesProcessed: " + batchesProcessed);
batchesProcessed++;
allPromises.push(await batch.commit())
if(batchesProcessed === batchArray.length){
console.log("Done pushing " + batchesProcessed + " batches!");
allPromises.push(
admin.firestore().collection('users').doc(context.params.userId).collection('uploads').doc(context.params.uploadId).set({
processed: true,
response: {
type: RESPONSE_TYPES.SUCCESS,
message: `Successfully sent ${emailMessages.length} tracking emails and updated ${ordersProcessed} orders on the database with this tracking info.`,
read: false,
timestamp: Date.now()
}
}, { merge: true }).then(() => {
console.log("Set upload processed to true.")
})
);
}
})
);
}).catch((error: any) => {
console.log("Error with sending tracking emails: " + error);
allPromises.push(
admin.firestore().collection('users').doc(context.params.userId).collection('uploads').doc(context.params.uploadId).set({
processed: true,
response: {
type: RESPONSE_TYPES.ERROR,
message: `Error sending tracking emails: ${error}`,
read: false,
timestamp: Date.now()
}
}, { merge: true }).then(() => {
console.log("Set upload processed to true.")
}).catch(err => {
console.error("Error setting processed to true: " + err)
})
);
})
)
} else {
console.error("Error parsing shipments CSV: " + parseError)
allPromises.push(
admin.firestore().collection('users').doc(context.params.userId).collection('uploads').doc(context.params.uploadId).set({
processed: true,
response: {
type: RESPONSE_TYPES.ERROR,
message: `Error parsing shipments CSV: ${parseError}`,
read: false,
timestamp: Date.now()
}
}, { merge: true }).then(() => {
console.log("Set upload processed to true and parseError")
}).catch(error => {
console.error("Error setting processed to true and parseError: " + error)
})
);
}
})
);
return Promise.all(allPromises)
} catch(error) {
console.error("Error: " + error);
return;
}
});
【问题讨论】：
标签： node.js firebase google-cloud-functions sendgrid