【问题标题】:Connection closed before finishing the inserts in the database在完成数据库中的插入之前连接关闭
【发布时间】:2026-01-28 07:45:01
【问题描述】:

我正在 Node.js 中创建我的第一个应用程序,它应该按顺序执行这些操作:

  1. 连接到 PostgreSQL 数据库
  2. 创建一个表(如果它已经存在,做任何事情)
  3. 对两个网站进行网页抓取并将信息保存在数据库中
  4. 使用 D3.js 和数据库中的数据创建可视化效果
  5. 它与数据库断开连接。

我发布了一个关于插入操作的previous question。现在问题是异步/等待。

我知道 Node.js 是异步的,而我应该同步执行操作。 我使用的是 Node.js v8.9.4,所以我使用 async/await 构造。

我读了一点“如何异步/等待,但我发现很难正确使用它。

这是我的应用程序的方案(app.js 文件):

const postgreSQLlib = require('./middlewares/postgreSQLlib.js')
const scraperCovIt = require('./routers/scraperCovIt.js');
const scraperCov = require('./routers/scraperCov.js');

const start = async function() {
    // STEP 0 - Start
    await console.log('\nSTART');

    // STEP 1 - Connect to db
    await postgreSQLlib.connect();

    // STEP 2 - Create tables
    var queryCreateCoverages = {
        text: 'CREATE TABLE IF NOT EXISTS coverages ('+
                    'id SERIAL PRIMARY KEY,' +
                    'vaccine VARCHAR(64) NOT NULL,' + 
                    'country VARCHAR(255) NOT NULL,' +
                    'region VARCHAR(255),' +
                    'year VARCHAR(4) NOT NULL,' +
                    'value VARCHAR(12) NOT NULL);'
    };
    var queryRes = await postgreSQLlib.query(queryCreateCoverages, '[CREATE TABLE coverages]');

    // STEP 3 - Get data
    await scraperCovIt.download();
    await scraperCov.download();

    // STEP 4 - Disconnect from db
    await postgreSQLlib.disconnect();

    // STEP 5 - End
    return '\nFINISH';
}

// start application
start()
.then(function(res) {
    console.log(res);
})
.catch(function(err) {
    console.log(err);
});

postgreSQLlib.js

const {Client} = require('pg'); 
const connectionString = 'postgres://admin:admin@localhost:5432/db';
let client;
var methods = {};

methods.connect = async function() {
    client = new Client({connectionString});
    return await client.connect()
    .then(async function() {
        await console.log('\nConnected to ' + client.database + ' at ' + client.host + ':' + client.port + ' as ' + client.user + ' (pass: ' + client.password + ')');
    })
    .catch(function(err) {
        console.log('\nError during connection to PostgreSQL');
        throw err;
    });
}

methods.query = async function(query, print) {
    return await client.query(query)
    .then(function(res) {
        console.log(print, 'OK query');
        return res;
    })
    .catch(function(err) { 
        console.log(print, 'ERR query');
    });
}

methods.disconnect = async function() {
    return await client.end()
    .then(function() {
        console.log('\nConnection has ended');
    })
    .catch(function(err) {
        console.log('\nError during clossing connection');
        throw err;
    }); 
}

module.exports = methods;

scraperCov.js

var cheerio = require('cheerio');
var request = require('request-promise');
var postgreSQLlib = require('../middlewares/postgreSQLlib.js');

var methods = {};

var countries = {
    'Albania': 'ALB',
    'Austria': 'AUT',
    'Belgium': 'BEL'
};

methods.download = async function(req, res) {
    for(country in countries) {
        console.log('\nCOUNTRY:', country);
        var url = 'http://apps.who.int/immunization_monitoring/globalsummary/coverages?c=' + countries[country];
        let res = await request(url);
        insert(res);
    }
}

module.exports = methods;

let insert = async function(html) {
    $ = cheerio.load(html);

    var years = [];
    var vaccines = [];
    var coverages = [];

    $('.ts .year').each(function() {
        years.push($(this).text().trim());
    });
    $('.ts .odd td a, .ts .even td a').each(function() {
        vaccines.push($(this).text().trim());
    });
    $('.ts .odd .statistics_small, .ts .even .statistics_small').each(function() {
        coverages.push($(this).text().trim());
    });

    const numYears = years.length;
    const numVaccines = vaccines.length;
    for(var vaccineIdx = 0; vaccineIdx < numVaccines; vaccineIdx++) {
        for(var yearIdx = 0; yearIdx < numYears; yearIdx++) {
            let obj = {
                year: years[yearIdx],
                country: country,
                region: '',
                vaccine: vaccines[vaccineIdx],
                coverage: coverages[vaccineIdx*numYears + yearIdx]
           }

            // save on db
            const queryInsert = {
                text: 'INSERT INTO coverages (vaccine, country, region, year, value) VALUES ($1, $2, $3, $4, $5);',
                values: [vaccines[vaccineIdx], country, '', years[yearIdx], coverages[vaccineIdx*numYears + yearIdx]]
            }
            var printText = '[INSERT ' + country + ' IN coverages]';
            var queryRes = await postgreSQLlib.query(queryInsert, printText);
        }
    }
}

scraperCovIt.js

var textract = require('textract');
var postgreSQLlib = require('../middlewares/postgreSQLlib.js');

var methods = {};
var jsons = [];
var mainUrl = 'http://www.salute.gov.it/portale/documentazione/p6_2_8_3_1.jsp?id=20';
var urls = [ 
    {year: '2013', link: 'http://www.salute.gov.it/imgs/C_17_tavole_20_allegati_iitemAllegati_0_fileAllegati_itemFile_1_file.pdf'}, 
    {year: '2012', link: 'http://www.salute.gov.it/imgs/C_17_tavole_20_allegati_iitemAllegati_5_fileAllegati_itemFile_0_file.pdf'}, 
    {year: '2011', link: 'http://www.salute.gov.it/imgs/C_17_tavole_20_allegati_iitemAllegati_6_fileAllegati_itemFile_0_file.pdf'}
];

methods.download = async function(req, res) {
    await extractText();
}

async function extractText() {
    var config = {
        preserveLineBreaks: true
    };
    urls.forEach(function(url) {
        textract.fromUrl(url.link, config, async function(error, text) {
            if(error) {
                throw error;
            }
            switch(url.year) {
                case '2011': 
                case '2012':
                    await extractTextType1(url, text);
                    break;
                case '2013': 
                    await extractTextType2(url, text);
                    break;
                default:
                    await console.log('Error: no case');
            }
        });
    });
}

async function extractTextType1(url, text) {
    var matrix = [];
    var map = [];
    var vaccines = [];
    var regionsTemp = [];
    var regions = [];
    var regionLength = [1, 2, 1, 2, 3, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1];

    var textArray = text.split('\n');
    for(var i = 0; i < 23; i++) {
        matrix[i] = textArray[i].split(' ');
    }

    matrix[0].shift();
    vaccines = matrix[0];
    map[0] = vaccines;

    for(var i = 0; i < regionLength.length; i++) {
        var j = i + 1; 
        var indexToRemove = 0;
        var numberToRemove = regionLength[i];
        var region = matrix[j].splice(indexToRemove, numberToRemove);
        regionsTemp.push(region);
        map[j+1] = matrix[j];
    }

    for(var i = 0; i < regionsTemp.length; i++) {
        var region = '';
        if(regionLength[i] > 1) {
            region = regionsTemp[i].join(' ');
        }
        else {
            region = regionsTemp[i].join('');
        }
        regions.push(region);
    }
    map[1] = regions;

    for(var i = 0; i < map.length; i++) {
        for(var j = 0; j < map[i].length; j++) {
            map[i][j] = map[i][j].replace(/\r/g, '');
        }
    }

    vaccines = map.shift();
    regions = map.shift();

    var thisJson = map.reduce(function(result, v, i) {
        v.forEach(async function(o, k) {
            var obj = createJsonObjectCoverage(url.year, 'Italy', vaccines[k], regions[i], o);
            // save on db
            const queryInsertDoNothing = {
                text: 'INSERT INTO coverages (vaccine, country, region, year, value) VALUES ($1, $2, $3, $4, $5) ON CONFLICT (vaccine, country, region, year, value) DO NOTHING;',
                values: [vaccines[k], 'Italy', regions[i], url.year, o]
            };
            var printText = '[INSERT Italy IN coverages]';
            var queryRes = await postgreSQLlib.query(queryInsertDoNothing, printText);
        });
        return result;
    }, jsons);
}

async function extractTextType2(url, text) {
    // similar code to extractTextType1(url, text)
}

function createJsonObjectCoverage(year, country, vaccine, region, coverage) {
    return {
        year: year, 
        country: country, 
        region: region, 
        vaccine: vaccine, 
        coverage: coverage
    };
}

module.exports = methods;

问题是数据库中的插入不能正常工作。

scraperCov.js 运行 app.js 的结果是:

START

Connected to db at localhost:5432 as admin (pass: admin)
[CREATE TABLE coverages] OK query

COUNTRY: Albania

COUNTRY: Austria
[INSERT Albania IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query

COUNTRY: Belgium
[INSERT Austria IN coverages] OK query
[INSERT Austria IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query
[INSERT Belgium IN coverages] OK query

Connection has ended

FINISH
[INSERT Belgium IN coverages] ERR query

从打印中可以看出,最后出现错误,因为在插入结束之前连接已关闭。 另外,插入的顺序不正确(COUNTRY: Albania这个词下什么都没有,而COUNTRY: Austria这个词下有一些与阿尔巴尼亚相关的查询结果等)。

使用 scraperCovIt.js 运行 app.js 的结果是:

START

Connected to db at localhost:5432 as admin (pass: admin)
[CREATE TABLE coverages] OK query

Connection has ended

FINISH

在这种情况下,插入并没有完全执行。

我该如何解决?

这些天我试图解决这个问题,但我不知道该怎么做。这是我第一次使用async/await,遇到了很多麻烦。

感谢任何想帮助我的人

【问题讨论】:

  • 您似乎对await 的作用有些困惑。像await console.log(...) 这样的语句与console.log(...) 完全没有什么不同。 await 做了一些有用的事情,等待与异步操作相关的承诺。我的猜测是您在一个或多个不返回承诺的函数上使用await,并期望它实际上等待异步完成,但除非函数返回与异步操作。
  • 此外,return await fn()return fn() 没有什么不同。如果fn() 返回一个promise,那么你可以直接返回这个promise。
  • 我不知道你正在使用的库(特别是数据库),但你应该访问await 的每一次使用,并确保你等待的是一个承诺。如果没有,您要么需要更改要调用的函数以返回承诺,要么需要与使用 await 不同的设计。
  • 这是一个明显错误的:await extractText();。如果您查看extractText(),它确实返回了一个promise(只是因为它被声明为async),但是该promise 根本没有连接到函数内部的异步操作。整个 extractText() 函数必须重新设计,以便在实际完成时通过 Promise 进行通信。你不能成功地混合回调和承诺,这是 extractText() 在其实现中所做的。
  • @jfriend00 那么,scraperCovIt.js 中的问题在于v.forEach(async function(o, k) {,因为我将回调函数称为async?在 scraperCov.js 我有 dowloadasync 但不返回任何承诺,对吧?然后是await request 它应该返回一个承诺,但实际上它没有,对吧?我有点困惑

标签: node.js postgresql async-await


【解决方案1】:

正如我们在 cmets 中讨论的那样,这里有很多错误。以下是部分概念列表:

  1. await 仅在等待与完成异步操作直接相关的承诺时才等待异步操作。将await 用于除了promise 之外的任何东西都没有任何用处。它没有什么魔力可以知道某些底层异步操作何时完成。 await 只遵循一个承诺。

  2. .forEach() 循环内的await 不会暂停该循环的执行。

  3. 不要混用回调代码和承诺代码。混合它们时,真的很难让它们正常工作。如果您有一些只需要回调的代码,则将其转换为返回 Promise 的新函数。您通常可以使用util.promisify() 在一行代码中创建新函数。

  4. 我不可能攻击你的整个代码,因为我什至不知道它在做什么,但我会挑选几个例子告诉你如何修复它们。我开始查看extractText(),它返回一个promise(只是因为它被声明为async,但该promise 根本没有连接到它内部的异步操作。

  5. 这让我想到了extractText() 调用的一些函数,例如extractTextType1(),所以我将首先向您展示如何修复它。

extractTextType1() 中,您在.forEach() 循环内使用await postgreSQLlib.query(...),该循环位于.reduce() 循环内。 await 根本不会导致这两种类型的循环暂停。因此,您的 extractTextType1() 函数将在任何这些查询完成之前很久就返回。由于看起来这些查询(实际上是插入)不必排序并且可以并行运行,我们可以简单地将所有查询中的 Promise 收集到一个数组中,然后使用Promise.all() 来跟踪它们何时全部完成完毕。

该代码如下所示:

function extractTextType1(url, text) {
    var matrix = [];
    var map = [];
    var vaccines = [];
    var regionsTemp = [];
    var regions = [];
    var regionLength = [1, 2, 1, 2, 3, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1];

    var textArray = text.split('\n');
    for(var i = 0; i < 23; i++) {
        matrix[i] = textArray[i].split(' ');
    }

    matrix[0].shift();
    vaccines = matrix[0];
    map[0] = vaccines;

    for(var i = 0; i < regionLength.length; i++) {
        var j = i + 1; 
        var indexToRemove = 0;
        var numberToRemove = regionLength[i];
        var region = matrix[j].splice(indexToRemove, numberToRemove);
        regionsTemp.push(region);
        map[j+1] = matrix[j];
    }

    for(var i = 0; i < regionsTemp.length; i++) {
        var region = '';
        if(regionLength[i] > 1) {
            region = regionsTemp[i].join(' ');
        }
        else {
            region = regionsTemp[i].join('');
        }
        regions.push(region);
    }
    map[1] = regions;

    for(var i = 0; i < map.length; i++) {
        for(var j = 0; j < map[i].length; j++) {
            map[i][j] = map[i][j].replace(/\r/g, '');
        }
    }

    vaccines = map.shift();
    regions = map.shift();

    let promises = [];
    var thisJson = map.reduce(function(result, v, i) {
        v.forEach(async function(o, k) {
            var obj = createJsonObjectCoverage(url.year, 'Italy', vaccines[k], regions[i], o);
            // save on db
            const queryInsertDoNothing = {
                text: 'INSERT INTO coverages (vaccine, country, region, year, value) VALUES ($1, $2, $3, $4, $5) ON CONFLICT (vaccine, country, region, year, value) DO NOTHING;',
                values: [vaccines[k], 'Italy', regions[i], url.year, o]
            };
            var printText = '[INSERT Italy IN coverages]';
            promises.push(postgreSQLlib.query(queryInsertDoNothing, printText));
        });
        return result;
    }, jsons);
    return Promise.all(promises);
}

现在extraTextType1() 返回一个仅在内部所有异步操作完成时才解析的承诺。如果其中任何一个失败,它将拒绝。

另外,请注意extraTextType1() 不再声明为async,因为没有使用await。它只是直接返回一个承诺。如果您需要对这些插入进行排序,那么循环将需要更彻底的重写才能一次完成。


假设您对extraTextType2() 应用了类似的修复,现在让我们修复包含函数extractText()。在这里,我们承诺textract.fromUrl(),因此我们不会尝试混合回调和承诺,我们将.forEach() 循环更改为for/of 循环,以便await 将与循环一起使用。然后,从console.log() 中删除await

const util = require('util');
const textractFromUrl = util.promisify(textract.fromUrl);

async function extractText() {
    var config = {
        preserveLineBreaks: true
    };
    for (let url of urls) {
        let text = await textractFromUrl(url.link, config);
        switch(url.year) {
            case '2011': 
            case '2012':
                await extractTextType1(url, text);
                break;
            case '2013': 
                await extractTextType2(url, text);
                break;
            default:
                console.log('Error: no case');
        }
    }
}

然后,您可以通过以下更改来清理下载功能:

methods.download = async function(req, res) {
    await extractText();
}

到这里:

methods.download = function(req, res) {
    return extractText();
}

【讨论】:

  • 非常感谢。我已经测试了代码,它运行良好。现在我尝试理解它,然后尝试解决另一个文件。你有什么指南/教程可以建议我更好地理解async/await吗?
  • @ComeDown - async/await 的常见错误是在您了解异步编程和承诺之前尝试使用它。这些是先决条件。 Async/await 不要取代这种理解。它们在某些情况下使编程更容易,但仍然必须完全理解异步编程以及如何将 Promise 用于异步编程。然后,也只有到那时,你才能利用async/await 来简化一些代码。试图使用async/await 而不是理解异步编程和承诺会给你带来麻烦。
  • 谢谢。今天我也解决了scraperCov.js的问题。我学到了很多。谢谢!