如何优化 .NET EF Core 批量插入？

Question

我正在开发一个用于在学校订餐的应用。这需要最终用户为不同类型（总秘书处、教师、学生......）添加帐户。

为此我制作了一个可靠的界面，但添加 1000 名学生需要 111 秒，因为一次插入需要经过所有查询。

这是插入的代码：

public async Task<List<FullPupil>> AddPupilAccountsBySchoolId(List<FullPupil> pupils, string schoolId, string password, string hashedPassword, Types type) {
        foreach (FullPupil pupil in pupils) {
            Accounts checkAccount = await _context.Accounts.FirstOrDefaultAsync(a => a.Email == pupil.Email);

            if (checkAccount == null) {
                Accounts newAccount = new Accounts() {
                    Email = pupil.Email,
                    FirstTime = true,
                    FirstTimePassword = password,
                    Password = hashedPassword,
                    Name = pupil.Name,
                    SurName = pupil.SurName,
                    EmailSend = false,
                };

                await _context.Accounts.AddAsync(newAccount);
                checkAccount = newAccount;
            }

            AccountsTypes checkAccountsTypes = await _context.AccountsTypes.FirstOrDefaultAsync(a => a.AccountId == checkAccount.AccountId && a.TypeId == type.TypeId);

            if (checkAccountsTypes == null) {
                AccountsTypes accountsTypes = new AccountsTypes() {
                    AccountId = checkAccount.AccountId,
                    TypeId = type.TypeId,
                };

                await _context.AccountsTypes.AddAsync(accountsTypes);
            }

            Pupils checkPupil = await _context.Pupils.FirstOrDefaultAsync(a => a.AccountId == checkAccount.AccountId);

            if (checkPupil == null) {
                Pupils newPupil = new Pupils() {
                    AccountId = checkAccount.AccountId,
                    ClassId = Guid.Parse(pupil.Class[0].Id),
                    CodeId = Guid.Parse(pupil.Code[0].Id),
                    ModifiedDate = DateTime.Now,
                };

                await _context.Pupils.AddAsync(newPupil);
            } else {
                checkPupil.ModifiedDate = DateTime.Now;
            }

            pupil.AccountId = checkAccount.AccountId;
        }

        await _context.SaveChangesAsync();
        return pupils;
    }

（我知道所有账号都会有相同的密码，这只是为了测试目的）

这是使用的数据库结构：

（Pupils 中的 classId 与学校的唯一班级相关联）

一个帐户可以具有多个功能，也可以在多个学校具有相同功能，例如：一位教师可以是多所学校的教师，同时也是另一所学校的家长。 （AccountsTypes 是 Accounts 和 Types 之间的多对多关系）

因此，插入需要检查该帐户是否已经存在，如果不存在，则添加它，否则什么也不做，继续检查它是否已经是这所学校的学生，如果不是, 添加它，否则什么都不做等等。

我想到了一个解决方案，只需添加所有内容，如果它已经存在，则无需先检查就对其进行更新，但我找不到任何有关如何执行此操作的信息。

那么我该如何优化它，这样它就不会花费很长时间，也不必检查它是否已经存在，并且可能将其插入到一个 addAsync 中，以便将其添加到所有相关表中？

这也需要非常可靠，以免失败。

Answer 1

EF 本身并不是为批量操作而设计的，一切都应该通过ChangeTracker。

我建议使用linq2db.EntityFrameworkCore（请注意，我是创建者之一）。

库支持临时表，并且它是 onw LINQ 转换器，这使得 linq2db 对于 ETL 任务非常强大。

public async Task<List<FullPupil>> AddPupilAccountsBySchoolId(List<FullPupil> pupils, string schoolId, string password, string hashedPassword, Types type) 
{
    using var db = _context.CreateLinqToDBConnection();

    // Preprocess data for temporary table
    var pupilsProjection = pupils.Select(pupil => new 
    {
        pupil.Email,
        pupil.Name,
        pupil.SurName,
        ClassId = Guid.Parse(pupil.Class[0].Id),
        CodeId = Guid.Parse(pupil.Code[0].Id),
    });

    // here we create temporary table and execute BulkCopy for inserting items into database
    using var tempPupil = await db.CreateTempTableAsync(pupilsProjection, tableName: "TempPupils");

    var missingAccounts = 
        from t in tempPupil
        join a in _context.Accounts on t.Email equals a.Email into gj
        from a in gj.DefaultIfEmpty()
        where a == null
        select t;

    // Inserting missed accounts
    await missingAccounts.InsertAsync(_context.Accounts.ToLinqToDBTable(), pupil =>
        new Accounts 
        {
            Email = pupil.Email,
            FirstTime = true,
            FirstTimePassword = password,
            Password = hashedPassword,
            Name = pupil.Name,
            SurName = pupil.SurName,
            EmailSend = false,
        }
    );

    // Helper query for Acount <-> Pupil association
    var pupilWithAccount =
        from t in tempPupil
        join a in _context.Accounts on t.Email equals t.Email
        select new 
        {
            Pupil = t,
            Account = a
        };

    var missingAccountTypes = 
        from pa in in pupilWithAccount
        join at in _context.AccountsTypes on new { pa.Account.AccountId, pa.Account.TypeId } equals { at.AccountId, type.TypeId } into gj
        from at in gj.DefaultIfEmpty()
        where at == null
        select pa.Account;
    
    // Inserting missing Account Types
    await missingAccountTypes.InsertAsync(_context.AccountsTypes.ToLinqToDBTable(), 
        a => new AccountsTypes()
        {
            AccountId = a.AccountId,
            TypeId = type.TypeId,
        }
    );

    var missingPupils = 
        from pa in pupilWithAccount
        join p in _context.Pupils on pa.AccountId equals p.AccountId into gj
        from p in gj.DefaultIfEmpty()
        where p == null
        select pa;

    // Insering missing pupils
    await missingPupils.InsertAsync(_context.Pupils.ToLinqToDBTable(), 
        pa => new Pupil
        {
            AccountId = pa.Account.AccountId,
            ClassId = pa.Pupil.ClassId,
            CodeId = pa.Pupil.CodeId,
            ModifiedDate = Sql.CurrentTimestamp,
        }
    );

    // retrieve current account Id
    var newIds = await pupilWithAccount
        .Select(pa => new { paAccount.Account.AccountId, paAccount.Account.Email })
        .ToDictionaryAsyncLinqToDB(x => x.Email, x => x.AccountId);

    foreach (var pupil in pupils)
    {
        if (newIds.TryGetValue(pupil.Email, out var accountId))
            pupil.AccountId = accountId;
    }

    return pupils;
}

请注意，这个示例是我凭记忆写的，可能有错别字。