【发布时间】：2011-02-11 13:56:22
【问题描述】：
我有问题。
我正在开发一个项目，但我被困在这部分：
我想从文本文件加载数据并将其存储到数据库访问 事情是每个文本文件中的数据大约 12.000 行数据 每个文本文件大约需要 10 分钟来处理它..
注意：在存储数据之前，我将每一行数据与文本文件分开并将其放入字符串中，然后检查数据是否已经在数据库中。 如果在数据库内部我更新它。如果没有，那么我使用插入语句..
我正在使用 C# 来开发这个程序？ 有没有最快的方法来加载和存储这些数据？
更新：
这是我的代码，希望它有助于理解我的问题：
using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Data;
using System.Drawing;
using System.Text;
using System.Windows.Forms;
using System.IO;
using System.Collections;
using System.Data.OleDb;
namespace DAF
{
public partial class FrontForm : Form
{
public Boolean status;
public FrontForm()
{
InitializeComponent();
//define location of the database
string connection = @"Provider=Microsoft.Jet.OLEDB.4.0;Data Source=C:\Users\PC\Desktop\Graduation Project\Research\DAF\Data\DAFLogisticDepartment.mdb";
//define location of the text file data
DirectoryInfo di = new DirectoryInfo(@"C:\Users\PC\Desktop\Graduation Project\Research\DAF\Data\");
FileInfo[] fiarr = di.GetFiles("*.txt");
//define connection to database
OleDbConnection con = new OleDbConnection(connection);
String query;
OleDbDataReader rdr = null;
con.Open();
//get all table from database
OleDbCommand cmd = con.CreateCommand();
DataTable dt = con.GetSchema("tables");
DataRow[] dttable = dt.Select();
con.Close();
//read each new textfile inside the folder
foreach (FileInfo fri in fiarr)
{
StreamReader sr = new StreamReader(fri.FullName, System.Text.Encoding.Default);
String line;
String tabledbs, dbsName;
while ((line = sr.ReadLine()) != null)
{
String VRSD, locationID, truckID, yearIn, yearOut, weekIn, weekOut, dayIn, dayOut, timeIn, timeOut, route;
int plantID;
//process each line of data and put into each variable
VRSD = line.Substring(0, 4).Trim();
plantID = Convert.ToInt32(line.Substring(4, 1).Trim());
locationID = line.Substring(5, 4).Trim();
truckID = line.Substring(24, 5).Trim();
yearIn = line.Substring(32, 4).Trim();
weekIn = line.Substring(36, 2).Trim();
dayIn = line.Substring(38, 1).Trim();
timeIn = line.Substring(39, 8).Trim();
yearOut = line.Substring(47, 4).Trim();
weekOut = line.Substring(51, 2).Trim();
dayOut = line.Substring(53, 1).Trim();
timeOut = line.Substring(54, 8).Trim();
route = line.Substring(64, 2).Trim();
//make database name
dbsName = plantID + locationID;
con.Open();
//check if the table exist in database
for (int i = 0; i < dttable.Length - 9; i++)
{
tabledbs = dttable[i]["TABLE_NAME"].ToString();
ArrayList indexlist = new ArrayList();
if (tabledbs == dbsName)
{
//if the table exist, status = true
status = true;
break;
}
}
con.Close();
con.Open();
if (status == true)
{
try
{
//if the data not in the system, insert statement
query = @"insert into " + plantID + locationID + " values('" + VRSD.ToString() + "'," + plantID + ",'" + locationID + "','" + truckID + "','" + yearIn + "','" + weekIn + "','" + dayIn + "','" + timeIn + "','" + yearOut + "','" + weekOut + "','" + dayOut + "','" + timeOut + "')";
cmd = new OleDbCommand(query, con);
rdr = cmd.ExecuteReader();
con.Close();
}
catch
{
//if the data in the system, update statement
query = @"update " + dbsName + " set YearIn='" + yearIn + "', YearOut='" + yearOut + "', WeekIn='" + weekIn + "', WeekOut='" + weekOut + "', DayIn='" + dayIn + "', DayOut='" + dayOut + "', TimeIn='" + timeIn + "', TimeOut='" + timeOut + "' where LocationID='" + locationID + "' and PlantID=" + plantID + "";
cmd = new OleDbCommand(query, con);
rdr = cmd.ExecuteReader();
con.Close();
}
}
else
{
//create new table
string attribute = "VRSD String,PlantID Integer, LocationID String, TruckID String," +
"YearIn String, WeekIn String, DayIn String, TimeIn String," +
"YearOut String, WeekOut String, DayOut String, TimeOut String";
query = @"CREATE TABLE " + plantID + locationID + "(" + attribute + ")";
cmd = new OleDbCommand(query, con);
cmd.ExecuteNonQuery();
//insert the data
query = @"insert into " + plantID + locationID + " values('" + VRSD.ToString() + "'," + plantID + ",'" + locationID + "','" + truckID + "','" + yearIn + "','" + weekIn + "','" + dayIn + "','" + timeIn + "','" + yearOut + "','" + weekOut + "','" + dayOut + "','" + timeOut + "')";
cmd = new OleDbCommand(query, con);
rdr = cmd.ExecuteReader();
con.Close();
}
status = false;
}
sr.Close();
//after the text file load into database, the text file moved to history folder
MessageBox.Show(fri.FullName.ToString(), "File Manager", MessageBoxButtons.OK);
fri.MoveTo(@"C:\Users\PC\Desktop\Graduation Project\Research\DAF\Data\History\" + fri.Name.ToString() + ".txt");
}
}
private void button2_Click(object sender, EventArgs e)
{
StandardReport sr = new StandardReport();
sr.Show();
}
private void FrontForm_Load(object sender, EventArgs e)
{
}
}
}
【问题讨论】：
-
10分钟有这么糟糕吗？您需要执行多少文件以及多久执行一次？你能并行化你正在做的任务吗？您是否对此进行了概要分析以查看时间被占用的地方？
-
事情是这样的：我的老板每周要加载 6 个文本文件。每个文本文件包含超过 12.000 行数据。每次我的老板运行程序时，程序都会加载每个文本文件，检查数据库的每一行数据是否存在。如果存在，它会更新它，如果不存在，它会插入到 dbs.. 每个文本文件大约需要 10 分钟。如果有 6 个文本文件，则需要 1 小时..
-
我添加了我的代码.. 希望你能帮助我.. :)
-
@Yohanes 还有一个问题，您希望插入的文本文件的格式是什么？少量数据样本会有所帮助。
-
您有 12k 行，x6...您希望其中大部分是新行还是现有行？
标签： c# ms-access import text-files