【发布时间】：2020-03-20 20:30:30
【问题描述】：
我通过一个链接来：https://github.com/hyee/OpenCSV，由于 setAsyncMode、RESULT_FETCH_SIZE，它大大提高了 JDBC ResultSet 到 CSV 的写入时间
//Extract ResultSet to CSV file, auto-compress if the fileName extension is ".zip" or ".gz"
//Returns number of records extracted
public int ResultSet2CSV(final ResultSet rs, final String fileName, final String header, final boolean aync) throws Exception {
try (CSVWriter writer = new CSVWriter(fileName)) {
//Define fetch size(default as 30000 rows), higher to be faster performance but takes more memory
ResultSetHelperService.RESULT_FETCH_SIZE=10000;
//Define MAX extract rows, -1 means unlimited.
ResultSetHelperService.MAX_FETCH_ROWS=20000;
writer.setAsyncMode(aync);
int result = writer.writeAll(rs, true);
return result - 1;
}
}
但问题是我不知道如何将上述合并到我的要求中。由于该链接涉及许多其他类，因此我不确定它们的作用以及是否需要它来满足我的要求。尽管如此，我还是尝试过，但每当我启用 2 条注释行代码时，它都无法编译。下面是我的代码。
任何关于我如何实现这一点的帮助将不胜感激。
package test;
import java.io.BufferedWriter;
import java.io.FileWriter;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.Statement;
import java.util.Date;
import com.opencsv.CSVWriter;
import com.opencsv.ResultSetHelperService;
public class OpenCSVTest1
{
static Connection con =null;
static Statement stmt = null;
static ResultSet rs = null;
public static void main(String args[]) throws Exception
{
connection ();
retrieveData(con);
}
private static void connection() throws Exception
{
try
{
Class.forName("<jdbcdriver>");
con = DriverManager.getConnection("jdbc:","<username>","<pass>");
System.out.println("Connection successful");
}
catch (Exception e)
{
System.out.println("Exception while establishing sql connection");
throw e;
}
}
private static void retrieveData(Connection con) throws Exception
{
try
{
stmt=con.createStatement();
stmt = con.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
String query = "SELECT * FROM dbo.tablename";
rs=stmt.executeQuery(query);
CSVWriter writer = new CSVWriter(new BufferedWriter(new FileWriter("C:\\Data\\File1.csv")));
ResultSetHelperService service = new ResultSetHelperService();
/*** ResultSetHelperService.RESULT_FETCH_SIZE=10000; ***/ // to add
service.setDateTimeFormat("yyyy-MM-dd HH:mm:ss.SSS");
System.out.println("**** Started writing Data to CSV **** " + new Date());
writer.setResultService(service);
/*** writer.setAsyncMode(aync); ***/ // to add
int lines = writer.writeAll(rs, true, true, false);
writer.flush();
writer.close();
System.out.println("** OpenCSV -Completed writing the resultSet at " + new Date() + " Number of lines written to the file " + lines);
}
catch (Exception e)
{
System.out.println("Exception while retrieving data" );
e.printStackTrace();
throw e;
}
finally
{
rs.close();
stmt.close();
con.close();
}
}
}
更新
我已经更新了我的代码。现在代码正在使用 writeAll 方法一次在 CSV 中写入完整的结果集，这会导致时间消耗。
现在我要做的是将结果集批量写入 CSV，因为结果集的第一列将始终动态通过 SELECT 查询自动增量列 (Sqno) 生成，值为 (1,2,3 ..) 所以我不确定如何读取结果集的第一列并将其拆分以写入 CSV。可能是 HashMap 可能会有所帮助，所以如果需要，我还添加了结果集到哈希图的转换代码。
import com.opencsv.CSVWriter;
import com.opencsv.ResultSetHelperService;
import java.io.BufferedWriter;
import java.io.FileWriter;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.ResultSetMetaData;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
public class OpenCSVTest1
{
static int fetchlimit_src = 100;
static Connection con =null;
static Statement stmt = null;
static ResultSet rs = null;
static String filename = "C:\\Data\\filename.csv";
static CSVWriter writer;
public static void main(String args[])
{
try
{
connection();
retrieveData(con);
}
catch(Exception e)
{
System.out.println(e);
}
}
private static void connection() throws Exception
{
try
{
Class.forName("<jdbcdriver>");
con = DriverManager.getConnection("jdbc:","<username>","<pass>");
System.out.println("Connection successful");
}
catch (Exception e)
{
System.out.println("Exception while establishing sql connection");
throw e;
}
}
private static void retrieveData(Connection con) throws Exception
{
try
{
stmt=con.createStatement();
String query = "SELECT ROWNUM AS Sqno, * FROM dbo.tablename "; // Oracle
// String query = "SELECT ROW_NUMBER() OVER(ORDER BY Id ASC) AS Sqno, * FROM dbo.tablename "; // SQLServer
System.out.println(query);
stmt = con.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
stmt.setFetchSize(fetchlimit_src);
System.out.println("**** Started querying src **** " + new Date());
rs=stmt.executeQuery(query);
System.out.println("**** Completing querying src **** " + new Date());
// resultset_List(rs); // If required store resultset(rs) to HashMap
writetoCSV(rs,filename);
/** How to write resultset to CSV in batches instead of writing all at once to speed up write performance ?
* Hint: resultset first column is Autoincrement [Sqno] (1,2,3...) which might help to split result in batches.
*
**/
}
catch (Exception e)
{
System.out.println("Exception while retrieving data" );
e.printStackTrace();
throw e;
}
finally
{
rs.close();
stmt.close();
con.close();
}
}
private static List<Map<String, Object>> resultset_List(ResultSet rs) throws SQLException
{
ResultSetMetaData md = rs.getMetaData();
int columns = md.getColumnCount();
List<Map<String, Object>> rows = new ArrayList<Map<String, Object>>();
while (rs.next())
{
Map<String, Object> row = new HashMap<String, Object>(columns);
for(int i = 1; i <= columns; ++i)
{
row.put(md.getColumnName(i), rs.getObject(i));
}
rows.add(row);
}
// System.out.println(rows.toString());
return rows;
}
private static void writetoCSV(ResultSet rs, String filename) throws Exception
{
try
{
writer = new CSVWriter(new BufferedWriter(new FileWriter(filename)));
ResultSetHelperService service = new ResultSetHelperService();
service.setDateTimeFormat("yyyy-MM-dd HH:mm:ss.SSS");
long batchlimit = 1000;
long Sqno = 1;
ResultSetMetaData rsmd = rs.getMetaData();
String columnname = rsmd.getColumnLabel(1); // To retrieve columns with labels (for example SELECT ROWNUM AS Sqno)
System.out.println("**** Started writing Data to CSV **** " + new Date());
writer.setResultService(service);
int lines = writer.writeAll(rs, true, true, false);
System.out.println("** OpenCSV -Completed writing the resultSet at " + new Date() + " Number of lines written to the file " + lines);
}
catch (Exception e)
{
System.out.println("Exception while writing data" );
e.printStackTrace();
throw e;
}
finally
{
writer.flush();
writer.close();
}
}
}
【问题讨论】：