Thread: [Csvtosql-cvs] csvtosql_jdk50/src/net/sf/csv2sql/writers AbstractWriter.java,NONE,1.1 JdbcBatchWrite
Brought to you by:
davideconsonni
|
From: consonni d. <dav...@us...> - 2005-04-25 10:53:35
|
Update of /cvsroot/csvtosql/csvtosql_jdk50/src/net/sf/csv2sql/writers In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv15138/net/sf/csv2sql/writers Added Files: AbstractWriter.java JdbcBatchWriter.java JdbcWriter.java SqlFileWriter.java SqlMultipleFileWriter.java StandardOutputWriter.java ThreadedAbstractWriter.java ThreadedStandardOutputWriter.java Log Message: no message --- NEW FILE: StandardOutputWriter.java --- /* Copyright (C) 2004 Davide Consonni <dco...@en...> This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ package net.sf.csv2sql.writers; import java.util.HashMap; import net.sf.csv2sql.storage.exceptions.StorageException; import net.sf.csv2sql.writers.exceptions.WriterException; /** * Write generated statements to standard output using * <code>System.out.println();</code> method. * @see AbstractWriter AbstractWriter * @author <a href="mailto:dco...@en...">Davide Consonni</a> */ public class StandardOutputWriter extends AbstractWriter { /** * @see AbstractWriter#write */ public void write() throws WriterException { try { for (int i=0; i<getStorage().size(); i++) { System.out.println(getStorage().get(i)); } } catch (StorageException e) { throw new WriterException("cannot read data from temporary storage", e); } } public HashMap<String, String> requiredParameterList() { return null; } public HashMap<String, String> optionalParameterList() { return null; } } --- NEW FILE: SqlFileWriter.java --- /* Copyright (C) 2004 Davide Consonni <dco...@en...> This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ package net.sf.csv2sql.writers; import java.io.BufferedWriter; import java.io.File; import java.io.FileWriter; import java.io.IOException; import java.util.HashMap; import net.sf.csv2sql.storage.exceptions.StorageException; import net.sf.csv2sql.writers.exceptions.WriterException; /** * Write a new textfile with generated statements. * @see AbstractWriter AbstractWriter * @author <a href="mailto:dco...@en...">Davide Consonni</a> */ public class SqlFileWriter extends AbstractWriter { /** * @see AbstractWriter#write */ public void write() throws WriterException { try { File out = new File(getWriterProperties().getProperty("filename")); BufferedWriter output = new BufferedWriter(new FileWriter(out)); for (int i=0; i<getStorage().size(); i++) { output.write(getStorage().get(i)+"\n" ); } output.close(); } catch (StorageException e) { throw new WriterException("cannot read data from temporary storage", e); } catch (IOException e) { throw new WriterException("cannot write statement to file", e); } } protected HashMap<String, String> requiredParameterList() { HashMap<String, String> hm = new HashMap<String, String>(); hm.put("filename","output filename where result will be stored."); return hm; } protected HashMap<String, String> optionalParameterList() { return null; } } --- NEW FILE: JdbcWriter.java --- /* Copyright (C) 2004 Davide Consonni <dco...@en...> This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ package net.sf.csv2sql.writers; import java.io.File; import java.io.IOException; import java.sql.Connection; import java.sql.SQLException; import java.sql.Statement; import java.util.HashMap; import net.sf.csv2sql.storage.exceptions.StorageException; import net.sf.csv2sql.utils.DBUtils; import net.sf.csv2sql.writers.exceptions.InvalidParameterValueException; import net.sf.csv2sql.writers.exceptions.WriterException; /** * Load generated statements in a jdbc resource. * @see AbstractWriter AbstractWriter * @author <a href="mailto:dco...@en...">Davide Consonni</a> */ public class JdbcWriter extends AbstractWriter { //mandatory parameters String driver; String url; String username; String password; //optional parameters File jdbcJar = null; boolean commit; private void init() throws InvalidParameterValueException { driver = getWriterProperties().getProperty("driver"); url = getWriterProperties().getProperty("url"); username = getWriterProperties().getProperty("username"); password = getWriterProperties().getProperty("password"); //optional set autocommit commit = "true".equalsIgnoreCase(getWriterProperties().getProperty("commit")); //optional String strJdbcJar = getWriterProperties().getProperty("jdbcjar"); if (strJdbcJar!=null && !strJdbcJar.equals("")) { jdbcJar = new File(strJdbcJar); if (!jdbcJar.exists()) { throw new InvalidParameterValueException("jdbcjar", getWriterProperties().getProperty("jdbcjar")); } } } /** * @see AbstractWriter#write */ public void write() throws WriterException { Connection connection = null; Statement stmt = null; try { //get configuration options init(); if (jdbcJar!=null) { connection = DBUtils.openConnection(jdbcJar, driver, url, username, password); } else { connection = DBUtils.openConnection(driver, url, username, password); } stmt = connection.createStatement(); //start transaction if (commit) {connection.setAutoCommit(false);} for (int i=0; i<getStorage().size(); i++) { stmt.executeUpdate(getStorage().get(i)); } //stop transaction if (commit) {connection.commit();} } catch (StorageException e) { throw new WriterException("cannot read data from temporary storage", e); } catch (SQLException e) { throw new WriterException("cannot write statement to database", e); } catch (IOException e) { throw new WriterException("cannot find database driver", e); } catch (ClassNotFoundException e) { throw new WriterException("cannot load database driver", e); } finally { DBUtils.closeConnection(connection); } } protected HashMap<String, String> requiredParameterList() { HashMap<String, String> hm = new HashMap<String, String>(); hm.put("driver","jdbc driver classname."); hm.put("url","jdbc database url."); hm.put("username","jdbc username."); return hm; } protected HashMap<String, String> optionalParameterList() { HashMap<String, String> hm = new HashMap<String, String>(); hm.put("password","jdbc password."); hm.put("commit","true or false. specify the use of transactions."); hm.put("jdbcjar","url of jdbc driver (will be load with a custom classloader)."); return hm; } } --- NEW FILE: AbstractWriter.java --- /* Copyright (C) 2004 Davide Consonni <dco...@en...> This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ package net.sf.csv2sql.writers; import java.util.List; import java.util.ArrayList; import java.util.HashMap; import java.util.Properties; import net.sf.csv2sql.writers.exceptions.MissingRequiredParameterException; import net.sf.csv2sql.writers.exceptions.WriterException; import net.sf.csv2sql.storage.Storage; /** * The writer classes write statements on a output resource like * file, or jdbc, xml, ecc.. * <p> * <b>Extend this class</b> to define a new writer, you must implement * the <code>write()</code> method, the argument of this method are the statement * generated from render classes. * <p> * Implement methods * <code>requiredParameterList</code> and <code>optionalParameterList</code> * to return the list of required and the optional parameters * <p> * Options passed to this class (and extended) can be read which * <code>getWriterProperties()</code> getter * @author <a href="mailto:dco...@en...">Davide Consonni</a> */ public abstract class AbstractWriter { private Properties properties; private Storage storage; /** * write list of statements. * @throws WriterException if data could not be written (for any reason) */ public abstract void write() throws WriterException; public Properties getWriterProperties() { return this.properties; } /** * init writers' parameters * @see MissingRequiredParameterException MissingRequiredParameterException * @throws MissingRequiredParameterException if required parameter is missing */ public void configure(Properties properties, Storage storage) throws MissingRequiredParameterException { this.properties = properties; this.storage = storage; //check for required properties Object[] required = getRequiredParameterList().keySet().toArray(); for (int i=0; i< required.length; i++) { String parameter = (required[i]).toString(); String configParameter = properties.getProperty(parameter); if (configParameter==null) { throw new MissingRequiredParameterException(parameter); } } } protected Storage getStorage() { return this.storage; } public HashMap<String, String> getRequiredParameterList() { HashMap<String, String> hm = new HashMap<String, String>(); hm.putAll((commonRequiredParameterList()==null ? new HashMap<String, String>() : commonRequiredParameterList())); hm.putAll((requiredParameterList()==null ? new HashMap<String, String>() : requiredParameterList())); return hm; } public HashMap<String, String> getOptionalParameterList() { HashMap<String, String> hm = new HashMap<String, String>(); hm.putAll((commonOptionalParameterList()==null ? new HashMap<String, String>() : commonOptionalParameterList())); hm.putAll((optionalParameterList()==null ? new HashMap<String, String>() : optionalParameterList())); return hm; } private HashMap<String, String> commonRequiredParameterList() { return null; } private HashMap<String, String> commonOptionalParameterList() { return null; } protected abstract HashMap<String, String> requiredParameterList(); protected abstract HashMap<String, String> optionalParameterList(); } --- NEW FILE: ThreadedAbstractWriter.java --- /* Copyright (C) 2004 Davide Consonni <dco...@en...> This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ package net.sf.csv2sql.writers; import net.sf.csv2sql.writers.exceptions.WriterException; /** * An AbstractWriter with thread support. <code>write</code> method * automatically start the thread * @see AbstractWriter AbstractWriter * @author <a href="mailto:dco...@en...">Davide Consonni</a> */ public abstract class ThreadedAbstractWriter extends AbstractWriter implements Runnable { float progress; /** * start thread with write list of statements. * @throws WriterException if data could not be written (for any reason) */ public void write() throws WriterException { progress=0; Thread AbstractWriterThread = null; AbstractWriterThread = new Thread(this, "AbstractWriterThread"); AbstractWriterThread.start(); } /** * get the current progress % for writing operation. */ public final float getProgress() { return this.progress; } /** * when progres is 100% the writing operations ends. */ public final boolean isDone() { return progress==100; } protected final void setProgress(float progress) { if (progress>100 || progress<0) { throw new IllegalArgumentException("progress must be between 0 and 100"); } this.progress = progress; } } --- NEW FILE: JdbcBatchWriter.java --- /* Copyright (C) 2004 Radovan Sninsky <sn...@po...> This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ package net.sf.csv2sql.writers; import java.io.File; import java.io.IOException; import java.sql.Connection; import java.sql.SQLException; import java.sql.Statement; import java.util.HashMap; import net.sf.csv2sql.storage.exceptions.StorageException; import net.sf.csv2sql.utils.DBUtils; import net.sf.csv2sql.writers.exceptions.InvalidParameterValueException; import net.sf.csv2sql.writers.exceptions.WriterException; /** * Load generated statements in a jdbc resource with batch method. * @author <a href="mailto:sn...@bg...">Radovan Sninsky</a> * @version $Revision: 1.22 $, $Date: 2005/03/12 11:29:22 $ * @since 1.1 */ public class JdbcBatchWriter extends AbstractWriter { //mandatory parameters String driver; String url; String username; String password; int batchCount; //optional parameters int commitBatchCount; File jdbcJar = null; private void init() throws InvalidParameterValueException { driver = getWriterProperties().getProperty("driver"); url = getWriterProperties().getProperty("url"); username = getWriterProperties().getProperty("username"); password = getWriterProperties().getProperty("password"); batchCount = Integer.parseInt(getWriterProperties().getProperty("batchcount", "10")); //optional commitBatchCount = Integer.parseInt(getWriterProperties().getProperty("commitbatchcount", "0")); //optional String strJdbcJar = getWriterProperties().getProperty("jdbcjar"); if (strJdbcJar!=null && !strJdbcJar.equals("")) { jdbcJar = new File(strJdbcJar); if (!jdbcJar.exists()) { throw new InvalidParameterValueException("jdbcjar", getWriterProperties().getProperty("jdbcjar")); } } } /** * @see AbstractWriter#write */ public void write() throws WriterException{ //get configuration options init(); String currentSQL = null; int sqlIndex = 0; try { Connection connection = null; if (jdbcJar!=null) { connection = DBUtils.openConnection(jdbcJar, driver, url, username, password); } else { connection = DBUtils.openConnection(driver, url, username, password); } try { Statement stmt = connection.createStatement(); try { //start transaction if (commitBatchCount != 1) { connection.setAutoCommit(false); } int batchSize = 0; int commitIdx = 0; for (int i=0; i<getStorage().size(); i++) { String sql = getStorage().get(i); stmt.addBatch(sql); sqlIndex++; currentSQL = sql; if (++batchSize == batchCount) { batchSize = 0; stmt.executeBatch(); if (commitBatchCount > 1 && commitBatchCount == ++commitIdx) { connection.commit(); commitIdx = 0; } stmt.clearBatch(); } } //take care of the remaining records. stmt.executeBatch(); stmt.clearBatch(); //stop transaction if (commitBatchCount != 1) { connection.commit(); } } finally { stmt.close(); } } finally { connection.close(); } } catch (StorageException e) { throw new WriterException("cannot read data from temporary storage", e); } catch (SQLException e) { throw new WriterException("cannot write statement to database: " + sqlIndex + "\n" + e.getMessage() + "\n" + currentSQL, e); } catch (IOException e) { throw new WriterException("cannot find database driver", e); } catch (ClassNotFoundException e) { throw new WriterException("cannot load database driver", e); } } protected HashMap<String, String> requiredParameterList() { HashMap<String, String> hm = new HashMap<String, String>(); hm.put("driver","jdbc driver classname."); hm.put("url","jdbc database url."); hm.put("username","jdbc username."); hm.put("commitbatchcount","commitbatchcount."); return hm; } protected HashMap<String, String> optionalParameterList() { HashMap<String, String> hm = new HashMap<String, String>(); hm.put("password","jdbc password."); hm.put("commit","true or false. specify the use of transactions."); hm.put("jdbcjar","url of jdbc driver (will be load with a custom classloader)."); hm.put("batchcount","batchcount."); return hm; } } --- NEW FILE: ThreadedStandardOutputWriter.java --- /* Copyright (C) 2004 Davide Consonni <dco...@en...> This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ package net.sf.csv2sql.writers; import java.util.HashMap; import net.sf.csv2sql.storage.exceptions.StorageException; import net.sf.csv2sql.writers.exceptions.ThreadedInvalidParameterValueException; /** * Write generated statements to standard output using * <code>System.out.println();</code> method. * @see ThreadedAbstractWriter ThreadedAbstractWriter * @author <a href="mailto:dco...@en...">Davide Consonni</a> */ public class ThreadedStandardOutputWriter extends ThreadedAbstractWriter { private float calculateStopAtProgress() { if (getWriterProperties().getProperty("stopatprogress")==null) {return 101;} float stopAt = 101; try { stopAt = Float.parseFloat(getWriterProperties().getProperty("stopatprogress")); if (stopAt<0||stopAt>100) { stopAt = 101; } } catch (Exception e) { throw new ThreadedInvalidParameterValueException("stopatprogress", getWriterProperties().getProperty("stopatprogress")); } return stopAt; } /** * @see ThreadedAbstractWriter#write */ public void run() { try { setProgress(0); float total= getStorage().size(); float stopAt = calculateStopAtProgress(); for (int i=0; i<getStorage().size(); i++) { if (getProgress()>=stopAt) break; System.out.println(getStorage().get(i));; setProgress((i/total)*100); if ("true".equalsIgnoreCase(getWriterProperties().getProperty("printprogress"))) { System.err.println(getProgress()); } } setProgress(100); } catch (StorageException e) { e.printStackTrace(); } } protected HashMap<String, String> requiredParameterList() { return null; } protected HashMap<String, String> optionalParameterList() { HashMap<String, String> hm = new HashMap<String, String>(); hm.put("printprogress","print progress to standard error stream (default false)."); hm.put("stopatprogress","stop printing at this %."); return hm; } } --- NEW FILE: SqlMultipleFileWriter.java --- /* Copyright (C) 2004 Davide Consonni <dco...@en...> This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ package net.sf.csv2sql.writers; import java.io.BufferedWriter; import java.io.File; import java.io.FileWriter; import java.io.IOException; import java.util.HashMap; import net.sf.csv2sql.storage.exceptions.StorageException; import net.sf.csv2sql.writers.exceptions.InvalidParameterValueException; import net.sf.csv2sql.writers.exceptions.WriterException; /** * Write a new textfile with generated statements. * @see AbstractWriter AbstractWriter * @author <a href="mailto:dco...@en...">Davide Consonni</a> */ public class SqlMultipleFileWriter extends AbstractWriter { public static String formatSequence(int currentPage, int totalPages) { StringBuffer sb = new StringBuffer(); int zeroToAdd = (String.valueOf(totalPages)).length() - (String.valueOf(currentPage)).length(); for (int i=0; i<zeroToAdd; i++) {sb.append("0");} sb.append(String.valueOf(currentPage)); return sb.toString(); } private int calculateBreakLines() throws InvalidParameterValueException { int breaklines=10; String bl = getWriterProperties().getProperty("breaklines"); breaklines = (bl!=null) ? Integer.parseInt(bl) : 10; if (breaklines <= 0) { throw new InvalidParameterValueException( "breaklines", getWriterProperties().getProperty("breaklines") ); } return breaklines; } /** * @see AbstractWriter#write */ public void write() throws WriterException { try { int breaklines=calculateBreakLines(); int sequence=0; float pages = getStorage().size()/breaklines; if (pages<1) { //only one page File out = new File(getWriterProperties().getProperty("filename")); BufferedWriter output = new BufferedWriter(new FileWriter(out)); for (int i=0; i<getStorage().size(); i++) { output.write(getStorage().get(i)+"\n" ); } output.close(); } else { //more than one page //first n pages for (int i=1; i<=pages; i++) { File out = new File(getWriterProperties().getProperty("filename")+ "_"+ formatSequence(i, (int)pages)); BufferedWriter output = new BufferedWriter(new FileWriter(out)); int from = breaklines*(i-1); int to = (breaklines*(i)); for (int j=from; j<to; j++) { sequence++; output.write(getStorage().get(j) +"\n" ); } output.flush(); output.close(); } //last if exist if (pages!=1) { File out = new File(getWriterProperties().getProperty("filename")+ "_"+ formatSequence((int)pages, (int)pages)); BufferedWriter output = new BufferedWriter(new FileWriter(out)); for (int i=sequence; i<getStorage().size(); i++) { sequence++; output.write(getStorage().get(i) +"\n" ); } output.flush(); output.close(); } } } catch (StorageException e) { throw new WriterException("cannot read data from temporary storage", e); } catch (IOException e) { throw new WriterException("cannot write statement to file", e); } } protected HashMap<String, String> requiredParameterList() { HashMap<String, String> hm = new HashMap<String, String>(); hm.put("filename","output filename where result will be stored."); return hm; } protected HashMap<String, String> optionalParameterList() { HashMap<String, String> hm = new HashMap<String, String>(); hm.put("breaklines","number of lines before break the csvfile 10 if not specified."); return hm; } } |