#11 error while loading to mysql

closed-out-of-date
John Didion
None
7
2009-02-17
2003-12-12
Anonymous
No

Hi,

It's nice that you've added the database loading
functionality in this version of jwnl, but I've had some
problems with it.

My most recent one is (versions 1.3rc1, 1.3rc2, 1.3rc3):
I almost get the whole thing loaded when I get this
exception:

INFO: done inserting data for pos [POS: adverb]
Exception in thread "main"
java.util.ConcurrentModificationException
at java.util.HashMap$HashIterator.nextEntry
(HashMap.java:782)
at java.util.HashMap$KeyIterator.next
(HashMap.java:818)
at
com.mysql.jdbc.Connection.closeAllOpenStatements
(Connection.java:2538
)
at com.mysql.jdbc.Connection.close
(Connection.java:1067)
at
net.didion.jwnl.utilities.DictionaryToDatabase.main
(DictionaryToDatab
ase.java:71)

There is no source for your DictionaryToDatab
ase class so I can't do anything about it. What a pity.
What shall I do?

Hope you can fix it.

Best,

Piotr

Discussion

  • John Didion
    John Didion
    2004-01-13

    • priority: 5 --> 7
    • assigned_to: nobody --> jdidion
     
  • Logged In: NO

    The source code is.

    // Source File Name: DictionaryToDatabase.java

    import java.io.*;
    import java.sql.*;
    import java.util.*;
    import net.didion.jwnl.JWNL;
    import net.didion.jwnl.data.*;
    import net.didion.jwnl.dictionary.AbstractCachingDictionary;
    import net.didion.jwnl.dictionary.Dictionary;
    import net.didion.jwnl.dictionary.database.ConnectionManager;
    import net.didion.jwnl.util.MessageLog;
    import net.didion.jwnl.util.MessageLogLevel;

    public class DictionaryToDatabase
    {

    public static void main(String args[])
    {
    if(args.length < 4)
    {
    System.out.println("java
    net.didion.jwnl.utilities.DictionaryToDatabase <property
    file> <create tables script> <driver class> <connection url>
    [username [password]]");
    System.exit(-1);
    }
    try
    {
    JWNL.initialize(new FileInputStream(args[0]));
    }
    catch(Exception ex)
    {
    ex.printStackTrace();
    System.exit(-1);
    }
    Connection conn = null;
    try
    {
    String createTablesScript = args[1];
    ConnectionManager mgr = new
    ConnectionManager(args[2], args[3], args.length <= 4 ? null
    : args[4], args.length <= 5 ? null : args[5]);
    conn = mgr.getConnection();
    DictionaryToDatabase d2d = new
    DictionaryToDatabase(conn);
    d2d.createTables(createTablesScript);
    d2d.insertData();
    }
    catch(Exception e)
    {
    e.printStackTrace();
    System.exit(-1);
    }
    finally
    {
    if(conn != null)
    try
    {
    conn.close();
    }
    catch(SQLException ex) { }
    }
    }

    private static synchronized int nextId()
    {
    _id++;
    if(_log.isLevelEnabled(MessageLogLevel.DEBUG) && _id
    % 1000 == 0)
    {
    long temp = System.currentTimeMillis();
    _log.log(MessageLogLevel.DEBUG, "inserted " +
    _id + "th entry");
    _log.log(MessageLogLevel.DEBUG, "free memory: "
    + Runtime.getRuntime().freeMemory());
    _log.log(MessageLogLevel.DEBUG, "time: " + (temp
    - _time));
    _time = System.currentTimeMillis();
    }
    return _id;
    }

    public DictionaryToDatabase(Connection conn)
    {
    _iwIdToSynsetOffset = new HashMap();
    _synsetOffsetToId = new HashMap();
    _conn = conn;

    ((AbstractCachingDictionary)Dictionary.getInstance()).setCachingEnabled(false);
    }

    public void createTables(String script)
    throws IOException, SQLException
    {
    _log.log(MessageLogLevel.INFO, "creating tables");
    BufferedReader reader = new BufferedReader(new
    InputStreamReader(new FileInputStream(script)));
    StringBuffer buf = new StringBuffer();
    for(String line = reader.readLine(); line != null;
    line = reader.readLine())
    {
    line = line.trim();
    if(line.length() <= 0)
    continue;
    buf.append(line);
    if(line.endsWith(";"))
    {
    System.out.println(buf.toString());

    _conn.prepareStatement(buf.toString()).execute();
    buf = new StringBuffer();
    } else
    {
    buf.append(" ");
    }
    }

    _log.log(MessageLogLevel.INFO, "creating tables");
    }

    public void insertData()
    throws Exception
    {
    _time = System.currentTimeMillis();
    POS pos;
    for(Iterator posItr = POS.getAllPOS().iterator();
    posItr.hasNext(); _log.log(MessageLogLevel.INFO, "done
    inserting data for pos " + pos))
    {
    pos = (POS)posItr.next();
    _log.log(MessageLogLevel.INFO, "inserting data
    for pos " + pos);

    storeIndexWords(Dictionary.getInstance().getIndexWordIterator(pos));

    storeSynsets(Dictionary.getInstance().getSynsetIterator(pos));
    storeIndexWordSynsets();

    storeExceptions(Dictionary.getInstance().getExceptionIterator(pos));
    _iwIdToSynsetOffset.clear();
    _synsetOffsetToId.clear();
    }

    }

    private void storeIndexWords(Iterator itr)
    throws SQLException
    {
    _log.log(MessageLogLevel.INFO, "storing index words");
    PreparedStatement iwStmt =
    _conn.prepareStatement("INSERT INTO IndexWord VALUES(?,?,?)");
    int count = 0;
    do
    {
    if(!itr.hasNext())
    break;
    IndexWord iw = (IndexWord)itr.next();
    int id = nextId();
    iwStmt.setInt(1, id);
    iwStmt.setString(2, iw.getLemma());
    iwStmt.setString(3, iw.getPOS().getKey());
    iwStmt.execute();
    _iwIdToSynsetOffset.put(new Integer(id),
    iw.getSynsetOffsets());
    if(count++ % 100 == 0)
    System.out.println(count);
    } while(true);
    }

    private void storeSynsets(Iterator itr)
    throws SQLException
    {
    PreparedStatement synsetStmt =
    _conn.prepareStatement("INSERT INTO Synset VALUES(?,?,?,?,?)");
    PreparedStatement synsetWordStmt =
    _conn.prepareStatement("INSERT INTO SynsetWord
    VALUES(?,?,?,?)");
    PreparedStatement synsetPointerStmt =
    _conn.prepareStatement("INSERT INTO SynsetPointer
    VALUES(?,?,?,?,?,?,?)");
    PreparedStatement synsetVerbFrameStmt =
    _conn.prepareStatement("INSERT INTO SynsetVerbFrame
    VALUES(?,?,?,?)");
    _log.log(MessageLogLevel.INFO, "storing synsets");
    while(itr.hasNext())
    {
    Synset synset = (Synset)itr.next();
    int id = nextId();
    _synsetOffsetToId.put(new
    Long(synset.getOffset()), new Integer(id));
    synsetStmt.setInt(1, id);
    synsetStmt.setLong(2, synset.getOffset());
    synsetStmt.setString(3, synset.getPOS().getKey());
    synsetStmt.setBoolean(4,
    synset.isAdjectiveCluster());
    synsetStmt.setString(5, synset.getGloss());
    synsetStmt.execute();
    Word words[] = synset.getWords();
    synsetWordStmt.setInt(2, id);
    synsetVerbFrameStmt.setInt(2, id);
    for(int i = 0; i < words.length; i++)
    {
    synsetWordStmt.setInt(1, nextId());
    synsetWordStmt.setString(3,
    words[i].getLemma());
    synsetWordStmt.setInt(4, words[i].getIndex());
    synsetWordStmt.execute();
    if(!(words[i] instanceof Verb))
    continue;
    synsetVerbFrameStmt.setInt(4,
    words[i].getIndex());
    int flags[] =
    ((Verb)words[i]).getVerbFrameIndicies();
    for(int j = 0; j < flags.length; j++)
    {
    synsetVerbFrameStmt.setInt(1, nextId());
    synsetVerbFrameStmt.setInt(3, flags[j]);
    synsetVerbFrameStmt.execute();
    }

    }

    Pointer pointers[] = synset.getPointers();
    synsetPointerStmt.setInt(2, id);
    int i = 0;
    while(i < pointers.length)
    {
    synsetPointerStmt.setInt(1, nextId());
    synsetPointerStmt.setString(3,
    pointers[i].getType().getKey());
    synsetPointerStmt.setLong(4,
    pointers[i].getTargetOffset());
    synsetPointerStmt.setString(5,
    pointers[i].getTargetPOS().getKey());
    synsetPointerStmt.setInt(6,
    pointers[i].getSourceIndex());
    synsetPointerStmt.setInt(7,
    pointers[i].getTargetIndex());
    synsetPointerStmt.execute();
    i++;
    }
    }
    }

    private void storeIndexWordSynsets()
    throws SQLException
    {
    _log.log(MessageLogLevel.INFO, "storing index word
    synsets");
    PreparedStatement iwsStmt =
    _conn.prepareStatement("INSERT INTO IndexWordSynset
    VALUES(?,?,?)");
    for(Iterator itr =
    _iwIdToSynsetOffset.entrySet().iterator(); itr.hasNext();)
    {
    java.util.Map.Entry entry =
    (java.util.Map.Entry)itr.next();
    int iwId = ((Integer)entry.getKey()).intValue();
    iwsStmt.setInt(2, iwId);
    long offsets[] = (long[])entry.getValue();
    int i = 0;
    while(i < offsets.length)
    {
    Integer offset =
    (Integer)_synsetOffsetToId.get(new Long(offsets[i]));
    int synsetId = offset.intValue();
    iwsStmt.setInt(1, nextId());
    iwsStmt.setLong(3, synsetId);
    iwsStmt.execute();
    i++;
    }
    }

    }

    private void storeExceptions(Iterator itr)
    throws SQLException
    {
    _log.log(MessageLogLevel.INFO, "storing exceptions");
    PreparedStatement exStmt =
    _conn.prepareStatement("INSERT INTO Exception VALUES(?,?,?,?)");
    while(itr.hasNext())
    {
    Exc exc = (Exc)itr.next();
    exStmt.setString(4, exc.getLemma());
    Iterator excItr = exc.getExceptions().iterator();
    while(excItr.hasNext())
    {
    exStmt.setInt(1, nextId());
    exStmt.setString(2, exc.getPOS().getKey());
    exStmt.setString(3, (String)excItr.next());
    exStmt.execute();
    }
    }
    }

    private static final MessageLog _log;
    private static final String USAGE = "java
    net.didion.jwnl.utilities.DictionaryToDatabase <property
    file> <create tables script> <driver class> <connection url>
    [username [password]]";
    private static final String INSERT_INDEX_WORD = "INSERT
    INTO IndexWord VALUES(?,?,?)";
    private static final String INSERT_SYNSET = "INSERT INTO
    Synset VALUES(?,?,?,?,?)";
    private static final String INSERT_SYNSET_WORD = "INSERT
    INTO SynsetWord VALUES(?,?,?,?)";
    private static final String INSERT_SYNSET_POINTER =
    "INSERT INTO SynsetPointer VALUES(?,?,?,?,?,?,?)";
    private static final String INSERT_SYNSET_VERB_FRAME =
    "INSERT INTO SynsetVerbFrame VALUES(?,?,?,?)";
    private static final String INSERT_INDEX_WORD_SYNSET =
    "INSERT INTO IndexWordSynset VALUES(?,?,?)";
    private static final String INSERT_EXCEPTION = "INSERT
    INTO Exception VALUES(?,?,?,?)";
    private static int _id = 0;
    private static long _time = 0L;
    private Connection _conn;
    private Map _iwIdToSynsetOffset;
    private Map _synsetOffsetToId;

    static
    {
    _log = new MessageLog(DictionaryToDatabase.class);
    }
    }

     
  • Logged In: NO

    The source code is.

    // Source File Name: DictionaryToDatabase.java

    import java.io.*;
    import java.sql.*;
    import java.util.*;
    import net.didion.jwnl.JWNL;
    import net.didion.jwnl.data.*;
    import net.didion.jwnl.dictionary.AbstractCachingDictionary;
    import net.didion.jwnl.dictionary.Dictionary;
    import net.didion.jwnl.dictionary.database.ConnectionManager;
    import net.didion.jwnl.util.MessageLog;
    import net.didion.jwnl.util.MessageLogLevel;

    public class DictionaryToDatabase
    {

    public static void main(String args[])
    {
    if(args.length < 4)
    {
    System.out.println("java
    net.didion.jwnl.utilities.DictionaryToDatabase <property
    file> <create tables script> <driver class> <connection url>
    [username [password]]");
    System.exit(-1);
    }
    try
    {
    JWNL.initialize(new FileInputStream(args[0]));
    }
    catch(Exception ex)
    {
    ex.printStackTrace();
    System.exit(-1);
    }
    Connection conn = null;
    try
    {
    String createTablesScript = args[1];
    ConnectionManager mgr = new
    ConnectionManager(args[2], args[3], args.length <= 4 ? null
    : args[4], args.length <= 5 ? null : args[5]);
    conn = mgr.getConnection();
    DictionaryToDatabase d2d = new
    DictionaryToDatabase(conn);
    d2d.createTables(createTablesScript);
    d2d.insertData();
    }
    catch(Exception e)
    {
    e.printStackTrace();
    System.exit(-1);
    }
    finally
    {
    if(conn != null)
    try
    {
    conn.close();
    }
    catch(SQLException ex) { }
    }
    }

    private static synchronized int nextId()
    {
    _id++;
    if(_log.isLevelEnabled(MessageLogLevel.DEBUG) && _id
    % 1000 == 0)
    {
    long temp = System.currentTimeMillis();
    _log.log(MessageLogLevel.DEBUG, "inserted " +
    _id + "th entry");
    _log.log(MessageLogLevel.DEBUG, "free memory: "
    + Runtime.getRuntime().freeMemory());
    _log.log(MessageLogLevel.DEBUG, "time: " + (temp
    - _time));
    _time = System.currentTimeMillis();
    }
    return _id;
    }

    public DictionaryToDatabase(Connection conn)
    {
    _iwIdToSynsetOffset = new HashMap();
    _synsetOffsetToId = new HashMap();
    _conn = conn;

    ((AbstractCachingDictionary)Dictionary.getInstance()).setCachingEnabled(false);
    }

    public void createTables(String script)
    throws IOException, SQLException
    {
    _log.log(MessageLogLevel.INFO, "creating tables");
    BufferedReader reader = new BufferedReader(new
    InputStreamReader(new FileInputStream(script)));
    StringBuffer buf = new StringBuffer();
    for(String line = reader.readLine(); line != null;
    line = reader.readLine())
    {
    line = line.trim();
    if(line.length() <= 0)
    continue;
    buf.append(line);
    if(line.endsWith(";"))
    {
    System.out.println(buf.toString());

    _conn.prepareStatement(buf.toString()).execute();
    buf = new StringBuffer();
    } else
    {
    buf.append(" ");
    }
    }

    _log.log(MessageLogLevel.INFO, "creating tables");
    }

    public void insertData()
    throws Exception
    {
    _time = System.currentTimeMillis();
    POS pos;
    for(Iterator posItr = POS.getAllPOS().iterator();
    posItr.hasNext(); _log.log(MessageLogLevel.INFO, "done
    inserting data for pos " + pos))
    {
    pos = (POS)posItr.next();
    _log.log(MessageLogLevel.INFO, "inserting data
    for pos " + pos);

    storeIndexWords(Dictionary.getInstance().getIndexWordIterator(pos));

    storeSynsets(Dictionary.getInstance().getSynsetIterator(pos));
    storeIndexWordSynsets();

    storeExceptions(Dictionary.getInstance().getExceptionIterator(pos));
    _iwIdToSynsetOffset.clear();
    _synsetOffsetToId.clear();
    }

    }

    private void storeIndexWords(Iterator itr)
    throws SQLException
    {
    _log.log(MessageLogLevel.INFO, "storing index words");
    PreparedStatement iwStmt =
    _conn.prepareStatement("INSERT INTO IndexWord VALUES(?,?,?)");
    int count = 0;
    do
    {
    if(!itr.hasNext())
    break;
    IndexWord iw = (IndexWord)itr.next();
    int id = nextId();
    iwStmt.setInt(1, id);
    iwStmt.setString(2, iw.getLemma());
    iwStmt.setString(3, iw.getPOS().getKey());
    iwStmt.execute();
    _iwIdToSynsetOffset.put(new Integer(id),
    iw.getSynsetOffsets());
    if(count++ % 100 == 0)
    System.out.println(count);
    } while(true);
    }

    private void storeSynsets(Iterator itr)
    throws SQLException
    {
    PreparedStatement synsetStmt =
    _conn.prepareStatement("INSERT INTO Synset VALUES(?,?,?,?,?)");
    PreparedStatement synsetWordStmt =
    _conn.prepareStatement("INSERT INTO SynsetWord
    VALUES(?,?,?,?)");
    PreparedStatement synsetPointerStmt =
    _conn.prepareStatement("INSERT INTO SynsetPointer
    VALUES(?,?,?,?,?,?,?)");
    PreparedStatement synsetVerbFrameStmt =
    _conn.prepareStatement("INSERT INTO SynsetVerbFrame
    VALUES(?,?,?,?)");
    _log.log(MessageLogLevel.INFO, "storing synsets");
    while(itr.hasNext())
    {
    Synset synset = (Synset)itr.next();
    int id = nextId();
    _synsetOffsetToId.put(new
    Long(synset.getOffset()), new Integer(id));
    synsetStmt.setInt(1, id);
    synsetStmt.setLong(2, synset.getOffset());
    synsetStmt.setString(3, synset.getPOS().getKey());
    synsetStmt.setBoolean(4,
    synset.isAdjectiveCluster());
    synsetStmt.setString(5, synset.getGloss());
    synsetStmt.execute();
    Word words[] = synset.getWords();
    synsetWordStmt.setInt(2, id);
    synsetVerbFrameStmt.setInt(2, id);
    for(int i = 0; i < words.length; i++)
    {
    synsetWordStmt.setInt(1, nextId());
    synsetWordStmt.setString(3,
    words[i].getLemma());
    synsetWordStmt.setInt(4, words[i].getIndex());
    synsetWordStmt.execute();
    if(!(words[i] instanceof Verb))
    continue;
    synsetVerbFrameStmt.setInt(4,
    words[i].getIndex());
    int flags[] =
    ((Verb)words[i]).getVerbFrameIndicies();
    for(int j = 0; j < flags.length; j++)
    {
    synsetVerbFrameStmt.setInt(1, nextId());
    synsetVerbFrameStmt.setInt(3, flags[j]);
    synsetVerbFrameStmt.execute();
    }

    }

    Pointer pointers[] = synset.getPointers();
    synsetPointerStmt.setInt(2, id);
    int i = 0;
    while(i < pointers.length)
    {
    synsetPointerStmt.setInt(1, nextId());
    synsetPointerStmt.setString(3,
    pointers[i].getType().getKey());
    synsetPointerStmt.setLong(4,
    pointers[i].getTargetOffset());
    synsetPointerStmt.setString(5,
    pointers[i].getTargetPOS().getKey());
    synsetPointerStmt.setInt(6,
    pointers[i].getSourceIndex());
    synsetPointerStmt.setInt(7,
    pointers[i].getTargetIndex());
    synsetPointerStmt.execute();
    i++;
    }
    }
    }

    private void storeIndexWordSynsets()
    throws SQLException
    {
    _log.log(MessageLogLevel.INFO, "storing index word
    synsets");
    PreparedStatement iwsStmt =
    _conn.prepareStatement("INSERT INTO IndexWordSynset
    VALUES(?,?,?)");
    for(Iterator itr =
    _iwIdToSynsetOffset.entrySet().iterator(); itr.hasNext();)
    {
    java.util.Map.Entry entry =
    (java.util.Map.Entry)itr.next();
    int iwId = ((Integer)entry.getKey()).intValue();
    iwsStmt.setInt(2, iwId);
    long offsets[] = (long[])entry.getValue();
    int i = 0;
    while(i < offsets.length)
    {
    Integer offset =
    (Integer)_synsetOffsetToId.get(new Long(offsets[i]));
    int synsetId = offset.intValue();
    iwsStmt.setInt(1, nextId());
    iwsStmt.setLong(3, synsetId);
    iwsStmt.execute();
    i++;
    }
    }

    }

    private void storeExceptions(Iterator itr)
    throws SQLException
    {
    _log.log(MessageLogLevel.INFO, "storing exceptions");
    PreparedStatement exStmt =
    _conn.prepareStatement("INSERT INTO Exception VALUES(?,?,?,?)");
    while(itr.hasNext())
    {
    Exc exc = (Exc)itr.next();
    exStmt.setString(4, exc.getLemma());
    Iterator excItr = exc.getExceptions().iterator();
    while(excItr.hasNext())
    {
    exStmt.setInt(1, nextId());
    exStmt.setString(2, exc.getPOS().getKey());
    exStmt.setString(3, (String)excItr.next());
    exStmt.execute();
    }
    }
    }

    private static final MessageLog _log;
    private static final String USAGE = "java
    net.didion.jwnl.utilities.DictionaryToDatabase <property
    file> <create tables script> <driver class> <connection url>
    [username [password]]";
    private static final String INSERT_INDEX_WORD = "INSERT
    INTO IndexWord VALUES(?,?,?)";
    private static final String INSERT_SYNSET = "INSERT INTO
    Synset VALUES(?,?,?,?,?)";
    private static final String INSERT_SYNSET_WORD = "INSERT
    INTO SynsetWord VALUES(?,?,?,?)";
    private static final String INSERT_SYNSET_POINTER =
    "INSERT INTO SynsetPointer VALUES(?,?,?,?,?,?,?)";
    private static final String INSERT_SYNSET_VERB_FRAME =
    "INSERT INTO SynsetVerbFrame VALUES(?,?,?,?)";
    private static final String INSERT_INDEX_WORD_SYNSET =
    "INSERT INTO IndexWordSynset VALUES(?,?,?)";
    private static final String INSERT_EXCEPTION = "INSERT
    INTO Exception VALUES(?,?,?,?)";
    private static int _id = 0;
    private static long _time = 0L;
    private Connection _conn;
    private Map _iwIdToSynsetOffset;
    private Map _synsetOffsetToId;

    static
    {
    _log = new MessageLog(DictionaryToDatabase.class);
    }
    }

     
  • Brett Walenz
    Brett Walenz
    2009-02-17

    • status: open --> closed-out-of-date
     
  • Brett Walenz
    Brett Walenz
    2009-02-17

    Closing a really really old bug that somehow slipped through the cracks.