maxent-commit — Receive email whenever a new commit is made to the repository.

[Maxent-commit] CVS: maxent/src/main/java/opennlp/maxent GISModel.java, 1.1, 1.2

[Thomas M. <tsm...@us...>]

Update of /cvsroot/maxent/maxent/src/main/java/opennlp/maxent
In directory 23jxhf1.ch3.sourceforge.com:/tmp/cvs-serv5168/src/main/java/opennlp/maxent

Modified Files:
	GISModel.java 
Log Message:
Changed abstract model to better support creating full model for re-tagging.

Index: GISModel.java
===================================================================
RCS file: /cvsroot/maxent/maxent/src/main/java/opennlp/maxent/GISModel.java,v
retrieving revision 1.1
retrieving revision 1.2
diff -C2 -d -r1.1 -r1.2
*** GISModel.java	22 Jan 2009 23:23:34 -0000	1.1
--- GISModel.java	19 Mar 2009 13:04:31 -0000	1.2
***************
*** 61,65 ****
        super(params,predLabels,outcomeNames,correctionConstant,correctionParam);
        this.prior = prior;
!       prior.setLabels(ocNames, predLabels);
        modelType = ModelType.Maxent;
      }
--- 61,65 ----
        super(params,predLabels,outcomeNames,correctionConstant,correctionParam);
        this.prior = prior;
!       prior.setLabels(outcomeNames, predLabels);
        modelType = ModelType.Maxent;
      }

[Maxent-commit] CVS: maxent/src/main/java/opennlp/model AbstractModel.java, 1.3, 1.4

[Thomas M. <tsm...@us...>]

Update of /cvsroot/maxent/maxent/src/main/java/opennlp/model
In directory 23jxhf1.ch3.sourceforge.com:/tmp/cvs-serv5168/src/main/java/opennlp/model

Modified Files:
	AbstractModel.java 
Log Message:
Changed abstract model to better support creating full model for re-tagging.

Index: AbstractModel.java
===================================================================
RCS file: /cvsroot/maxent/maxent/src/main/java/opennlp/model/AbstractModel.java,v
retrieving revision 1.3
retrieving revision 1.4
diff -C2 -d -r1.3 -r1.4
*** AbstractModel.java	15 Mar 2009 03:11:44 -0000	1.3
--- AbstractModel.java	19 Mar 2009 13:04:31 -0000	1.4
***************
*** 24,31 ****
  public abstract class AbstractModel implements MaxentModel {
  
!   /** Maping between predicates/contexts and an integer representing them. */
    protected Map<String,Integer> pmap;
    /** The names of the outcomes. */
!   protected String[] ocNames;
    /** Parameters for the model. */
    protected EvalParameters evalParams;
--- 24,31 ----
  public abstract class AbstractModel implements MaxentModel {
  
!   /** Mapping between predicates/contexts and an integer representing them. */
    protected Map<String,Integer> pmap;
    /** The names of the outcomes. */
!   protected String[] outcomeNames;
    /** Parameters for the model. */
    protected EvalParameters evalParams;
***************
*** 37,49 ****
    /** The type of the model. */
    protected ModelType modelType;
  
    public AbstractModel(Context[] params, String[] predLabels, String[] outcomeNames) {
      init(predLabels,outcomeNames);
!     this.evalParams = new EvalParameters(params,ocNames.length);
    }
  
    public AbstractModel(Context[] params, String[] predLabels, String[] outcomeNames, int correctionConstant,double correctionParam) {
      init(predLabels,outcomeNames);
!     this.evalParams = new EvalParameters(params,correctionParam,correctionConstant,ocNames.length);
    }
    
--- 37,55 ----
    /** The type of the model. */
    protected ModelType modelType;
+   
+   public AbstractModel(Context[] params, String[] predLabels, Map<String,Integer> pmap, String[] outcomeNames) {
+     this.pmap = pmap;
+     this.outcomeNames =  outcomeNames;
+     this.evalParams = new EvalParameters(params,outcomeNames.length);
+   }
  
    public AbstractModel(Context[] params, String[] predLabels, String[] outcomeNames) {
      init(predLabels,outcomeNames);
!     this.evalParams = new EvalParameters(params,outcomeNames.length);
    }
  
    public AbstractModel(Context[] params, String[] predLabels, String[] outcomeNames, int correctionConstant,double correctionParam) {
      init(predLabels,outcomeNames);
!     this.evalParams = new EvalParameters(params,correctionParam,correctionConstant,outcomeNames.length);
    }
    
***************
*** 53,57 ****
        pmap.put(predLabels[i], i);
      }
!     this.ocNames =  outcomeNames;
    }
  
--- 59,63 ----
        pmap.put(predLabels[i], i);
      }
!     this.outcomeNames =  outcomeNames;
    }
  
***************
*** 69,73 ****
        for (int i = 1; i<ocs.length; i++)
            if (ocs[i] > ocs[best]) best = i;
!       return ocNames[best];
    }
    
--- 75,79 ----
        for (int i = 1; i<ocs.length; i++)
            if (ocs[i] > ocs[best]) best = i;
!       return outcomeNames[best];
    }
    
***************
*** 89,93 ****
     */
    public final String getAllOutcomes(double[] ocs) {
!       if (ocs.length != ocNames.length) {
            return "The double array sent as a parameter to GISModel.getAllOutcomes() must not have been produced by this model.";
        }
--- 95,99 ----
     */
    public final String getAllOutcomes(double[] ocs) {
!       if (ocs.length != outcomeNames.length) {
            return "The double array sent as a parameter to GISModel.getAllOutcomes() must not have been produced by this model.";
        }
***************
*** 95,101 ****
          DecimalFormat df =  new DecimalFormat("0.0000");
          StringBuffer sb = new StringBuffer(ocs.length*2);
!         sb.append(ocNames[0]).append("[").append(df.format(ocs[0])).append("]");
          for (int i = 1; i<ocs.length; i++) {
!           sb.append("  ").append(ocNames[i]).append("[").append(df.format(ocs[i])).append("]");
          }
          return sb.toString();
--- 101,107 ----
          DecimalFormat df =  new DecimalFormat("0.0000");
          StringBuffer sb = new StringBuffer(ocs.length*2);
!         sb.append(outcomeNames[0]).append("[").append(df.format(ocs[0])).append("]");
          for (int i = 1; i<ocs.length; i++) {
!           sb.append("  ").append(outcomeNames[i]).append("[").append(df.format(ocs[i])).append("]");
          }
          return sb.toString();
***************
*** 110,114 ****
     */
    public final String getOutcome(int i) {
!       return ocNames[i];
    }
  
--- 116,120 ----
     */
    public final String getOutcome(int i) {
!       return outcomeNames[i];
    }
  
***************
*** 122,127 ****
     **/
    public int getIndex(String outcome) {
!       for (int i=0; i<ocNames.length; i++) {
!           if (ocNames[i].equals(outcome))
                return i;
        }
--- 128,133 ----
     **/
    public int getIndex(String outcome) {
!       for (int i=0; i<outcomeNames.length; i++) {
!           if (outcomeNames[i].equals(outcome))
                return i;
        }
***************
*** 157,161 ****
        data[0] = evalParams.getParams();
        data[1] = pmap;
!       data[2] = ocNames;
        data[3] = new Integer((int)evalParams.getCorrectionConstant());
        data[4] = new Double(evalParams.getCorrectionParam());
--- 163,167 ----
        data[0] = evalParams.getParams();
        data[1] = pmap;
!       data[2] = outcomeNames;
        data[3] = new Integer((int)evalParams.getCorrectionConstant());
        data[4] = new Double(evalParams.getCorrectionParam());

[Maxent-commit] CVS: maxent/src/main/java/opennlp/perceptron PerceptronModel.java, 1.2, 1.3 SimplePerceptronSequenceTrainer.java, 1.1, 1.2

[Thomas M. <tsm...@us...>]

Update of /cvsroot/maxent/maxent/src/main/java/opennlp/perceptron
In directory 23jxhf1.ch3.sourceforge.com:/tmp/cvs-serv5168/src/main/java/opennlp/perceptron

Modified Files:
	PerceptronModel.java SimplePerceptronSequenceTrainer.java 
Log Message:
Changed abstract model to better support creating full model for re-tagging.

Index: PerceptronModel.java
===================================================================
RCS file: /cvsroot/maxent/maxent/src/main/java/opennlp/perceptron/PerceptronModel.java,v
retrieving revision 1.2
retrieving revision 1.3
diff -C2 -d -r1.2 -r1.3
*** PerceptronModel.java	15 Mar 2009 03:30:26 -0000	1.2
--- PerceptronModel.java	19 Mar 2009 13:04:31 -0000	1.3
***************
*** 22,25 ****
--- 22,26 ----
  import java.io.InputStreamReader;
  import java.text.DecimalFormat;
+ import java.util.Map;
  
  import opennlp.model.AbstractModel;
***************
*** 29,32 ****
--- 30,37 ----
  public class PerceptronModel extends AbstractModel {
  
+   public PerceptronModel(Context[] params, String[] predLabels, Map<String,Integer> pmap, String[] outcomeNames) {
+     super(params,predLabels,outcomeNames);
+     modelType = ModelType.Perceptron;
+   }
    
    public PerceptronModel(Context[] params, String[] predLabels, String[] outcomeNames) {

Index: SimplePerceptronSequenceTrainer.java
===================================================================
RCS file: /cvsroot/maxent/maxent/src/main/java/opennlp/perceptron/SimplePerceptronSequenceTrainer.java,v
retrieving revision 1.1
retrieving revision 1.2
diff -C2 -d -r1.1 -r1.2
*** SimplePerceptronSequenceTrainer.java	15 Mar 2009 03:32:06 -0000	1.1
--- SimplePerceptronSequenceTrainer.java	19 Mar 2009 13:04:31 -0000	1.2
***************
*** 181,185 ****
      int si=0;
      for (Sequence sequence : sequenceStream) {
!       Event[] taggerEvents = sequenceStream.updateContext(sequence, new PerceptronModel(params,predLabels,outcomeLabels));
        Event[] events = sequence.getEvents();
        for (int ei=0;ei<events.length;ei++,oei++) {
--- 181,185 ----
      int si=0;
      for (Sequence sequence : sequenceStream) {
!       Event[] taggerEvents = sequenceStream.updateContext(sequence, new PerceptronModel(params,predLabels,pmap,outcomeLabels));
        Event[] events = sequence.getEvents();
        for (int ei=0;ei<events.length;ei++,oei++) {

[Maxent-commit] CVS: maxent/src/main/java/opennlp/perceptron PerceptronTrainer.java, 1.2, 1.3

[Thomas M. <tsm...@us...>]

Update of /cvsroot/maxent/maxent/src/main/java/opennlp/perceptron
In directory 23jxhf1.ch3.sourceforge.com:/tmp/cvs-serv29677/src/main/java/opennlp/perceptron

Modified Files:
	PerceptronTrainer.java 
Log Message:
Cheanged back to updating each classifier which is above 1 when wrong answer is given.

Index: PerceptronTrainer.java
===================================================================
RCS file: /cvsroot/maxent/maxent/src/main/java/opennlp/perceptron/PerceptronTrainer.java,v
retrieving revision 1.2
retrieving revision 1.3
diff -C2 -d -r1.2 -r1.3
*** PerceptronTrainer.java	15 Mar 2009 03:31:40 -0000	1.2
--- PerceptronTrainer.java	17 Mar 2009 03:11:26 -0000	1.3
***************
*** 178,182 ****
      iteration--; //move to 0-based index
      int numCorrect = 0;
!     for (int ei = 0; ei < numUniqueEvents; ei++) {
        //Arrays.sort(contexts[ei]); only needed for debugging
        for (int ni=0;ni<this.numTimesEventsSeen[ei];ni++) {
--- 178,183 ----
      iteration--; //move to 0-based index
      int numCorrect = 0;
!     int oei = 0;
!     for (int ei = 0; ei < numUniqueEvents; ei++,oei++) {
        //Arrays.sort(contexts[ei]); only needed for debugging
        for (int ni=0;ni<this.numTimesEventsSeen[ei];ni++) {
***************
*** 197,234 ****
            }
          }
!         boolean correct = max == outcomeList[ei]; 
          if (correct) {
            numCorrect ++;
          }
!         if (!correct) {
!           int oi = outcomeList[ei];
!           for (int ci = 0; ci < contexts[ei].length; ci++) {
!             int pi = contexts[ei][ci];
!             if (values == null) {
!               params[pi].updateParameter(oi, 1);
!               params[pi].updateParameter(max,-1);
!             }
!             else {
!               params[pi].updateParameter(oi, values[ei][ci]);
!               params[pi].updateParameter(max, values[ei][ci]*-1);
!             }
!             if (useAverage) {
!               if (updates[pi][oi][VALUE] != 0) {
!                 averageParams[pi].updateParameter(oi,updates[pi][oi][VALUE]*(numEvents*(iteration-updates[pi][oi][ITER])+(ei-updates[pi][oi][EVENT])));
!                 //System.err.println("p avp["+pi+"]."+oi+"="+averageParams[pi].getParameters()[oi]);
                }
!               //System.err.println("p updates["+pi+"]["+oi+"]=("+updates[pi][oi][ITER]+","+updates[pi][oi][EVENT]+","+updates[pi][oi][VALUE]+") + ("+iteration+","+ei+","+params[pi].getParameters()[oi]+") -> "+averageParams[pi].getParameters()[oi]);
!               updates[pi][oi][VALUE] = (int) params[pi].getParameters()[oi];
!               updates[pi][oi][ITER] = iteration;
!               updates[pi][oi][EVENT] = ei;
!               
!               if (updates[pi][max][VALUE] != 0) {
!                 averageParams[pi].updateParameter(max,updates[pi][max][VALUE]*(numEvents*(iteration-updates[pi][max][ITER])+(ei-updates[pi][max][EVENT])));
!                 //System.err.println("d avp["+pi+"]."+max+"="+averageParams[pi].getParameters()[max]);
                }
-               //System.err.println(ei+" d updates["+pi+"]["+max+"]=("+updates[pi][max][ITER]+","+updates[pi][max][EVENT]+","+updates[pi][max][VALUE]+") + ("+iteration+","+ei+","+params[pi].getParameters()[max]+") -> "+averageParams[pi].getParameters()[max]);
-               updates[pi][max][VALUE] = (int) params[pi].getParameters()[max];
-               updates[pi][max][ITER] = iteration;
-               updates[pi][max][EVENT] = ei;
              }
            }
--- 198,250 ----
            }
          }
!         boolean correct = max == outcomeList[oei]; 
          if (correct) {
            numCorrect ++;
          }
!         for (int oi = 0;oi<numOutcomes;oi++) {
!           if (oi == outcomeList[oei]) {
!             if (modelDistribution[oi] <= 0) {
!               for (int ci = 0; ci < contexts[ei].length; ci++) {
!                 int pi = contexts[ei][ci];
!                 if (values == null) {
!                   params[pi].updateParameter(oi, 1);
!                 }
!                 else {
!                   params[pi].updateParameter(oi, values[ei][ci]);
!                 }
!                 if (useAverage) {
!                   if (updates[pi][oi][VALUE] != 0) {
!                     averageParams[pi].updateParameter(oi,updates[pi][oi][VALUE]*(numEvents*(iteration-updates[pi][oi][ITER])+(ei-updates[pi][oi][EVENT])));
!                     //System.err.println("p avp["+pi+"]."+oi+"="+averageParams[pi].getParameters()[oi]);
!                   }
!                   //System.err.println("p updates["+pi+"]["+oi+"]=("+updates[pi][oi][ITER]+","+updates[pi][oi][EVENT]+","+updates[pi][oi][VALUE]+") + ("+iteration+","+ei+","+params[pi].getParameters()[oi]+") -> "+averageParams[pi].getParameters()[oi]);
!                   updates[pi][oi][VALUE] = (int) params[pi].getParameters()[oi];
!                   updates[pi][oi][ITER] = iteration;
!                   updates[pi][oi][EVENT] = ei;
!                 }
                }
!             }
!           }
!           else {
!             if (modelDistribution[oi] > 0) {
!               for (int ci = 0; ci < contexts[ei].length; ci++) {
!                 int pi = contexts[ei][ci];
!                 if (values == null) {
!                   params[pi].updateParameter(oi, -1);
!                 }
!                 else {
!                   params[pi].updateParameter(oi, -1*values[ei][ci]);
!                 }
!                 if (useAverage) {
!                   if (updates[pi][oi][VALUE] != 0) {
!                     averageParams[pi].updateParameter(oi,updates[pi][oi][VALUE]*(numEvents*(iteration-updates[pi][oi][ITER])+(ei-updates[pi][oi][EVENT])));
!                     //System.err.println("d avp["+pi+"]."+oi+"="+averageParams[pi].getParameters()[oi]);
!                   }
!                   //System.err.println(ei+" d updates["+pi+"]["+oi+"]=("+updates[pi][oi][ITER]+","+updates[pi][oi][EVENT]+","+updates[pi][oi][VALUE]+") + ("+iteration+","+ei+","+params[pi].getParameters()[oi]+") -> "+averageParams[pi].getParameters()[oi]);
!                   updates[pi][oi][VALUE] = (int) params[pi].getParameters()[oi];
!                   updates[pi][oi][ITER] = iteration;
!                   updates[pi][oi][EVENT] = ei;
!                 }
                }
              }
            }

[Maxent-commit] CVS: maxent/src/main/java/opennlp/maxent BasicEventStream.java, 1.1, 1.2

[Thomas M. <tsm...@us...>]

Update of /cvsroot/maxent/maxent/src/main/java/opennlp/maxent
In directory 23jxhf1.ch3.sourceforge.com:/tmp/cvs-serv15801/src/main/java/opennlp/maxent

Modified Files:
	BasicEventStream.java 
Log Message:
fixes license typo.

Index: BasicEventStream.java
===================================================================
RCS file: /cvsroot/maxent/maxent/src/main/java/opennlp/maxent/BasicEventStream.java,v
retrieving revision 1.1
retrieving revision 1.2
diff -C2 -d -r1.1 -r1.2
*** BasicEventStream.java	22 Jan 2009 23:23:34 -0000	1.1
--- BasicEventStream.java	15 Mar 2009 03:24:00 -0000	1.2
***************
*** 1,5 ****
  /*
   * Licensed to the Apache Software Foundation (ASF) under one or more
!  * contributor license agreemnets.  See the NOTICE file distributed with
   * this work for additional information regarding copyright ownership.
   * The ASF licenses this file to You under the Apache License, Version 2.0 
--- 1,5 ----
  /*
   * Licensed to the Apache Software Foundation (ASF) under one or more
!  * contributor license agreements.  See the NOTICE file distributed with
   * this work for additional information regarding copyright ownership.
   * The ASF licenses this file to You under the Apache License, Version 2.0 

[Maxent-commit] CVS: maxent/src/main/java/opennlp/perceptron SimplePerceptronSequenceTrainer.java, NONE, 1.1

[Thomas M. <tsm...@us...>]

Update of /cvsroot/maxent/maxent/src/main/java/opennlp/perceptron
In directory 23jxhf1.ch3.sourceforge.com:/tmp/cvs-serv30444/src/main/java/opennlp/perceptron

Added Files:
	SimplePerceptronSequenceTrainer.java 
Log Message:
Simple version of sequence trainer.

--- NEW FILE: SimplePerceptronSequenceTrainer.java ---
/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0 
 * (the "License"); you may not use this file except in compliance with 
 * the License. You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package opennlp.perceptron;

import java.util.HashMap;
import java.util.Map;

import opennlp.model.AbstractModel;
import opennlp.model.DataIndexer;
import opennlp.model.Event;
import opennlp.model.MutableContext;
import opennlp.model.OnePassDataIndexer;
import opennlp.model.Sequence;
import opennlp.model.SequenceStream;
import opennlp.model.SequenceStreamEventStream;

/**
 * Trains models for sequences using the perceptron algorithm.  Each outcome is represented as
 * a binary perceptron classifier.  This supports standard (integer) weighting as well
 * average weighting.  Sequence information is used in a simplified was to that described in:
 * Discriminative Training Methods for Hidden Markov Models: Theory and Experiments
 * with the Perceptron Algorithm. Michael Collins, EMNLP 2002.
 * Specifically only updates are applied to tokens which were incorrectly tagged by a sequence tagger
 * rather than to all feature across the sequence which differ from the training sequence.
 */
public class SimplePerceptronSequenceTrainer {

  private boolean printMessages = true;
  private int iterations;
  private SequenceStream sequenceStream;
  /** Number of events in the event set. */
  private int numEvents;

  /** Number of predicates. */
  private int numPreds; 
  /** Number of outcomes. */
  private int numOutcomes;

  /** List of outcomes for each event i, in context[i]. */
  private int[] outcomeList;
  
  private String[] outcomeLabels;

  double[] modelDistribution;

  /** Stores the average parameter values of each predicate during iteration. */
  private MutableContext[] averageParams;
  
  /** Mapping between context and an integer */ 
  private Map<String,Integer> pmap;

  private Map<String,Integer> omap;
  
  /** Stores the estimated parameter value of each predicate during iteration. */
  private MutableContext[] params;
  private boolean useAverage;
  private int[][][] updates;
  private int VALUE = 0;
  private int ITER = 1;
  private int EVENT = 2;
  
  private int[] allOutcomesPattern;
  private String[] predLabels;

  public AbstractModel trainModel(int iterations, SequenceStream sequenceStream, int cutoff, boolean useAverage) {
    this.iterations = iterations;
    this.sequenceStream = sequenceStream;
    DataIndexer di = new OnePassDataIndexer(new SequenceStreamEventStream(sequenceStream),cutoff,false);
    outcomeList  = di.getOutcomeList();
    predLabels = di.getPredLabels();
    pmap = new HashMap<String,Integer>();
    for (int pli=0;pli<predLabels.length;pli++) {
      pmap.put(predLabels[pli], pli);
    }
    display("Incorporating indexed data for training...  \n");
    this.useAverage = useAverage;
    numEvents = di.getNumEvents();

    this.iterations = iterations;
    outcomeLabels = di.getOutcomeLabels();
    omap = new HashMap<String,Integer>();
    for (int oli=0;oli<outcomeLabels.length;oli++) {
      omap.put(outcomeLabels[oli], oli);
    }
    outcomeList = di.getOutcomeList();

    numPreds = predLabels.length;
    numOutcomes = outcomeLabels.length;
    if (useAverage) {
      updates = new int[numPreds][numOutcomes][3];
    }
    
    display("done.\n");
    
    display("\tNumber of Event Tokens: " + numEvents + "\n");
    display("\t    Number of Outcomes: " + numOutcomes + "\n");
    display("\t  Number of Predicates: " + numPreds + "\n");
    

    params = new MutableContext[numPreds];
    if (useAverage) averageParams = new MutableContext[numPreds];
    
    allOutcomesPattern= new int[numOutcomes];
    for (int oi = 0; oi < numOutcomes; oi++) {
      allOutcomesPattern[oi] = oi;
    }
    
    for (int pi = 0; pi < numPreds; pi++) {
      params[pi]=new MutableContext(allOutcomesPattern,new double[numOutcomes]);
      if (useAverage) averageParams[pi] = new MutableContext(allOutcomesPattern,new double[numOutcomes]);
      for (int aoi=0;aoi<numOutcomes;aoi++) {
        params[pi].setParameter(aoi, 0.0);
        if (useAverage) averageParams[pi].setParameter(aoi, 0.0);
      }
    }
    modelDistribution = new double[numOutcomes];

    display("Computing model parameters...\n");
    findParameters(iterations);
    display("...done.\n");

    /*************** Create and return the model ******************/
    String[] updatedPredLabels = predLabels;
    /*
    String[] updatedPredLabels = new String[pmap.size()];
    for (String pred : pmap.keySet()) {
      updatedPredLabels[pmap.get(pred)]=pred;
    }
    */
    if (useAverage) {
      return new PerceptronModel(averageParams, updatedPredLabels, outcomeLabels);
    }
    else {
      return new PerceptronModel(params, updatedPredLabels, outcomeLabels);
    }
  }

  private void findParameters(int iterations) {
    display("Performing " + iterations + " iterations.\n");
    for (int i = 1; i <= iterations; i++) {
      if (i < 10)
        display("  " + i + ":  ");
      else if (i < 100)
        display(" " + i + ":  ");
      else
        display(i + ":  ");
      nextIteration(i);
    }
    if (useAverage) {
      trainingStats(averageParams);
    }
    else {
      trainingStats(params);
    }
  }

  private void display(String s) {
    if (printMessages)
      System.out.print(s);
  }

  public void nextIteration(int iteration) {
    iteration--; //move to 0-based index
    int numCorrect = 0;
    int oei=0;
    int si=0;
    for (Sequence sequence : sequenceStream) {
      Event[] taggerEvents = sequenceStream.updateContext(sequence, new PerceptronModel(params,predLabels,outcomeLabels));
      Event[] events = sequence.getEvents();
      for (int ei=0;ei<events.length;ei++,oei++) {
        String[] contextStrings = events[ei].getContext();
        int[] contexts = new int[contextStrings.length];
        float values[] = events[ei].getValues();
        for (int ci=0;ci<contexts.length;ci++) {
          Integer cmi = pmap.get(contextStrings[ci]);
          contexts[ci] = cmi;
        }
        int max = omap.get(taggerEvents[ei].getOutcome());
        boolean correct = max == outcomeList[oei]; 
        if (correct) {
          numCorrect ++;
        }
        for (int oi = 0;oi<numOutcomes;oi++) {
          if (oi == outcomeList[oei]) {
            if (!correct) {
              for (int ci = 0; ci < contexts.length; ci++) {
                int pi = contexts[ci];
                if (values == null) {
                  params[pi].updateParameter(oi, 1);
                }
                else {
                  params[pi].updateParameter(oi, values[ci]);
                }
                if (useAverage) {
                  if (updates[pi][oi][VALUE] != 0) {
                    averageParams[pi].updateParameter(oi,updates[pi][oi][VALUE]*(numEvents*(iteration-updates[pi][oi][ITER])+(oei-updates[pi][oi][EVENT])));
                    //System.err.println("p avp["+pi+"]."+oi+"="+averageParams[pi].getParameters()[oi]);
                  }
                  //System.err.println("p updates["+pi+"]["+oi+"]=("+updates[pi][oi][ITER]+","+updates[pi][oi][EVENT]+","+updates[pi][oi][VALUE]+") + ("+iteration+","+oei+","+params[pi].getParameters()[oi]+") -> "+averageParams[pi].getParameters()[oi]);
                  updates[pi][oi][VALUE] = (int) params[pi].getParameters()[oi];
                  updates[pi][oi][ITER] = iteration;
                  updates[pi][oi][EVENT] = oei;
                }
              }
            }
          }
          else {
            if (oi == max) { //case where it correct is taken by above if.
              for (int ci = 0; ci < contexts.length; ci++) {
                int pi = contexts[ci];
                if (values == null) {
                  params[pi].updateParameter(oi,-1);
                }
                else {
                  params[pi].updateParameter(oi, values[ci]*-1);
                }
                if (useAverage) {
                  if (updates[pi][oi][VALUE] != 0) {
                    averageParams[pi].updateParameter(oi,updates[pi][oi][VALUE]*(numEvents*(iteration-updates[pi][oi][ITER])+(oei-updates[pi][oi][EVENT])));
                    //System.err.println(oei+" d avp["+pi+"]."+oi+"="+averageParams[pi].getParameters()[oi]);
                  }
                  //System.err.println(oei+" d updates["+pi+"]["+oi+"]=("+updates[pi][oi][ITER]+","+updates[pi][oi][EVENT]+","+updates[pi][oi][VALUE]+") + ("+iteration+","+oei+","+params[pi].getParameters()[oi]+") -> "+averageParams[pi].getParameters()[oi]);
                  updates[pi][oi][VALUE] = (int) params[pi].getParameters()[oi];
                  updates[pi][oi][ITER] = iteration;
                  updates[pi][oi][EVENT] = oei;
                }
              }
            }
          }
        }
      }
      si++;
    }
    //finish average computation
    double totIterations = (double) iterations*numEvents;
    if (useAverage && iteration == iterations-1) {
      for (int pi = 0; pi < numPreds; pi++) {
        double[] predParams = averageParams[pi].getParameters();
        for (int oi = 0;oi<numOutcomes;oi++) {
          if (updates[pi][oi][VALUE] != 0) {
            predParams[oi] +=  updates[pi][oi][VALUE]*(numEvents*(iterations-updates[pi][oi][ITER])-updates[pi][oi][EVENT]);
          }
          if (predParams[oi] != 0) {
            predParams[oi] /=totIterations;  
            averageParams[pi].setParameter(oi, predParams[oi]);
            //System.err.println("updates["+pi+"]["+oi+"]=("+updates[pi][oi][ITER]+","+updates[pi][oi][EVENT]+","+updates[pi][oi][VALUE]+") + ("+iterations+","+0+","+params[pi].getParameters()[oi]+") -> "+averageParams[pi].getParameters()[oi]);
          }
        }
      }
    }
    display(". ("+numCorrect+"/"+numEvents+") "+((double) numCorrect / numEvents) + "\n");
  }
  
  private void trainingStats(MutableContext[] params) {
    int numCorrect = 0;
    int oei=0;
    for (Sequence sequence : sequenceStream) {
      Event[] taggerEvents = sequenceStream.updateContext(sequence, new PerceptronModel(params,predLabels,outcomeLabels));
      for (int ei=0;ei<taggerEvents.length;ei++,oei++) {
        int max = omap.get(taggerEvents[ei].getOutcome());
        if (max == outcomeList[oei]) {
          numCorrect ++;
        }
      }
    }
    display(". ("+numCorrect+"/"+numEvents+") "+((double) numCorrect / numEvents) + "\n");
  }
}

[Maxent-commit] CVS: maxent/src/main/java/opennlp/perceptron PerceptronTrainer.java, 1.1, 1.2

[Thomas M. <tsm...@us...>]

Update of /cvsroot/maxent/maxent/src/main/java/opennlp/perceptron
In directory 23jxhf1.ch3.sourceforge.com:/tmp/cvs-serv29844/src/main/java/opennlp/perceptron

Modified Files:
	PerceptronTrainer.java 
Log Message:
updated to be more efficient.

Index: PerceptronTrainer.java
===================================================================
RCS file: /cvsroot/maxent/maxent/src/main/java/opennlp/perceptron/PerceptronTrainer.java,v
retrieving revision 1.1
retrieving revision 1.2
diff -C2 -d -r1.1 -r1.2
*** PerceptronTrainer.java	22 Jan 2009 23:23:34 -0000	1.1
--- PerceptronTrainer.java	15 Mar 2009 03:31:40 -0000	1.2
***************
*** 1,2 ****
--- 1,19 ----
+ /*
+  * Licensed to the Apache Software Foundation (ASF) under one or more
+  * contributor license agreements.  See the NOTICE file distributed with
+  * this work for additional information regarding copyright ownership.
+  * The ASF licenses this file to You under the Apache License, Version 2.0 
+  * (the "License"); you may not use this file except in compliance with 
+  * the License. You may obtain a copy of the License at
+  *
+  *     http://www.apache.org/licenses/LICENSE-2.0
+  *
+  * Unless required by applicable law or agreed to in writing, software
+  * distributed under the License is distributed on an "AS IS" BASIS,
+  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  * See the License for the specific language governing permissions and
+  * limitations under the License.
+  */
+ 
  package opennlp.perceptron;
  
***************
*** 4,13 ****
  import opennlp.model.DataIndexer;
  import opennlp.model.EvalParameters;
- import opennlp.model.EvalParameters;
  import opennlp.model.MutableContext;
! 
  public class PerceptronTrainer {
  
!   /** Number of unique events which occured in the event set. */
    private int numUniqueEvents;
    /** Number of events in the event set. */
--- 21,36 ----
  import opennlp.model.DataIndexer;
  import opennlp.model.EvalParameters;
  import opennlp.model.MutableContext;
! /**
!  * Trains models using the perceptron algorithm.  Each outcome is represented as
!  * a binary perceptron classifier.  This supports standard (integer) weighting as well
!  * average weighting as described in:
!  * Discriminative Training Methods for Hidden Markov Models: Theory and Experiments
!  * with the Perceptron Algorithm. Michael Collins, EMNLP 2002.
!  *
!  */
  public class PerceptronTrainer {
  
!   /** Number of unique events which occurred in the event set. */
    private int numUniqueEvents;
    /** Number of events in the event set. */
***************
*** 101,109 ****
      }
      
-     int numActiveOutcomes = numOutcomes;
      for (int pi = 0; pi < numPreds; pi++) {
!       params[pi] = new MutableContext(allOutcomesPattern,new double[numActiveOutcomes]);
!       if (useAverage) averageParams[pi] = new MutableContext(allOutcomesPattern,new double[numActiveOutcomes]);
!       for (int aoi=0;aoi<numActiveOutcomes;aoi++) {
          params[pi].setParameter(aoi, 0.0);
          if (useAverage) averageParams[pi].setParameter(aoi, 0.0);
--- 124,131 ----
      }
      
      for (int pi = 0; pi < numPreds; pi++) {
!       params[pi] = new MutableContext(allOutcomesPattern,new double[numOutcomes]);
!       if (useAverage) averageParams[pi] = new MutableContext(allOutcomesPattern,new double[numOutcomes]);
!       for (int aoi=0;aoi<numOutcomes;aoi++) {
          params[pi].setParameter(aoi, 0.0);
          if (useAverage) averageParams[pi].setParameter(aoi, 0.0);
***************
*** 141,144 ****
--- 163,172 ----
        nextIteration(i);
      }
+     if (useAverage) {
+       trainingStats(averageParams);
+     }
+     else {
+       trainingStats(params);
+     }
      // kill a bunch of these big objects now that we don't need them
      numTimesEventsSeen = null;
***************
*** 146,155 ****
    }
    
!   /* Compute one iteration of Perceptron and retutn log-likelihood.*/
    private void nextIteration(int iteration) {
      iteration--; //move to 0-based index
      int numCorrect = 0;
      for (int ei = 0; ei < numUniqueEvents; ei++) {
        for (int ni=0;ni<this.numTimesEventsSeen[ei];ni++) {
          for (int oi = 0; oi < numOutcomes; oi++) {
            modelDistribution[oi] = 0;
--- 174,185 ----
    }
    
!   /* Compute one iteration of Perceptron and return log-likelihood.*/
    private void nextIteration(int iteration) {
      iteration--; //move to 0-based index
      int numCorrect = 0;
      for (int ei = 0; ei < numUniqueEvents; ei++) {
+       //Arrays.sort(contexts[ei]); only needed for debugging
        for (int ni=0;ni<this.numTimesEventsSeen[ei];ni++) {
+         //System.err.print("contexts["+ei+"]=");for (int ci=0;ci<contexts[ei].length;ci++) { System.err.print(" "+contexts[ei][ci]+" ");} System.err.println();
          for (int oi = 0; oi < numOutcomes; oi++) {
            modelDistribution[oi] = 0;
***************
*** 167,216 ****
            }
          }
!         if (max == outcomeList[ei]) {
!           numCorrect += numTimesEventsSeen[ei];
          }
!         for (int oi = 0;oi<numOutcomes;oi++) {
!           if (oi == outcomeList[ei]) {
!             if (modelDistribution[oi] <= 0) {
!               for (int ci = 0; ci < contexts[ei].length; ci++) {
!                 int pi = contexts[ei][ci];
!                 if (values == null) {
!                   params[pi].updateParameter(oi, 1);
!                 }
!                 else {
!                   params[pi].updateParameter(oi, values[ei][ci]);
!                 }
!                 if (useAverage) {
!                   if (updates[pi][oi][VALUE] != 0) {
!                     averageParams[pi].updateParameter(oi,updates[pi][oi][VALUE]*(numEvents*(iteration-updates[pi][oi][ITER])+(ei-updates[pi][oi][EVENT])));
!                   }
!                   //System.err.println("updates["+pi+"]["+oi+"]=("+updates[pi][oi][ITER]+","+updates[pi][oi][EVENT]+","+updates[pi][oi][VALUE]+") + ("+iteration+","+ei+","+params[pi].getParameters()[oi]+") -> "+averageParams[pi].getParameters()[oi]);
!                   updates[pi][oi][VALUE] = (int) params[pi].getParameters()[oi];
!                   updates[pi][oi][ITER] = iteration;
!                   updates[pi][oi][EVENT] = ei;
!                 }
!               }
              }
!           }
!           else {
!             if (modelDistribution[oi] > 0) {
!               for (int ci = 0; ci < contexts[ei].length; ci++) {
!                 int pi = contexts[ei][ci];
!                 if (values == null) {
!                   params[pi].updateParameter(oi,-1);
!                 }
!                 else {
!                   params[pi].updateParameter(oi, values[ei][ci]*-1);
!                 }
!                 if (useAverage) {
!                   if (updates[pi][oi][VALUE] != 0) {
!                     averageParams[pi].updateParameter(oi,updates[pi][oi][VALUE]*(numEvents*(iteration-updates[pi][oi][ITER])+(ei-updates[pi][oi][EVENT])));
!                   }
!                   //System.err.println("updates["+pi+"]["+oi+"]=("+updates[pi][oi][ITER]+","+updates[pi][oi][EVENT]+","+updates[pi][oi][VALUE]+") + ("+iteration+","+ei+","+params[pi].getParameters()[oi]+") -> "+averageParams[pi].getParameters()[oi]);
!                   updates[pi][oi][VALUE] = (int) params[pi].getParameters()[oi];
!                   updates[pi][oi][ITER] = iteration;
!                   updates[pi][oi][EVENT] = ei;
!                 }
                }
              }
            }
--- 197,234 ----
            }
          }
!         boolean correct = max == outcomeList[ei]; 
!         if (correct) {
!           numCorrect ++;
          }
!         if (!correct) {
!           int oi = outcomeList[ei];
!           for (int ci = 0; ci < contexts[ei].length; ci++) {
!             int pi = contexts[ei][ci];
!             if (values == null) {
!               params[pi].updateParameter(oi, 1);
!               params[pi].updateParameter(max,-1);
              }
!             else {
!               params[pi].updateParameter(oi, values[ei][ci]);
!               params[pi].updateParameter(max, values[ei][ci]*-1);
!             }
!             if (useAverage) {
!               if (updates[pi][oi][VALUE] != 0) {
!                 averageParams[pi].updateParameter(oi,updates[pi][oi][VALUE]*(numEvents*(iteration-updates[pi][oi][ITER])+(ei-updates[pi][oi][EVENT])));
!                 //System.err.println("p avp["+pi+"]."+oi+"="+averageParams[pi].getParameters()[oi]);
!               }
!               //System.err.println("p updates["+pi+"]["+oi+"]=("+updates[pi][oi][ITER]+","+updates[pi][oi][EVENT]+","+updates[pi][oi][VALUE]+") + ("+iteration+","+ei+","+params[pi].getParameters()[oi]+") -> "+averageParams[pi].getParameters()[oi]);
!               updates[pi][oi][VALUE] = (int) params[pi].getParameters()[oi];
!               updates[pi][oi][ITER] = iteration;
!               updates[pi][oi][EVENT] = ei;
!               
!               if (updates[pi][max][VALUE] != 0) {
!                 averageParams[pi].updateParameter(max,updates[pi][max][VALUE]*(numEvents*(iteration-updates[pi][max][ITER])+(ei-updates[pi][max][EVENT])));
!                 //System.err.println("d avp["+pi+"]."+max+"="+averageParams[pi].getParameters()[max]);
                }
+               //System.err.println(ei+" d updates["+pi+"]["+max+"]=("+updates[pi][max][ITER]+","+updates[pi][max][EVENT]+","+updates[pi][max][VALUE]+") + ("+iteration+","+ei+","+params[pi].getParameters()[max]+") -> "+averageParams[pi].getParameters()[max]);
+               updates[pi][max][VALUE] = (int) params[pi].getParameters()[max];
+               updates[pi][max][ITER] = iteration;
+               updates[pi][max][EVENT] = ei;
              }
            }
***************
*** 235,239 ****
        }
      }
!     display(". "+((double) numCorrect / numEvents) + "\n");
    }  
  }
--- 253,284 ----
        }
      }
!     display(". ("+numCorrect+"/"+numEvents+") "+((double) numCorrect / numEvents) + "\n");
    }  
+   
+   private void trainingStats(MutableContext[] params) {
+     int numCorrect = 0;
+     for (int ei = 0; ei < numUniqueEvents; ei++) {
+       for (int ni=0;ni<this.numTimesEventsSeen[ei];ni++) {
+         for (int oi = 0; oi < numOutcomes; oi++) {
+           modelDistribution[oi] = 0;
+         }
+         if (values != null) {
+           PerceptronModel.eval(contexts[ei], values[ei], modelDistribution, evalParams,false);
+         }
+         else {
+           PerceptronModel.eval(contexts[ei], null, modelDistribution, evalParams, false);
+         }
+         int max = 0;
+         for (int oi = 1; oi < numOutcomes; oi++) {
+           if (modelDistribution[oi] > modelDistribution[max]) {
+             max = oi;
+           }
+         }
+         if (max == outcomeList[ei]) {
+           numCorrect ++;
+         }
+       }
+     }
+     display(". ("+numCorrect+"/"+numEvents+") "+((double) numCorrect / numEvents) + "\n");
+   }
  }

[Maxent-commit] CVS: maxent/src/main/java/opennlp/perceptron PerceptronModelWriter.java, 1.1, 1.2

[Thomas M. <tsm...@us...>]

Update of /cvsroot/maxent/maxent/src/main/java/opennlp/perceptron
In directory 23jxhf1.ch3.sourceforge.com:/tmp/cvs-serv28276/src/main/java/opennlp/perceptron

Modified Files:
	PerceptronModelWriter.java 
Log Message:
added license; fixed some bugs.

Index: PerceptronModelWriter.java
===================================================================
RCS file: /cvsroot/maxent/maxent/src/main/java/opennlp/perceptron/PerceptronModelWriter.java,v
retrieving revision 1.1
retrieving revision 1.2
diff -C2 -d -r1.1 -r1.2
*** PerceptronModelWriter.java	22 Jan 2009 23:23:34 -0000	1.1
--- PerceptronModelWriter.java	15 Mar 2009 03:30:59 -0000	1.2
***************
*** 1,2 ****
--- 1,19 ----
+ /*
+  * Licensed to the Apache Software Foundation (ASF) under one or more
+  * contributor license agreements.  See the NOTICE file distributed with
+  * this work for additional information regarding copyright ownership.
+  * The ASF licenses this file to You under the Apache License, Version 2.0 
+  * (the "License"); you may not use this file except in compliance with 
+  * the License. You may obtain a copy of the License at
+  *
+  *     http://www.apache.org/licenses/LICENSE-2.0
+  *
+  * Unless required by applicable law or agreed to in writing, software
+  * distributed under the License is distributed on an "AS IS" BASIS,
+  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  * See the License for the specific language governing permissions and
+  * limitations under the License.
+  */
+ 
  package opennlp.perceptron;
  
***************
*** 17,22 ****
   * extending class to define precisely how the data should be stored.
   *
-  * @author      Jason Baldridge
-  * @version     $Revision$, $Date$
   */
  public abstract class PerceptronModelWriter extends AbstractModelWriter {
--- 34,37 ----
***************
*** 50,56 ****
          int numParams = 0;    
          double[] predParams = PARAMS[pid].getParameters();
          for (int pi=0;pi<predParams.length;pi++) {
            if (predParams[pi] != 0d) {
!             tmpOutcomes[numParams]=pi;
              tmpParams[numParams]=predParams[pi];
              numParams++;
--- 65,72 ----
          int numParams = 0;    
          double[] predParams = PARAMS[pid].getParameters();
+         int[] outcomePattern = PARAMS[pid].getOutcomes();
          for (int pi=0;pi<predParams.length;pi++) {
            if (predParams[pi] != 0d) {
!             tmpOutcomes[numParams]=outcomePattern[pi];
              tmpParams[numParams]=predParams[pi];
              numParams++;
***************
*** 70,73 ****
--- 86,90 ----
          }
        }
+       System.err.println("Compressed "+PARAMS.length+" parameters to "+numPreds);
        sortPreds = new ComparablePredicate[numPreds];
        for (int pid=0;pid<numPreds;pid++) {
***************
*** 79,97 ****
      
          
!     protected List compressOutcomes (ComparablePredicate[] sorted) {
        ComparablePredicate cp = sorted[0];
!       List outcomePatterns = new ArrayList();
!       List newGroup = new ArrayList();
        for (int i=0; i<sorted.length; i++) {
          if (cp.compareTo(sorted[i]) == 0) {
            newGroup.add(sorted[i]);
!         } else {	    
            cp = sorted[i];
            outcomePatterns.add(newGroup);
!           newGroup = new ArrayList();
            newGroup.add(sorted[i]);
!         }	    
        }
        outcomePatterns.add(newGroup);
        return outcomePatterns;
      }
--- 96,116 ----
      
          
!     protected List<List<ComparablePredicate>> computeOutcomePatterns(ComparablePredicate[] sorted) {
        ComparablePredicate cp = sorted[0];
!       List<List<ComparablePredicate>> outcomePatterns = new ArrayList<List<ComparablePredicate>>();
!       List<ComparablePredicate> newGroup = new ArrayList<ComparablePredicate>();
        for (int i=0; i<sorted.length; i++) {
          if (cp.compareTo(sorted[i]) == 0) {
            newGroup.add(sorted[i]);
!         } 
!         else {	    
            cp = sorted[i];
            outcomePatterns.add(newGroup);
!           newGroup = new ArrayList<ComparablePredicate>();
            newGroup.add(sorted[i]);
!         }
        }
        outcomePatterns.add(newGroup);
+       System.err.println(outcomePatterns.size()+" outcome patterns");
        return outcomePatterns;
      }
***************
*** 120,131 ****
        // compactly than as the entire list.
        ComparablePredicate[] sorted = sortValues();
!       List compressed = compressOutcomes(sorted);
        
        writeInt(compressed.size());
        
        for (int i=0; i<compressed.size(); i++) {
!         List a = (List)compressed.get(i);
!         writeUTF(a.size()
!             + ((ComparablePredicate)a.get(0)).toString());
        } 
        
--- 139,149 ----
        // compactly than as the entire list.
        ComparablePredicate[] sorted = sortValues();
!       List<List<ComparablePredicate>> compressed = computeOutcomePatterns(sorted);
        
        writeInt(compressed.size());
        
        for (int i=0; i<compressed.size(); i++) {
!         List<ComparablePredicate> a = compressed.get(i);
!         writeUTF(a.size() + a.get(0).toString());
        } 
        

[Maxent-commit] CVS: maxent/src/main/java/opennlp/perceptron PerceptronModel.java, 1.1, 1.2

[Thomas M. <tsm...@us...>]

Update of /cvsroot/maxent/maxent/src/main/java/opennlp/perceptron
In directory 23jxhf1.ch3.sourceforge.com:/tmp/cvs-serv27842/src/main/java/opennlp/perceptron

Modified Files:
	PerceptronModel.java 
Log Message:
added license.

Index: PerceptronModel.java
===================================================================
RCS file: /cvsroot/maxent/maxent/src/main/java/opennlp/perceptron/PerceptronModel.java,v
retrieving revision 1.1
retrieving revision 1.2
diff -C2 -d -r1.1 -r1.2
*** PerceptronModel.java	22 Jan 2009 23:23:34 -0000	1.1
--- PerceptronModel.java	15 Mar 2009 03:30:26 -0000	1.2
***************
*** 1,2 ****
--- 1,19 ----
+ /*
+  * Licensed to the Apache Software Foundation (ASF) under one or more
+  * contributor license agreements.  See the NOTICE file distributed with
+  * this work for additional information regarding copyright ownership.
+  * The ASF licenses this file to You under the Apache License, Version 2.0 
+  * (the "License"); you may not use this file except in compliance with 
+  * the License. You may obtain a copy of the License at
+  *
+  *     http://www.apache.org/licenses/LICENSE-2.0
+  *
+  * Unless required by applicable law or agreed to in writing, software
+  * distributed under the License is distributed on an "AS IS" BASIS,
+  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  * See the License for the specific language governing permissions and
+  * limitations under the License.
+  */
+ 
  package opennlp.perceptron;
  
***************
*** 88,92 ****
        }
      }
-     //System.err.println("Perceptron Model: "+java.util.Arrays.asList(prior));
      return prior;
    }
--- 105,108 ----

[Maxent-commit] CVS: maxent/src/main/java/opennlp/perceptron PerceptronModelReader.java, 1.1, 1.2 PlainTextPerceptronModelReader.java, 1.1, 1.2 BinaryPerceptronModelReader.java, 1.1, 1.2 PlainTextPerceptronModelWriter.java, 1.1, 1.2 SuffixSensitivePerceptronModelWriter.java, 1.1, 1.2 BinaryPerceptronModelWriter.java, 1.1, 1.2

[Thomas M. <tsm...@us...>]

Update of /cvsroot/maxent/maxent/src/main/java/opennlp/perceptron
In directory 23jxhf1.ch3.sourceforge.com:/tmp/cvs-serv27454/src/main/java/opennlp/perceptron

Modified Files:
	PerceptronModelReader.java PlainTextPerceptronModelReader.java 
	BinaryPerceptronModelReader.java 
	PlainTextPerceptronModelWriter.java 
	SuffixSensitivePerceptronModelWriter.java 
	BinaryPerceptronModelWriter.java 
Log Message:
fixed

Index: PerceptronModelReader.java
===================================================================
RCS file: /cvsroot/maxent/maxent/src/main/java/opennlp/perceptron/PerceptronModelReader.java,v
retrieving revision 1.1
retrieving revision 1.2
diff -C2 -d -r1.1 -r1.2
*** PerceptronModelReader.java	22 Jan 2009 23:23:34 -0000	1.1
--- PerceptronModelReader.java	15 Mar 2009 03:29:56 -0000	1.2
***************
*** 1,2 ****
--- 1,19 ----
+ /*
+  * Licensed to the Apache Software Foundation (ASF) under one or more
+  * contributor license agreements.  See the NOTICE file distributed with
+  * this work for additional information regarding copyright ownership.
+  * The ASF licenses this file to You under the Apache License, Version 2.0 
+  * (the "License"); you may not use this file except in compliance with 
+  * the License. You may obtain a copy of the License at
+  *
+  *     http://www.apache.org/licenses/LICENSE-2.0
+  *
+  * Unless required by applicable law or agreed to in writing, software
+  * distributed under the License is distributed on an "AS IS" BASIS,
+  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  * See the License for the specific language governing permissions and
+  * limitations under the License.
+  */
+ 
  package opennlp.perceptron;
  
***************
*** 10,17 ****
  
  /**
!  * Abstract parent class for readers of GISModels.
   *
-  * @author      Jason Baldridge
-  * @version     $Revision$, $Date$
   */
  public class PerceptronModelReader extends AbstractModelReader {
--- 27,32 ----
  
  /**
!  * Abstract parent class for readers of Perceptron.
   *
   */
  public class PerceptronModelReader extends AbstractModelReader {

Index: PlainTextPerceptronModelReader.java
===================================================================
RCS file: /cvsroot/maxent/maxent/src/main/java/opennlp/perceptron/PlainTextPerceptronModelReader.java,v
retrieving revision 1.1
retrieving revision 1.2
diff -C2 -d -r1.1 -r1.2
*** PlainTextPerceptronModelReader.java	22 Jan 2009 23:23:34 -0000	1.1
--- PlainTextPerceptronModelReader.java	15 Mar 2009 03:29:56 -0000	1.2
***************
*** 1,2 ****
--- 1,19 ----
+ /*
+  * Licensed to the Apache Software Foundation (ASF) under one or more
+  * contributor license agreements.  See the NOTICE file distributed with
+  * this work for additional information regarding copyright ownership.
+  * The ASF licenses this file to You under the Apache License, Version 2.0 
+  * (the "License"); you may not use this file except in compliance with 
+  * the License. You may obtain a copy of the License at
+  *
+  *     http://www.apache.org/licenses/LICENSE-2.0
+  *
+  * Unless required by applicable law or agreed to in writing, software
+  * distributed under the License is distributed on an "AS IS" BASIS,
+  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  * See the License for the specific language governing permissions and
+  * limitations under the License.
+  */
+ 
  package opennlp.perceptron;
  

Index: BinaryPerceptronModelReader.java
===================================================================
RCS file: /cvsroot/maxent/maxent/src/main/java/opennlp/perceptron/BinaryPerceptronModelReader.java,v
retrieving revision 1.1
retrieving revision 1.2
diff -C2 -d -r1.1 -r1.2
*** BinaryPerceptronModelReader.java	22 Jan 2009 23:23:34 -0000	1.1
--- BinaryPerceptronModelReader.java	15 Mar 2009 03:29:56 -0000	1.2
***************
*** 1,2 ****
--- 1,19 ----
+ /*
+  * Licensed to the Apache Software Foundation (ASF) under one or more
+  * contributor license agreements.  See the NOTICE file distributed with
+  * this work for additional information regarding copyright ownership.
+  * The ASF licenses this file to You under the Apache License, Version 2.0 
+  * (the "License"); you may not use this file except in compliance with 
+  * the License. You may obtain a copy of the License at
+  *
+  *     http://www.apache.org/licenses/LICENSE-2.0
+  *
+  * Unless required by applicable law or agreed to in writing, software
+  * distributed under the License is distributed on an "AS IS" BASIS,
+  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  * See the License for the specific language governing permissions and
+  * limitations under the License.
+  */
+ 
  package opennlp.perceptron;
  

Index: PlainTextPerceptronModelWriter.java
===================================================================
RCS file: /cvsroot/maxent/maxent/src/main/java/opennlp/perceptron/PlainTextPerceptronModelWriter.java,v
retrieving revision 1.1
retrieving revision 1.2
diff -C2 -d -r1.1 -r1.2
*** PlainTextPerceptronModelWriter.java	22 Jan 2009 23:23:34 -0000	1.1
--- PlainTextPerceptronModelWriter.java	15 Mar 2009 03:29:56 -0000	1.2
***************
*** 1,19 ****
! ///////////////////////////////////////////////////////////////////////////////
! // Copyright (C) 2001 Jason Baldridge and Gann Bierner
! //
! // This library is free software; you can redistribute it and/or
! // modify it under the terms of the GNU Lesser General Public
! // License as published by the Free Software Foundation; either
! // version 2.1 of the License, or (at your option) any later version.
! //
! // This library is distributed in the hope that it will be useful,
! // but WITHOUT ANY WARRANTY; without even the implied warranty of
! // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
! // GNU General Public License for more details.
! //
! // You should have received a copy of the GNU Lesser General Public
! // License along with this program; if not, write to the Free Software
! // Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
! //////////////////////////////////////////////////////////////////////////////   
  package opennlp.perceptron;
  
--- 1,19 ----
! /*
!  * Licensed to the Apache Software Foundation (ASF) under one or more
!  * contributor license agreements.  See the NOTICE file distributed with
!  * this work for additional information regarding copyright ownership.
!  * The ASF licenses this file to You under the Apache License, Version 2.0 
!  * (the "License"); you may not use this file except in compliance with 
!  * the License. You may obtain a copy of the License at
!  *
!  *     http://www.apache.org/licenses/LICENSE-2.0
!  *
!  * Unless required by applicable law or agreed to in writing, software
!  * distributed under the License is distributed on an "AS IS" BASIS,
!  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
!  * See the License for the specific language governing permissions and
!  * limitations under the License.
!  */
! 
  package opennlp.perceptron;
  
***************
*** 31,93 ****
  /**
   * Model writer that saves models in plain text format.
-  *
-  * @author      Jason Baldridge
-  * @version     $Revision$, $Date$
   */
  public class PlainTextPerceptronModelWriter extends PerceptronModelWriter {
!     BufferedWriter output;
!     
!    /**
!      * Constructor which takes a PerceptronModel and a File and prepares itself to
!      * write the model to that file. Detects whether the file is gzipped or not
!      * based on whether the suffix contains ".gz".
!      *
!      * @param model The PerceptronModel which is to be persisted.
!      * @param f The File in which the model is to be persisted.
!      */
!      public PlainTextPerceptronModelWriter (AbstractModel model, File f)
! 	throws IOException, FileNotFoundException {
  
! 	super(model);
! 	if (f.getName().endsWith(".gz")) {
! 	    output = new BufferedWriter(new OutputStreamWriter(
!   	                 new GZIPOutputStream(new FileOutputStream(f))));
! 	}
! 	else {
! 	    output = new BufferedWriter(new FileWriter(f));
! 	}
!     }
  
!    /**
!      * Constructor which takes a PerceptronModel and a BufferedWriter and prepares
!      * itself to write the model to that writer.
!      *
!      * @param model The PerceptronModel which is to be persisted.
!      * @param bw The BufferedWriter which will be used to persist the model.
!      */
!     public PlainTextPerceptronModelWriter (AbstractModel model, BufferedWriter bw) {
! 	super(model);
! 	output = bw;
      }
! 
!     public void writeUTF (String s) throws java.io.IOException {
! 	output.write(s);
! 	output.newLine();
      }
  
!     public void writeInt (int i) throws java.io.IOException {
! 	output.write(Integer.toString(i));
! 	output.newLine();
!     }
!     
!     public void writeDouble (double d) throws java.io.IOException {
! 	output.write(Double.toString(d));
! 	output.newLine();
!     }
  
-     public void close () throws java.io.IOException {
- 	output.flush();
- 	output.close();
-     }
-     
  }
--- 31,90 ----
  /**
   * Model writer that saves models in plain text format.
   */
  public class PlainTextPerceptronModelWriter extends PerceptronModelWriter {
!   BufferedWriter output;
  
!   /**
!    * Constructor which takes a PerceptronModel and a File and prepares itself to
!    * write the model to that file. Detects whether the file is gzipped or not
!    * based on whether the suffix contains ".gz".
!    *
!    * @param model The PerceptronModel which is to be persisted.
!    * @param f The File in which the model is to be persisted.
!    */
!   public PlainTextPerceptronModelWriter (AbstractModel model, File f)
!   throws IOException, FileNotFoundException {
  
!     super(model);
!     if (f.getName().endsWith(".gz")) {
!       output = new BufferedWriter(new OutputStreamWriter(
!           new GZIPOutputStream(new FileOutputStream(f))));
      }
!     else {
!       output = new BufferedWriter(new FileWriter(f));
      }
+   }
  
!   /**
!    * Constructor which takes a PerceptronModel and a BufferedWriter and prepares
!    * itself to write the model to that writer.
!    *
!    * @param model The PerceptronModel which is to be persisted.
!    * @param bw The BufferedWriter which will be used to persist the model.
!    */
!   public PlainTextPerceptronModelWriter (AbstractModel model, BufferedWriter bw) {
!     super(model);
!     output = bw;
!   }
! 
!   public void writeUTF (String s) throws java.io.IOException {
!     output.write(s);
!     output.newLine();
!   }
! 
!   public void writeInt (int i) throws java.io.IOException {
!     output.write(Integer.toString(i));
!     output.newLine();
!   }
! 
!   public void writeDouble (double d) throws java.io.IOException {
!     output.write(Double.toString(d));
!     output.newLine();
!   }
! 
!   public void close () throws java.io.IOException {
!     output.flush();
!     output.close();
!   }
  
  }

Index: SuffixSensitivePerceptronModelWriter.java
===================================================================
RCS file: /cvsroot/maxent/maxent/src/main/java/opennlp/perceptron/SuffixSensitivePerceptronModelWriter.java,v
retrieving revision 1.1
retrieving revision 1.2
diff -C2 -d -r1.1 -r1.2
*** SuffixSensitivePerceptronModelWriter.java	22 Jan 2009 23:23:34 -0000	1.1
--- SuffixSensitivePerceptronModelWriter.java	15 Mar 2009 03:29:56 -0000	1.2
***************
*** 1,19 ****
! ///////////////////////////////////////////////////////////////////////////////
! // Copyright (C) 2001 Jason Baldridge and Gann Bierner
! //
! // This library is free software; you can redistribute it and/or
! // modify it under the terms of the GNU Lesser General Public
! // License as published by the Free Software Foundation; either
! // version 2.1 of the License, or (at your option) any later version.
! //
! // This library is distributed in the hope that it will be useful,
! // but WITHOUT ANY WARRANTY; without even the implied warranty of
! // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
! // GNU General Public License for more details.
! //
! // You should have received a copy of the GNU Lesser General Public
! // License along with this program; if not, write to the Free Software
! // Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
! //////////////////////////////////////////////////////////////////////////////   
  package opennlp.perceptron;
  
--- 1,18 ----
! /*
!  * Licensed to the Apache Software Foundation (ASF) under one or more
!  * contributor license agreements.  See the NOTICE file distributed with
!  * this work for additional information regarding copyright ownership.
!  * The ASF licenses this file to You under the Apache License, Version 2.0 
!  * (the "License"); you may not use this file except in compliance with 
!  * the License. You may obtain a copy of the License at
!  *
!  *     http://www.apache.org/licenses/LICENSE-2.0
!  *
!  * Unless required by applicable law or agreed to in writing, software
!  * distributed under the License is distributed on an "AS IS" BASIS,
!  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
!  * See the License for the specific language governing permissions and
!  * limitations under the License.
!  */
  package opennlp.perceptron;
  

Index: BinaryPerceptronModelWriter.java
===================================================================
RCS file: /cvsroot/maxent/maxent/src/main/java/opennlp/perceptron/BinaryPerceptronModelWriter.java,v
retrieving revision 1.1
retrieving revision 1.2
diff -C2 -d -r1.1 -r1.2
*** BinaryPerceptronModelWriter.java	22 Jan 2009 23:23:34 -0000	1.1
--- BinaryPerceptronModelWriter.java	15 Mar 2009 03:29:56 -0000	1.2
***************
*** 1,18 ****
! ///////////////////////////////////////////////////////////////////////////////
! //Copyright (C) 2001 Jason Baldridge and Gann Bierner
! 
! //This library is free software; you can redistribute it and/or
! //modify it under the terms of the GNU Lesser General Public
! //License as published by the Free Software Foundation; either
! //version 2.1 of the License, or (at your option) any later version.
! 
! //This library is distributed in the hope that it will be useful,
! //but WITHOUT ANY WARRANTY; without even the implied warranty of
! //MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
! //GNU General Public License for more details.
! 
! //You should have received a copy of the GNU Lesser General Public
! //License along with this program; if not, write to the Free Software
! //Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
  
  package opennlp.perceptron;
--- 1,18 ----
! /*
!  * Licensed to the Apache Software Foundation (ASF) under one or more
!  * contributor license agreements.  See the NOTICE file distributed with
!  * this work for additional information regarding copyright ownership.
!  * The ASF licenses this file to You under the Apache License, Version 2.0 
!  * (the "License"); you may not use this file except in compliance with 
!  * the License. You may obtain a copy of the License at
!  *
!  *     http://www.apache.org/licenses/LICENSE-2.0
!  *
!  * Unless required by applicable law or agreed to in writing, software
!  * distributed under the License is distributed on an "AS IS" BASIS,
!  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
!  * See the License for the specific language governing permissions and
!  * limitations under the License.
!  */
  
  package opennlp.perceptron;
***************
*** 28,34 ****
  /**
   * Model writer that saves models in binary format.
-  *
-  * @author      Jason Baldridge
-  * @version     $Revision$, $Date$
   */
  public class BinaryPerceptronModelWriter extends PerceptronModelWriter {
--- 28,31 ----

[Maxent-commit] CVS: maxent/src/main/java/opennlp/model Sequence.java, NONE, 1.1 SequenceStream.java, NONE, 1.1 SequenceStreamEventStream.java, NONE, 1.1

[Thomas M. <tsm...@us...>]

Update of /cvsroot/maxent/maxent/src/main/java/opennlp/model
In directory 23jxhf1.ch3.sourceforge.com:/tmp/cvs-serv18050/src/main/java/opennlp/model

Added Files:
	Sequence.java SequenceStream.java 
	SequenceStreamEventStream.java 
Log Message:
Classes to model sequence data.

--- NEW FILE: Sequence.java ---
/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0 
 * (the "License"); you may not use this file except in compliance with 
 * the License. You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package opennlp.model;

/**
 * Class which models a sequence. 
 * @param <T> The type of the object which is the source of this sequence.
 */
public class Sequence<T> {

	private Event[] events;
	private T source;
	
	/**
	 * Creates a new sequence made up of the specified events and derived from
	 * the specified source.
	 * @param events The events of the sequence.
	 * @param source The source object for this sequence.
	 */
	public Sequence(Event[] events, T source) {
		this.events = events;
		this.source = source;
	}
	
	/**
	 * Returns the events which make up this sequence.
	 * @return the events which make up this sequence.
	 */
	public Event[] getEvents() {
		return events;
	}
	
	/**
	 * Returns an object from which this sequence can be derived.
	 * This object is used when the events for this sequence need to be
	 * re-derived such as in a call to SequenceStream.updateContext. 
	 * @return an object from which this sequence can be derived.
	 */
	public T getSource() {
		return source;
	}
}

--- NEW FILE: SequenceStream.java ---
/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0 
 * (the "License"); you may not use this file except in compliance with 
 * the License. You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package opennlp.model;

/**
 *  Interface for streams of sequences used to train sequence models. 
 */
public interface SequenceStream extends Iterable<Sequence> {	
  /**
   * Creates a new event array based on the outcomes predicted by the specified parameters 
   * for the specified sequence.
   * @param sequence The sequence to be evaluated.
   * @param ep The parameters of the current model.
   * @return
   */
  public Event[] updateContext(Sequence sequence, AbstractModel model);
  
}

--- NEW FILE: SequenceStreamEventStream.java ---
package opennlp.model;

import java.util.Iterator;

/**
 * Class which turns a sequence stream into an event stream. 
 *
 */
public class SequenceStreamEventStream implements EventStream {

  private Iterator<Sequence> sequenceIterator;
  int eventIndex = -1;
  Event[] events;
  
  public SequenceStreamEventStream(SequenceStream sequenceStream) {
    this.sequenceIterator = sequenceStream.iterator();
  }
  
  public boolean hasNext() {
    if (events != null && eventIndex < events.length) {
      return true;
    }
    else {
      if (sequenceIterator.hasNext()) {
        Sequence s = sequenceIterator.next();
        eventIndex = 0;
        events = s.getEvents();
        return true;
      }
      else {
        return false;
      }
    }
  }

  public Event next() {
    return events[eventIndex++];
  }

  public void remove() {
    throw new UnsupportedOperationException();
  }

}

[Maxent-commit] CVS: maxent/src/main/java/opennlp/model TwoPassDataIndexer.java, 1.1, 1.2

[Thomas M. <tsm...@us...>]

Update of /cvsroot/maxent/maxent/src/main/java/opennlp/model
In directory 23jxhf1.ch3.sourceforge.com:/tmp/cvs-serv17513/src/main/java/opennlp/model

Modified Files:
	TwoPassDataIndexer.java 
Log Message:
Makes messages reflect use of sorting or not.

Index: TwoPassDataIndexer.java
===================================================================
RCS file: /cvsroot/maxent/maxent/src/main/java/opennlp/model/TwoPassDataIndexer.java,v
retrieving revision 1.1
retrieving revision 1.2
diff -C2 -d -r1.1 -r1.2
*** TwoPassDataIndexer.java	22 Jan 2009 23:23:33 -0000	1.1
--- TwoPassDataIndexer.java	15 Mar 2009 03:25:05 -0000	1.2
***************
*** 88,92 ****
        System.out.println("done.");
  
!       System.out.print("Sorting and merging events... ");
        sortAndMerge(eventsToCompare,sort);
        System.out.println("Done indexing.");
--- 88,97 ----
        System.out.println("done.");
  
!       if (sort) { 
!         System.out.print("Sorting and merging events... ");
!       }
!       else {
!         System.out.print("Collecting events... ");
!       }
        sortAndMerge(eventsToCompare,sort);
        System.out.println("Done indexing.");

[Maxent-commit] CVS: maxent/src/main/java/opennlp/model GenericModelWriter.java, 1.1, 1.2 GenericModelReader.java, 1.2, 1.3

[Thomas M. <tsm...@us...>]

Update of /cvsroot/maxent/maxent/src/main/java/opennlp/model
In directory 23jxhf1.ch3.sourceforge.com:/tmp/cvs-serv16652/src/main/java/opennlp/model

Modified Files:
	GenericModelWriter.java GenericModelReader.java 
Log Message:
adds license.

Index: GenericModelWriter.java
===================================================================
RCS file: /cvsroot/maxent/maxent/src/main/java/opennlp/model/GenericModelWriter.java,v
retrieving revision 1.1
retrieving revision 1.2
diff -C2 -d -r1.1 -r1.2
*** GenericModelWriter.java	22 Jan 2009 23:23:33 -0000	1.1
--- GenericModelWriter.java	15 Mar 2009 03:24:32 -0000	1.2
***************
*** 1,2 ****
--- 1,19 ----
+ /*
+  * Licensed to the Apache Software Foundation (ASF) under one or more
+  * contributor license agreements.  See the NOTICE file distributed with
+  * this work for additional information regarding copyright ownership.
+  * The ASF licenses this file to You under the Apache License, Version 2.0 
+  * (the "License"); you may not use this file except in compliance with 
+  * the License. You may obtain a copy of the License at
+  *
+  *     http://www.apache.org/licenses/LICENSE-2.0
+  *
+  * Unless required by applicable law or agreed to in writing, software
+  * distributed under the License is distributed on an "AS IS" BASIS,
+  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  * See the License for the specific language governing permissions and
+  * limitations under the License.
+  */
+ 
  package opennlp.model;
  

Index: GenericModelReader.java
===================================================================
RCS file: /cvsroot/maxent/maxent/src/main/java/opennlp/model/GenericModelReader.java,v
retrieving revision 1.2
retrieving revision 1.3
diff -C2 -d -r1.2 -r1.3
*** GenericModelReader.java	15 Mar 2009 03:13:47 -0000	1.2
--- GenericModelReader.java	15 Mar 2009 03:24:32 -0000	1.3
***************
*** 1,2 ****
--- 1,19 ----
+ /*
+  * Licensed to the Apache Software Foundation (ASF) under one or more
+  * contributor license agreements.  See the NOTICE file distributed with
+  * this work for additional information regarding copyright ownership.
+  * The ASF licenses this file to You under the Apache License, Version 2.0 
+  * (the "License"); you may not use this file except in compliance with 
+  * the License. You may obtain a copy of the License at
+  *
+  *     http://www.apache.org/licenses/LICENSE-2.0
+  *
+  * Unless required by applicable law or agreed to in writing, software
+  * distributed under the License is distributed on an "AS IS" BASIS,
+  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  * See the License for the specific language governing permissions and
+  * limitations under the License.
+  */
+ 
  package opennlp.model;
  

[Maxent-commit] CVS: maxent/src/main/java/opennlp/model GenericModelReader.java, 1.1, 1.2

[Thomas M. <tsm...@us...>]

Update of /cvsroot/maxent/maxent/src/main/java/opennlp/model
In directory 23jxhf1.ch3.sourceforge.com:/tmp/cvs-serv31018/src/main/java/opennlp/model

Modified Files:
	GenericModelReader.java 
Log Message:
added main to convert models.

Index: GenericModelReader.java
===================================================================
RCS file: /cvsroot/maxent/maxent/src/main/java/opennlp/model/GenericModelReader.java,v
retrieving revision 1.1
retrieving revision 1.2
diff -C2 -d -r1.1 -r1.2
*** GenericModelReader.java	22 Jan 2009 23:23:33 -0000	1.1
--- GenericModelReader.java	15 Mar 2009 03:13:47 -0000	1.2
***************
*** 36,38 ****
--- 36,43 ----
      return delegateModelReader.constructModel();
    }
+   
+   public static void main(String[] args) throws IOException {
+     AbstractModel m =  new GenericModelReader(new File(args[0])).getModel();
+     new GenericModelWriter( m, new File(args[1])).persist();
+   }
  }

[Maxent-commit] CVS: maxent/src/main/java/opennlp/model ComparablePredicate.java, 1.1, 1.2

[Thomas M. <tsm...@us...>]

Update of /cvsroot/maxent/maxent/src/main/java/opennlp/model
In directory 23jxhf1.ch3.sourceforge.com:/tmp/cvs-serv30597/src/main/java/opennlp/model

Modified Files:
	ComparablePredicate.java 
Log Message:
reformatted.

Index: ComparablePredicate.java
===================================================================
RCS file: /cvsroot/maxent/maxent/src/main/java/opennlp/model/ComparablePredicate.java,v
retrieving revision 1.1
retrieving revision 1.2
diff -C2 -d -r1.1 -r1.2
*** ComparablePredicate.java	22 Jan 2009 23:23:33 -0000	1.1
--- ComparablePredicate.java	15 Mar 2009 03:12:11 -0000	1.2
***************
*** 26,62 ****
   * @version $Revision$, $Date$
   */
! public class ComparablePredicate implements Comparable {
!     public String name;
!     public int[] outcomes;
!     public double[] params;
!     
!     public ComparablePredicate(String n, int[] ocs, double[] ps) {
! 	name = n;
! 	outcomes = ocs;
! 	params = ps;
!     }
  
!     public int compareTo(Object o) {
! 	ComparablePredicate cp = (ComparablePredicate)o;
  
! 	int smallerLength = (outcomes.length > cp.outcomes.length?
! 			     cp.outcomes.length : outcomes.length);
  
! 	for (int i=0; i<smallerLength; i++) {
! 	    if (outcomes[i] < cp.outcomes[i]) return -1;
! 	    else if (outcomes[i] > cp.outcomes[i]) return 1;
! 	}
  
! 	if (outcomes.length < cp.outcomes.length) return -1;
! 	else if (outcomes.length > cp.outcomes.length) return 1;
  
! 	return 0;
!     }
  
!     public String toString() {
! 	String s = "";
! 	for (int i=0; i<outcomes.length; i++) s+= " "+outcomes[i];
! 	return s;
!     }
  
  }
--- 26,60 ----
   * @version $Revision$, $Date$
   */
! public class ComparablePredicate implements Comparable<ComparablePredicate> {
!   public String name;
!   public int[] outcomes;
!   public double[] params;
  
!   public ComparablePredicate(String n, int[] ocs, double[] ps) {
!     name = n;
!     outcomes = ocs;
!     params = ps;
!   }
  
!   public int compareTo(ComparablePredicate cp) {
!     int smallerLength = (outcomes.length > cp.outcomes.length?
!         cp.outcomes.length : outcomes.length);
  
!     for (int i=0; i<smallerLength; i++) {
!       if (outcomes[i] < cp.outcomes[i]) return -1;
!       else if (outcomes[i] > cp.outcomes[i]) return 1;
!     }
  
!     if (outcomes.length < cp.outcomes.length) return -1;
!     else if (outcomes.length > cp.outcomes.length) return 1;
  
!     return 0;
!   }
  
!   public String toString() {
!     String s = "";
!     for (int i=0; i<outcomes.length; i++) s+= " "+outcomes[i];
!     return s;
!   }
  
  }

[Maxent-commit] CVS: maxent/src/main/java/opennlp/model AbstractModel.java, 1.2, 1.3

[Thomas M. <tsm...@us...>]

Update of /cvsroot/maxent/maxent/src/main/java/opennlp/model
In directory 23jxhf1.ch3.sourceforge.com:/tmp/cvs-serv30453/src/main/java/opennlp/model

Modified Files:
	AbstractModel.java 
Log Message:
minor javadoc additions.

Index: AbstractModel.java
===================================================================
RCS file: /cvsroot/maxent/maxent/src/main/java/opennlp/model/AbstractModel.java,v
retrieving revision 1.2
retrieving revision 1.3
diff -C2 -d -r1.2 -r1.3
*** AbstractModel.java	6 Feb 2009 03:39:20 -0000	1.2
--- AbstractModel.java	15 Mar 2009 03:11:44 -0000	1.3
***************
*** 28,34 ****
--- 28,39 ----
    /** The names of the outcomes. */
    protected String[] ocNames;
+   /** Parameters for the model. */
    protected EvalParameters evalParams;
+   /** Prior distribution for this model. */
    protected Prior prior;
+   
    public enum ModelType {Maxent,Perceptron};
+   
+   /** The type of the model. */
    protected ModelType modelType;
  

[Maxent-commit] CVS: maxent/src/main/java/opennlp/model AbstractDataIndexer.java, 1.1, 1.2

[Thomas M. <tsm...@us...>]

Update of /cvsroot/maxent/maxent/src/main/java/opennlp/model
In directory 23jxhf1.ch3.sourceforge.com:/tmp/cvs-serv30084/src/main/java/opennlp/model

Modified Files:
	AbstractDataIndexer.java 
Log Message:
made comment reflect whether data was being sorted.

Index: AbstractDataIndexer.java
===================================================================
RCS file: /cvsroot/maxent/maxent/src/main/java/opennlp/model/AbstractDataIndexer.java,v
retrieving revision 1.1
retrieving revision 1.2
diff -C2 -d -r1.1 -r1.2
*** AbstractDataIndexer.java	22 Jan 2009 23:23:33 -0000	1.1
--- AbstractDataIndexer.java	15 Mar 2009 03:10:20 -0000	1.2
***************
*** 105,109 ****
        numUniqueEvents = eventsToCompare.size();
      }
!     System.out.println("done. Reduced " + numEvents + " events to " + numUniqueEvents + ".");
  
      contexts = new int[numUniqueEvents][];
--- 105,109 ----
        numUniqueEvents = eventsToCompare.size();
      }
!     if (sort) System.out.println("done. Reduced " + numEvents + " events to " + numUniqueEvents + ".");
  
      contexts = new int[numUniqueEvents][];

[Maxent-commit] CVS: maxent/src/main/java/opennlp/maxent GISTrainer.java, 1.1, 1.2

[Thomas M. <tsm...@us...>]

Update of /cvsroot/maxent/maxent/src/main/java/opennlp/maxent
In directory 23jxhf1.ch3.sourceforge.com:/tmp/cvs-serv29904/src/main/java/opennlp/maxent

Modified Files:
	GISTrainer.java 
Log Message:
fixed typos in javadoc and license.

Index: GISTrainer.java
===================================================================
RCS file: /cvsroot/maxent/maxent/src/main/java/opennlp/maxent/GISTrainer.java,v
retrieving revision 1.1
retrieving revision 1.2
diff -C2 -d -r1.1 -r1.2
*** GISTrainer.java	22 Jan 2009 23:23:34 -0000	1.1
--- GISTrainer.java	15 Mar 2009 03:09:47 -0000	1.2
***************
*** 1,5 ****
  /*
   * Licensed to the Apache Software Foundation (ASF) under one or more
!  * contributor license agreemnets.  See the NOTICE file distributed with
   * this work for additional information regarding copyright ownership.
   * The ASF licenses this file to You under the Apache License, Version 2.0 
--- 1,5 ----
  /*
   * Licensed to the Apache Software Foundation (ASF) under one or more
!  * contributor license agreements.  See the NOTICE file distributed with
   * this work for additional information regarding copyright ownership.
   * The ASF licenses this file to You under the Apache License, Version 2.0 
***************
*** 41,45 ****
   * 
   * A prior can be used to train models which converge to the distribution which minimizes the
!  * relative entropy between the distribution specified by the emperical constraints of the training
   * data and the specified prior.  By default, the uniform distribution is used as the prior.
   *    
--- 41,45 ----
   * 
   * A prior can be used to train models which converge to the distribution which minimizes the
!  * relative entropy between the distribution specified by the empirical constraints of the training
   * data and the specified prior.  By default, the uniform distribution is used as the prior.
   *    

[Maxent-commit] CVS: maxent/src/main/java/opennlp/maxent GIS.java, 1.1, 1.2

[Thomas M. <tsm...@us...>]

Update of /cvsroot/maxent/maxent/src/main/java/opennlp/maxent
In directory 23jxhf1.ch3.sourceforge.com:/tmp/cvs-serv29643/src/main/java/opennlp/maxent

Modified Files:
	GIS.java 
Log Message:
fixed bug not passing cut off.
fixed type in license.

Index: GIS.java
===================================================================
RCS file: /cvsroot/maxent/maxent/src/main/java/opennlp/maxent/GIS.java,v
retrieving revision 1.1
retrieving revision 1.2
diff -C2 -d -r1.1 -r1.2
*** GIS.java	22 Jan 2009 23:23:34 -0000	1.1
--- GIS.java	15 Mar 2009 03:09:05 -0000	1.2
***************
*** 1,5 ****
  /*
   * Licensed to the Apache Software Foundation (ASF) under one or more
!  * contributor license agreemnets.  See the NOTICE file distributed with
   * this work for additional information regarding copyright ownership.
   * The ASF licenses this file to You under the Apache License, Version 2.0 
--- 1,5 ----
  /*
   * Licensed to the Apache Software Foundation (ASF) under one or more
!  * contributor license agreements.  See the NOTICE file distributed with
   * this work for additional information regarding copyright ownership.
   * The ASF licenses this file to You under the Apache License, Version 2.0 
***************
*** 167,171 ****
        }
        else {
!         return trainer.trainModel(iterations, indexer,0);
        }
      }  
--- 167,171 ----
        }
        else {
!         return trainer.trainModel(iterations, indexer,cutoff);
        }
      }  

[Maxent-commit] CVS: maxent/src/main/java/opennlp/model AbstractModel.java, 1.1, 1.2

[Thomas M. <tsm...@us...>]

Update of /cvsroot/maxent/maxent/src/main/java/opennlp/model
In directory 23jxhf1.ch3.sourceforge.com:/tmp/cvs-serv25320/src/main/java/opennlp/model

Modified Files:
	AbstractModel.java 
Log Message:
Fixed bug with switched parameters.
Updated test to show results before checking assert.

Index: AbstractModel.java
===================================================================
RCS file: /cvsroot/maxent/maxent/src/main/java/opennlp/model/AbstractModel.java,v
retrieving revision 1.1
retrieving revision 1.2
diff -C2 -d -r1.1 -r1.2
*** AbstractModel.java	22 Jan 2009 23:23:33 -0000	1.1
--- AbstractModel.java	6 Feb 2009 03:39:20 -0000	1.2
***************
*** 40,44 ****
    public AbstractModel(Context[] params, String[] predLabels, String[] outcomeNames, int correctionConstant,double correctionParam) {
      init(predLabels,outcomeNames);
!     this.evalParams = new EvalParameters(params,correctionConstant,correctionParam,ocNames.length);
    }
    
--- 40,44 ----
    public AbstractModel(Context[] params, String[] predLabels, String[] outcomeNames, int correctionConstant,double correctionParam) {
      init(predLabels,outcomeNames);
!     this.evalParams = new EvalParameters(params,correctionParam,correctionConstant,ocNames.length);
    }
    

[Maxent-commit] CVS: maxent/src/test/java/opennlp/maxent RealValueModelTest.java, 1.1, 1.2

[Thomas M. <tsm...@us...>]

Update of /cvsroot/maxent/maxent/src/test/java/opennlp/maxent
In directory 23jxhf1.ch3.sourceforge.com:/tmp/cvs-serv25320/src/test/java/opennlp/maxent

Modified Files:
	RealValueModelTest.java 
Log Message:
Fixed bug with switched parameters.
Updated test to show results before checking assert.

Index: RealValueModelTest.java
===================================================================
RCS file: /cvsroot/maxent/maxent/src/test/java/opennlp/maxent/RealValueModelTest.java,v
retrieving revision 1.1
retrieving revision 1.2
diff -C2 -d -r1.1 -r1.2
*** RealValueModelTest.java	22 Jan 2009 23:23:35 -0000	1.1
--- RealValueModelTest.java	6 Feb 2009 03:39:21 -0000	1.2
***************
*** 24,30 ****
      assertEquals(realResults.length, repeatResults.length);

      for(int i=0; i<realResults.length; i++) {

-       assertEquals(realResults[i], repeatResults[i], 0.01f);

        System.out.println(String.format("classifiy with realModel: %1$s = %2$f", realModel.getOutcome(i), realResults[i]));

        System.out.println(String.format("classifiy with repeatModel: %1$s = %2$f", repeatModel.getOutcome(i), repeatResults[i]));

      }

  

--- 24,30 ----
      assertEquals(realResults.length, repeatResults.length);

      for(int i=0; i<realResults.length; i++) {

        System.out.println(String.format("classifiy with realModel: %1$s = %2$f", realModel.getOutcome(i), realResults[i]));

        System.out.println(String.format("classifiy with repeatModel: %1$s = %2$f", repeatModel.getOutcome(i), repeatResults[i]));

+       assertEquals(realResults[i], repeatResults[i], 0.01f);

      }

  

***************
*** 36,42 ****
      assertEquals(realResults.length, repeatResults.length);

      for(int i=0; i<realResults.length; i++) {

-       assertEquals(realResults[i], repeatResults[i], 0.01f);

        System.out.println(String.format("classifiy with realModel: %1$s = %2$f", realModel.getOutcome(i), realResults[i]));

        System.out.println(String.format("classifiy with repeatModel: %1$s = %2$f", repeatModel.getOutcome(i), repeatResults[i]));

      }

  

--- 36,42 ----
      assertEquals(realResults.length, repeatResults.length);

      for(int i=0; i<realResults.length; i++) {

        System.out.println(String.format("classifiy with realModel: %1$s = %2$f", realModel.getOutcome(i), realResults[i]));

        System.out.println(String.format("classifiy with repeatModel: %1$s = %2$f", repeatModel.getOutcome(i), repeatResults[i]));

+       assertEquals(realResults[i], repeatResults[i], 0.01f);      

      }

  

[Maxent-commit] CVS: maxent/src/main/java/opennlp/model Context.java, 1.1, 1.2

[Joern K. <joe...@us...>]

Update of /cvsroot/maxent/maxent/src/main/java/opennlp/model
In directory 23jxhf1.ch3.sourceforge.com:/tmp/cvs-serv28135/src/main/java/opennlp/model

Modified Files:
	Context.java 
Log Message:
typos


Index: Context.java
===================================================================
RCS file: /cvsroot/maxent/maxent/src/main/java/opennlp/model/Context.java,v
retrieving revision 1.1
retrieving revision 1.2
diff -C2 -d -r1.1 -r1.2
*** Context.java	22 Jan 2009 23:23:33 -0000	1.1
--- Context.java	5 Feb 2009 13:08:23 -0000	1.2
***************
*** 19,25 ****
  
  /**
!  * Class which associates a real valueed parameter or expected value with a particular contextual
!  * predicate or feature.  This is used to store maxent model parameters as well as model and emperical
!  * expected values.  
   * @author Tom Morton
   *
--- 19,25 ----
  
  /**
!  * Class which associates a real valued parameter or expected value with a particular contextual
!  * predicate or feature.  This is used to store maxent model parameters as well as model and empirical
!  * expected values.
   * @author Tom Morton
   *
***************
*** 33,40 ****
    
    /**
!    * Creates a new parametes object with the specifed parameters associated with the specified
     * outcome pattern.
     * @param outcomePattern Array of outcomes for which parameters exists for this context.
!    * @param parameters Paramaters for the outcomes specified.
     */
    public Context(int[] outcomePattern, double[] parameters) {
--- 33,40 ----
    
    /**
!    * Creates a new parameters object with the specified parameters associated with the specified
     * outcome pattern.
     * @param outcomePattern Array of outcomes for which parameters exists for this context.
!    * @param parameters Parameters for the outcomes specified.
     */
    public Context(int[] outcomePattern, double[] parameters) {
***************
*** 52,57 ****
    
    /**
!    * Returns the paramaters or expected values for the outcomes which occur with this context.
!    * @return Array of paramaters for the outcomes of this context.
     */
    public double[] getParameters() {
--- 52,57 ----
    
    /**
!    * Returns the parameters or expected values for the outcomes which occur with this context.
!    * @return Array of parameters for the outcomes of this context.
     */
    public double[] getParameters() {

[Maxent-commit] CVS: maxent/src/main/java/opennlp/model DataIndexer.java, 1.1, 1.2

[Joern K. <joe...@us...>]

Update of /cvsroot/maxent/maxent/src/main/java/opennlp/model
In directory 23jxhf1.ch3.sourceforge.com:/tmp/cvs-serv22855/src/main/java/opennlp/model

Modified Files:
	DataIndexer.java 
Log Message:
typo

Index: DataIndexer.java
===================================================================
RCS file: /cvsroot/maxent/maxent/src/main/java/opennlp/model/DataIndexer.java,v
retrieving revision 1.1
retrieving revision 1.2
diff -C2 -d -r1.1 -r1.2
*** DataIndexer.java	22 Jan 2009 23:23:33 -0000	1.1
--- DataIndexer.java	5 Feb 2009 12:32:36 -0000	1.2
***************
*** 23,27 ****
    /**
     * Returns the array of predicates seen in each event. 
!    * @return a 2-D array whose first dimenstion is the event index and array this refers to contains
     * the contexts for that event. 
     */
--- 23,27 ----
    /**
     * Returns the array of predicates seen in each event. 
!    * @return a 2-D array whose first dimension is the event index and array this refers to contains
     * the contexts for that event. 
     */

[Maxent-commit] CVS: maxent/src/main/java/opennlp/model EventCollector.java, NONE, 1.1 Event.java, NONE, 1.1 Context.java, NONE, 1.1 AbstractModel.java, NONE, 1.1 UniformPrior.java, NONE, 1.1 GenericModelReader.java, NONE, 1.1 PlainTextFileDataReader.java, NONE, 1.1 EventStream.java, NONE, 1.1 GenericModelWriter.java, NONE, 1.1 AbstractModelWriter.java, NONE, 1.1 RealValueFileEventStream.java, NONE, 1.1 Prior.java, NONE, 1.1 EventCollectorAsStream.java, NONE, 1.1 ObjectDataReader.java, NONE, 1.1 AbstractEventStream.java, NONE, 1.1 AbstractDataIndexer.java, NONE, 1.1 MaxentModel.java, NONE, 1.1 FileEventStream.java, NONE, 1.1 OnePassRealValueDataIndexer.java, NONE, 1.1 ComparablePredicate.java, NONE, 1.1 TwoPassDataIndexer.java, NONE, 1.1 OnePassDataIndexer.java, NONE, 1.1 DataReader.java, NONE, 1.1 ComparableEvent.java, NONE, 1.1 EvalParameters.java, NONE, 1.1 DynamicEvalParameters.java, NONE, 1.1 AbstractModelReader.java, NONE, 1.1 DataIndexer.java, NONE, 1.1 BinaryFileDataReader.java, NONE, 1.1 MutableContext.java, NONE, 1.1

[Joern K. <joe...@us...>]

Update of /cvsroot/maxent/maxent/src/main/java/opennlp/model
In directory 23jxhf1.ch3.sourceforge.com:/tmp/cvs-serv21001/src/main/java/opennlp/model

Added Files:
	EventCollector.java Event.java Context.java AbstractModel.java 
	UniformPrior.java GenericModelReader.java 
	PlainTextFileDataReader.java EventStream.java 
	GenericModelWriter.java AbstractModelWriter.java 
	RealValueFileEventStream.java Prior.java 
	EventCollectorAsStream.java ObjectDataReader.java 
	AbstractEventStream.java AbstractDataIndexer.java 
	MaxentModel.java FileEventStream.java 
	OnePassRealValueDataIndexer.java ComparablePredicate.java 
	TwoPassDataIndexer.java OnePassDataIndexer.java 
	DataReader.java ComparableEvent.java EvalParameters.java 
	DynamicEvalParameters.java AbstractModelReader.java 
	DataIndexer.java BinaryFileDataReader.java MutableContext.java 
Log Message:
Introduced maven build.

--- NEW FILE: EventCollector.java ---
/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreemnets.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0 
 * (the "License"); you may not use this file except in compliance with 
 * the License. You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package opennlp.model;



/**
 * An interface for objects which read events during training.
 *
 * @author      Jason Baldridge
 * @version $Revision: 1.1 $, $Date: 2009/01/22 23:23:33 $
 */
public interface EventCollector {

    /**
     * Return the events which this EventCollector has gathered.  It must get
     * its data from a constructor.
     *
     * @return the events that this EventCollector has gathered
     */
    public Event[] getEvents();

    /**
     * Return the events which this EventCollector has gathered based on
     * whether we wish to train a model or evaluate one based on those
     * events.
     * 
     * @param evalMode true if we are evaluating based on the events, false if
     *                 we are training.
     * @return the events that this EventCollector has gathered
     */
    public Event[] getEvents(boolean evalMode);
}

--- NEW FILE: Event.java ---
/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreemnets.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0 
 * (the "License"); you may not use this file except in compliance with 
 * the License. You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package opennlp.model;


/**
 * The context of a decision point during training.  This includes
 * contextual predicates and an outcome.
 *
 * @author      Jason Baldridge
 * @version     $Revision: 1.1 $, $Date: 2009/01/22 23:23:33 $
 */
public class Event {
    private String outcome;
    private String[] context;
    private float[] values;
    
    public Event(String outcome, String[] context) {
      this(outcome,context,null);
    }
    
    public Event(String outcome, String[] context, float[] values) {
      this.outcome = outcome;
      this.context = context;
      this.values = values;
    }
    
    public String getOutcome() { 
      return outcome; 
    }
    
    public String[] getContext() { 
      return context; 
    }
    
    public float[] getValues() {
      return values;
    }
    
    public String toString() {
      StringBuffer sb = new StringBuffer();
      sb.append(outcome).append(" [");
      if (context.length > 0) {
        sb.append(context[0]);
        if (values != null) {
          sb.append("="+values[0]);
        }
      }
      for (int ci=1;ci<context.length;ci++) {
        sb.append(" ").append(context[ci]);
        if (values != null) {
          sb.append("="+values[ci]);
        }
      }
      sb.append("]");
      return sb.toString();
    }
    
}

--- NEW FILE: Context.java ---
/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreemnets.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0 
 * (the "License"); you may not use this file except in compliance with 
 * the License. You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package opennlp.model;

/**
 * Class which associates a real valueed parameter or expected value with a particular contextual
 * predicate or feature.  This is used to store maxent model parameters as well as model and emperical
 * expected values.  
 * @author Tom Morton
 *
 */
public class Context {

  /** The real valued parameters or expected values for this context. */
  protected double[] parameters;
  /** The outcomes which occur with this context. */
  protected int[] outcomes;
  
  /**
   * Creates a new parametes object with the specifed parameters associated with the specified
   * outcome pattern.
   * @param outcomePattern Array of outcomes for which parameters exists for this context.
   * @param parameters Paramaters for the outcomes specified.
   */
  public Context(int[] outcomePattern, double[] parameters) {
    this.outcomes = outcomePattern;
    this.parameters = parameters;
  }
  
  /**
   * Returns the outcomes for which parameters exists for this context.
   * @return Array of outcomes for which parameters exists for this context.
   */
  public int[] getOutcomes() {
    return outcomes;
  }
  
  /**
   * Returns the paramaters or expected values for the outcomes which occur with this context.
   * @return Array of paramaters for the outcomes of this context.
   */
  public double[] getParameters() {
    return parameters;
  }
}

--- NEW FILE: AbstractModel.java ---
/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreemnets.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0 
 * (the "License"); you may not use this file except in compliance with 
 * the License. You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package opennlp.model;

import java.text.DecimalFormat;
import java.util.HashMap;
import java.util.Map;

public abstract class AbstractModel implements MaxentModel {

  /** Maping between predicates/contexts and an integer representing them. */
  protected Map<String,Integer> pmap;
  /** The names of the outcomes. */
  protected String[] ocNames;
  protected EvalParameters evalParams;
  protected Prior prior;
  public enum ModelType {Maxent,Perceptron};
  protected ModelType modelType;

  public AbstractModel(Context[] params, String[] predLabels, String[] outcomeNames) {
    init(predLabels,outcomeNames);
    this.evalParams = new EvalParameters(params,ocNames.length);
  }

  public AbstractModel(Context[] params, String[] predLabels, String[] outcomeNames, int correctionConstant,double correctionParam) {
    init(predLabels,outcomeNames);
    this.evalParams = new EvalParameters(params,correctionConstant,correctionParam,ocNames.length);
  }
  
  private void init(String[] predLabels, String[] outcomeNames){
    this.pmap = new HashMap<String,Integer>(predLabels.length);
    for (int i=0; i<predLabels.length; i++) {
      pmap.put(predLabels[i], i);
    }
    this.ocNames =  outcomeNames;
  }


  /**
   * Return the name of the outcome corresponding to the highest likelihood
   * in the parameter ocs.
   *
   * @param ocs A double[] as returned by the eval(String[] context)
   *            method.
   * @return    The name of the most likely outcome.
   */
  public final String getBestOutcome(double[] ocs) {
      int best = 0;
      for (int i = 1; i<ocs.length; i++)
          if (ocs[i] > ocs[best]) best = i;
      return ocNames[best];
  }
  
  public ModelType getModelType(){
    return modelType;
  }

  /**
   * Return a string matching all the outcome names with all the
   * probabilities produced by the <code>eval(String[] context)</code>
   * method.
   *
   * @param ocs A <code>double[]</code> as returned by the
   *            <code>eval(String[] context)</code>
   *            method.
   * @return    String containing outcome names paired with the normalized
   *            probability (contained in the <code>double[] ocs</code>)
   *            for each one.
   */
  public final String getAllOutcomes(double[] ocs) {
      if (ocs.length != ocNames.length) {
          return "The double array sent as a parameter to GISModel.getAllOutcomes() must not have been produced by this model.";
      }
      else {
        DecimalFormat df =  new DecimalFormat("0.0000");
        StringBuffer sb = new StringBuffer(ocs.length*2);
        sb.append(ocNames[0]).append("[").append(df.format(ocs[0])).append("]");
        for (int i = 1; i<ocs.length; i++) {
          sb.append("  ").append(ocNames[i]).append("[").append(df.format(ocs[i])).append("]");
        }
        return sb.toString();
      }
  }

  /**
   * Return the name of an outcome corresponding to an int id.
   *
   * @param i An outcome id.
   * @return  The name of the outcome associated with that id.
   */
  public final String getOutcome(int i) {
      return ocNames[i];
  }

  /**
   * Gets the index associated with the String name of the given outcome.
   *
   * @param outcome the String name of the outcome for which the
   *          index is desired
   * @return the index if the given outcome label exists for this
   * model, -1 if it does not.
   **/
  public int getIndex(String outcome) {
      for (int i=0; i<ocNames.length; i++) {
          if (ocNames[i].equals(outcome))
              return i;
      }
      return -1;
  }

  public int getNumOutcomes() {
    return(evalParams.getNumOutcomes());
  }

  /**
   * Provides the fundamental data structures which encode the maxent model
   * information.  This method will usually only be needed by
   * GISModelWriters.  The following values are held in the Object array
   * which is returned by this method:
   *
   * <li>index 0: opennlp.maxent.Context[] containing the model
   *            parameters  
   * <li>index 1: java.util.Map containing the mapping of model predicates
   *            to unique integers
   * <li>index 2: java.lang.String[] containing the names of the outcomes,
   *            stored in the index of the array which represents their
   * 	          unique ids in the model.
   * <li>index 3: java.lang.Integer containing the value of the models
   *            correction constant
   * <li>index 4: java.lang.Double containing the value of the models
   *            correction parameter
   *
   * @return An Object[] with the values as described above.
   */
  public final Object[] getDataStructures() {
      Object[] data = new Object[5];
      data[0] = evalParams.getParams();
      data[1] = pmap;
      data[2] = ocNames;
      data[3] = new Integer((int)evalParams.getCorrectionConstant());
      data[4] = new Double(evalParams.getCorrectionParam());
      return data;
  }
}

--- NEW FILE: UniformPrior.java ---
/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreemnets.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0 
 * (the "License"); you may not use this file except in compliance with 
 * the License. You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package opennlp.model;

/**
 * Provide a maximum entropy model with a uniform prior.
 * @author Tom Morton
 *
 */
public class UniformPrior implements Prior {

  private int numOutcomes;
  private double r;
    
  public void logPrior(double[] dist, int[] context, float[] values) {
    for (int oi=0;oi<numOutcomes;oi++) {
      dist[oi] = r;
    }
  }
  
  public void logPrior(double[] dist, int[] context) {
    logPrior(dist,context,null);
  }

  public void setLabels(String[] outcomeLabels, String[] contextLabels) {
    this.numOutcomes = outcomeLabels.length;
    r = Math.log(1.0/numOutcomes);
  }
}

--- NEW FILE: GenericModelReader.java ---
package opennlp.model;

import java.io.File;
import java.io.IOException;

import opennlp.maxent.io.GISModelReader;
import opennlp.perceptron.PerceptronModelReader;

public class GenericModelReader extends AbstractModelReader {

  private AbstractModelReader delegateModelReader;
  
  public GenericModelReader (File f) throws IOException {
    super(f);
  }
  
  public GenericModelReader(DataReader dataReader) {
    super(dataReader);
  }
  
  public void checkModelType() throws IOException {
    String modelType = readUTF();
    if (modelType.equals("Perceptron")) {
      delegateModelReader = new PerceptronModelReader(this.dataReader);
    }
    else if (modelType.equals("GIS")) {
      delegateModelReader = new GISModelReader(this.dataReader);
    }
    else {
      throw new IOException("Unknown model format: "+modelType);
    }
  }
  

  public AbstractModel constructModel() throws IOException {
    return delegateModelReader.constructModel();
  }
}

--- NEW FILE: PlainTextFileDataReader.java ---
package opennlp.model;

import java.io.BufferedInputStream;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.zip.GZIPInputStream;

public class PlainTextFileDataReader implements DataReader {

  private BufferedReader input;
  
  public PlainTextFileDataReader(File f) throws IOException {
    if (f.getName().endsWith(".gz")) {
      input = new BufferedReader(new InputStreamReader(new BufferedInputStream(new GZIPInputStream(new BufferedInputStream(new FileInputStream(f))))));
    }
    else {
      input = new BufferedReader(new InputStreamReader(new BufferedInputStream(new FileInputStream(f))));
    }
  }
  
  public PlainTextFileDataReader(InputStream in) {
    input = new BufferedReader(new InputStreamReader(in));
  }
  
  public PlainTextFileDataReader(BufferedReader in) {
    input = in;
  }
  
  public double readDouble() throws IOException {
    return Double.parseDouble(input.readLine());
  }

  public int readInt() throws IOException {
    return Integer.parseInt(input.readLine());
  }

  public String readUTF() throws IOException {
    return input.readLine();
  }

}

--- NEW FILE: EventStream.java ---
/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreemnets.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0 
 * (the "License"); you may not use this file except in compliance with 
 * the License. You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package opennlp.model;

import java.util.Iterator;

/**
 * A object which can deliver a stream of training events for the GIS
 * procedure (or others such as IIS if and when they are implemented).
 * EventStreams don't need to use opennlp.maxent.DataStreams, but doing so
 * would provide greater flexibility for producing events from data stored in
 * different formats.
 *
 * @author      Jason Baldridge
 * @version     $Revision: 1.1 $, $Date: 2009/01/22 23:23:33 $
 *
 */
public interface EventStream extends Iterator<Event>{

    /**
     * Returns the next Event object held in this EventStream.
     *
     * @return the Event object which is next in this EventStream
     */
    public Event next ();
 
    /**
     * Test whether there are any Events remaining in this EventStream.
     *
     * @return true if this EventStream has more Events
     */
    public boolean hasNext ();
    
}


--- NEW FILE: GenericModelWriter.java ---
package opennlp.model;

import java.io.BufferedWriter;
import java.io.DataOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.util.zip.GZIPOutputStream;

import opennlp.maxent.io.BinaryGISModelWriter;
import opennlp.maxent.io.PlainTextGISModelWriter;
import opennlp.model.AbstractModel.ModelType;
import opennlp.perceptron.BinaryPerceptronModelWriter;
import opennlp.perceptron.PlainTextPerceptronModelWriter;

public class GenericModelWriter extends AbstractModelWriter {

  private AbstractModelWriter delegateWriter;
  
  public GenericModelWriter(AbstractModel model, File file) throws IOException {
    String filename = file.getName();
    OutputStream os;
    // handle the zipped/not zipped distinction
    if (filename.endsWith(".gz")) {
      os = new GZIPOutputStream(new FileOutputStream(file));
      filename = filename.substring(0,filename.length()-3);
    }
    else {
      os = new FileOutputStream(file);
    }
    
    // handle the different formats
    if (filename.endsWith(".bin")) {
      init(model,new DataOutputStream(os));
    }
    else {  // filename ends with ".txt"
      init(model,new BufferedWriter(new OutputStreamWriter(os)));
    }
  }
  
  public GenericModelWriter(AbstractModel model, DataOutputStream dos) {
    init(model,dos);
  }
  
  private void init(AbstractModel model, DataOutputStream dos) {
    if (model.getModelType() == ModelType.Perceptron) {
      delegateWriter = new BinaryPerceptronModelWriter(model,dos);
    }
    else if (model.getModelType() == ModelType.Maxent) {
      delegateWriter = new BinaryGISModelWriter(model,dos);
    }
  }
  
  private void init(AbstractModel model, BufferedWriter bw) {
    if (model.getModelType() == ModelType.Perceptron) {
      delegateWriter = new PlainTextPerceptronModelWriter(model,bw);
    }
    else if (model.getModelType() == ModelType.Maxent) {
      delegateWriter = new PlainTextGISModelWriter(model,bw);
    }
  }

  @Override
  public void close() throws IOException {
    delegateWriter.close();
  }

  @Override
  public void persist() throws IOException {
    delegateWriter.persist();
  }

  @Override
  public void writeDouble(double d) throws IOException {
    delegateWriter.writeDouble(d);
  }

  @Override
  public void writeInt(int i) throws IOException {
    delegateWriter.writeInt(i);
  }

  @Override
  public void writeUTF(String s) throws IOException {
    delegateWriter.writeUTF(s);
  }
  
}

--- NEW FILE: AbstractModelWriter.java ---
package opennlp.model;


public abstract class AbstractModelWriter {

  public AbstractModelWriter() {
    super();
  }

  public abstract void writeUTF(String s) throws java.io.IOException;

  public abstract void writeInt(int i) throws java.io.IOException;

  public abstract void writeDouble(double d) throws java.io.IOException;

  public abstract void close() throws java.io.IOException;

  public abstract void persist() throws java.io.IOException;

}

--- NEW FILE: RealValueFileEventStream.java ---
/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreemnets.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0 
 * (the "License"); you may not use this file except in compliance with 
 * the License. You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package opennlp.model;

import java.io.File;
import java.io.IOException;
import java.util.StringTokenizer;

import opennlp.maxent.GIS;
import opennlp.maxent.io.SuffixSensitiveGISModelWriter;

public class RealValueFileEventStream extends FileEventStream {

  public RealValueFileEventStream(String fileName) throws IOException {
    super(fileName);
  }
  
  public RealValueFileEventStream(File file) throws IOException {
    super(file);
  }
  
  /**
   * Parses the specified contexts and re-populates context array with features and returns the values
   * for these features.
   * If all values are unspecified, then null is returned.
   * @param contexts The contexts with real values specified.
   * @return The value for each context or null if all values are unspecified.
   */
  public static float[] parseContexts(String[] contexts) {
    boolean hasRealValue = false;
    float[] values = new float[contexts.length]; 
    for (int ci = 0; ci < contexts.length; ci++) {
      int ei = contexts[ci].lastIndexOf("=");
      if (ei > 0 && ei+1 < contexts[ci].length()) {
        boolean gotReal = true;
        try {
          values[ci] = Float.parseFloat(contexts[ci].substring(ei+1));
        }
        catch (NumberFormatException e) {
          gotReal = false;
          System.err.println("Unable to determine value in context:"+contexts[ci]);
          values[ci] = 1;
        }
        if (gotReal) {
          if (values[ci] < 0) {
            throw new RuntimeException("Negitive values are not allowed: "+contexts[ci]);
          }
          contexts[ci] = contexts[ci].substring(0,ei);
          hasRealValue = true;
        }
      }
      else {
        values[ci] = 1;
      }
    }
    if (!hasRealValue) {
      values = null;
    }
    return values;
  }
    
  public Event next() {
    int si = line.indexOf(' ');
    String outcome = line.substring(0,si);
    String[] contexts = line.substring(si+1).split(" ");
    float[] values = parseContexts(contexts);
    return (new Event(outcome, contexts, values));
  }  
  
  /**
   * Trains and writes a model based on the events in the specified event file.
   * the name of the model created is based on the event file name.
   * @param args eventfile [iterations cuttoff]
   * @throws IOException when the eventfile can not be read or the model file can not be written.
   */
  public static void main(String[] args) throws IOException {
    if (args.length == 0) {
      System.err.println("Usage: RealValueFileEventStream eventfile [iterations cutoff]");
      System.exit(1);
    }
    int ai=0;
    String eventFile = args[ai++];
    EventStream es = new RealValueFileEventStream(eventFile);
    int iterations = 100;
    int cutoff = 5;
    if (ai < args.length) {
      iterations = Integer.parseInt(args[ai++]);
      cutoff = Integer.parseInt(args[ai++]);
    }
    AbstractModel model = GIS.trainModel(iterations,new OnePassRealValueDataIndexer(es,cutoff));
    new SuffixSensitiveGISModelWriter(model, new File(eventFile+".bin.gz")).persist();
  }
}

--- NEW FILE: Prior.java ---
/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreemnets.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0 
 * (the "License"); you may not use this file except in compliance with 
 * the License. You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package opennlp.model;

/**
 * This interface allows one to implement a prior distribution for use in
 * maximum entropy model training.  
 * @author Tom Morton
 *
 */
public interface Prior {
  /**
   * Populates the specified array with the the log of the distribution for the specified context.  
   * The returned array will be overwritten and needs to be re-initialized with every call to this method.  
   * @param dist An array to be populated with the log of the prior distribution.
   * @param context The indices of the contextual predicates for an event.
   */
  public void logPrior(double[] dist, int[] context);
  
  /**
   * Populates the specified array with the the log of the distribution for the specified context.  
   * The returned array will be overwritten and needs to be re-initialized with every call to this method.  
   * @param dist An array to be populated with the log of the prior distribution.
   * @param context The indices of the contextual predicates for an event.
   * @param values The values associated with the context. 
   */
  public void logPrior(double[] dist, int[] context, float[] values);

  /**
   * Method to specify the label for the outcomes and contexts.  This is used to map 
   * integer outcomes and contexts to their string values.  This method is called prior
   * to any call to #logPrior.
   * @param outcomeLabels An array of each outcome label.
   * @param contextLabels An array of each context label.
   */
  public void setLabels(String[] outcomeLabels, String[] contextLabels);
}

--- NEW FILE: EventCollectorAsStream.java ---
/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreemnets.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0 
 * (the "License"); you may not use this file except in compliance with 
 * the License. You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package opennlp.model;

/**
 * A wrapper to turn EventCollectors created for Maxent 1.0 into EventStreams
 * for Maxent 1.2.  For efficiency, it would be best to convert your
 * EventCollector into a EventStream directly, but this will allow your
 * application to work with Maxent 1.2 with very little recoding.
 *
 * @author      Jason Baldridge
 * @version     $Revision: 1.1 $, $Date: 2009/01/22 23:23:33 $
 */
public final class EventCollectorAsStream extends AbstractEventStream {
    final Event[] events;
    final int numEvents;
    int index = 0;
    
    public EventCollectorAsStream (EventCollector ec) {
	events = ec.getEvents(false);
	numEvents = events.length;
    }
    
    public Event next () {
      return events[index++];
    }
    
    public boolean hasNext () {
      return (index < numEvents);
    }
 
}

--- NEW FILE: ObjectDataReader.java ---
package opennlp.model;

import java.io.IOException;
import java.io.ObjectInputStream;

public class ObjectDataReader implements DataReader {

  protected ObjectInputStream ois;
  
  public ObjectDataReader(ObjectInputStream ois) {
    this.ois = ois;
  }
  
  public double readDouble() throws IOException {
    return ois.readDouble();
  }

  public int readInt() throws IOException {
    return ois.readInt();
  }

  public String readUTF() throws IOException {
    return ois.readUTF();
  }

}

--- NEW FILE: AbstractEventStream.java ---
package opennlp.model;


public abstract class AbstractEventStream implements  EventStream {

  public AbstractEventStream() {
    super();
  }

  public void remove() {
    throw new UnsupportedOperationException();
  }

}

--- NEW FILE: AbstractDataIndexer.java ---
/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreemnets.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0 
 * (the "License"); you may not use this file except in compliance with 
 * the License. You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package opennlp.model;

import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Set;


/**
 * Abstract class for collecting event and context counts used in training. 
 *
 */
public abstract class AbstractDataIndexer implements DataIndexer {

  private int numEvents;
  /** The integer contexts associated with each unique event. */ 
  protected int[][] contexts;
  /** The integer outcome associated with each unique event. */ 
  protected int[] outcomeList;
  /** The number of times an event occured in the training data. */
  protected int[] numTimesEventsSeen;
  /** The predicate/context names. */
  protected String[] predLabels;
  /** The names of the outcomes. */
  protected String[] outcomeLabels;
  /** The number of times each predicate occured. */
  protected int[] predCounts;

  public int[][] getContexts() {
    return contexts;
  }

  public int[] getNumTimesEventsSeen() {
    return numTimesEventsSeen;
  }

  public int[] getOutcomeList() {
    return outcomeList;
  }

  public String[] getPredLabels() {
    return predLabels;
  }

  public String[] getOutcomeLabels() {
    return outcomeLabels;
  }
  
  

  public int[] getPredCounts() {
    return predCounts;
  }

  /**
   * Sorts and uniques the array of comparable events and return the number of unique events.
   * This method will alter the eventsToCompare array -- it does an in place
   * sort, followed by an in place edit to remove duplicates.
   *
   * @param eventsToCompare a <code>ComparableEvent[]</code> value
   * @return The number of unique events in the specified list.
   * @since maxent 1.2.6
   */
  protected int sortAndMerge(List eventsToCompare, boolean sort) {
    int numUniqueEvents = 1;
    numEvents = eventsToCompare.size();
    if (sort) {
      Collections.sort(eventsToCompare);
      if (numEvents <= 1) {
        return numUniqueEvents; // nothing to do; edge case (see assertion)
      }

      ComparableEvent ce = (ComparableEvent) eventsToCompare.get(0);
      for (int i = 1; i < numEvents; i++) {
        ComparableEvent ce2 = (ComparableEvent) eventsToCompare.get(i);

        if (ce.compareTo(ce2) == 0) { 
          ce.seen++; // increment the seen count
          eventsToCompare.set(i, null); // kill the duplicate
        }
        else {
          ce = ce2; // a new champion emerges...
          numUniqueEvents++; // increment the # of unique events
        }
      }
    }
    else {
      numUniqueEvents = eventsToCompare.size();
    }
    System.out.println("done. Reduced " + numEvents + " events to " + numUniqueEvents + ".");

    contexts = new int[numUniqueEvents][];
    outcomeList = new int[numUniqueEvents];
    numTimesEventsSeen = new int[numUniqueEvents];

    for (int i = 0, j = 0; i < numEvents; i++) {
      ComparableEvent evt = (ComparableEvent) eventsToCompare.get(i);
      if (null == evt) {
        continue; // this was a dupe, skip over it.
      }
      numTimesEventsSeen[j] = evt.seen;
      outcomeList[j] = evt.outcome;
      contexts[j] = evt.predIndexes;
      ++j;
    }
    return numUniqueEvents;
  }
  
  
  public int getNumEvents() {
    return numEvents;
  }
  
  /**
   * Updates the set of predicated and counter with the specified event contexts and cutoff. 
   * @param ec The contexts/features which occur in a event.
   * @param predicateSet The set of predicates which will be used for model building.
   * @param counter The predicate counters.
   * @param cutoff The cutoff which determines whether a predicate is included.
   */
   protected static void update(String[] ec, Set predicateSet, Map<String,Integer> counter, int cutoff) {
    for (int j=0; j<ec.length; j++) {
      Integer i = counter.get(ec[j]);
      if (i == null) {
        counter.put(ec[j], 1);
      }
      else {
        counter.put(ec[j], i+1);
      }
      if (!predicateSet.contains(ec[j]) && counter.get(ec[j]) >= cutoff) {
        predicateSet.add(ec[j]);
      }
    }
  }

  /**
   * Utility method for creating a String[] array from a map whose
   * keys are labels (Strings) to be stored in the array and whose
   * values are the indices (Integers) at which the corresponding
   * labels should be inserted.
   *
   * @param labelToIndexMap a <code>TObjectIntHashMap</code> value
   * @return a <code>String[]</code> value
   * @since maxent 1.2.6
   */
  protected static String[] toIndexedStringArray(Map<String,Integer> labelToIndexMap) {
    final String[] array = new String[labelToIndexMap.size()];
    for (String label : labelToIndexMap.keySet()) {
      array[labelToIndexMap.get(label)] = label;
    }
    return array;
  }

  public float[][] getValues() {
    return null;
  }
  
  
}

--- NEW FILE: MaxentModel.java ---
/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreemnets.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0 
 * (the "License"); you may not use this file except in compliance with 
 * the License. You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package opennlp.model;

/**
 * Interface for maximum entropy models.
 *
 * @author      Jason Baldridge
 * @version     $Revision: 1.1 $, $Date: 2009/01/22 23:23:33 $
 **/
public interface MaxentModel {

  /**
   * Evaluates a context.
   *
   * @param context A list of String names of the contextual predicates
   *                which are to be evaluated together.
   * @return an array of the probabilities for each of the different
   *         outcomes, all of which sum to 1.
   *
   **/
  public double[] eval(String[] context);
  
  /**
     * Evaluates a context.
     *
     * @param context A list of String names of the contextual predicates
     *                which are to be evaluated together.
     * @param probs An array which is populated with the probabilities for each of the different
     *         outcomes, all of which sum to 1.
     * @return an array of the probabilities for each of the different outcomes, all of which sum to 1. 
     **/
  public double[] eval(String[] context, double probs[]);
  
  /**
   * Evaluates a contexts with the specified context values.
   * @param context A list of String names of the contextual predicates
     *                which are to be evaluated together.
   * @param values The values associated with each context.
   * @return an array of the probabilities for each of the different outcomes, all of which sum to 1.
   */
  public double[] eval(String[] context, float[] values);

  /**
   * Simple function to return the outcome associated with the index
   * containing the highest probability in the double[].
   *
   * @param outcomes A <code>double[]</code> as returned by the
   *            <code>eval(String[] context)</code>
   *            method.
   * @return the String name of the best outcome
   **/
  public String getBestOutcome(double[] outcomes);

  /**
   * Return a string matching all the outcome names with all the
   * probabilities produced by the <code>eval(String[]
   * context)</code> method.
   *
   * @param outcomes A <code>double[]</code> as returned by the
   *            <code>eval(String[] context)</code>
   *            method.
   * @return    String containing outcome names paired with the normalized
   *            probability (contained in the <code>double[] ocs</code>)
   *            for each one.
   **/
  public String getAllOutcomes(double[] outcomes);

  /**
   * Gets the String name of the outcome associated with the index
   * i.
   *
   * @param i the index for which the name of the associated outcome is
   *          desired.
   * @return the String name of the outcome
   **/
  public String getOutcome(int i);

  /**
   * Gets the index associated with the String name of the given
   * outcome.
   *
   * @param outcome the String name of the outcome for which the
   *          index is desired
   * @return the index if the given outcome label exists for this
   * model, -1 if it does not.
   **/
  public int getIndex(String outcome);

  /**
   * Returns the data structures relevant to storing the model.
   **/
  public Object[] getDataStructures();

  /** Returns the number of outcomes for this model.
   *  @return The number of outcomes.
   **/
  public int getNumOutcomes();

}

--- NEW FILE: FileEventStream.java ---
/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreemnets.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0 
 * (the "License"); you may not use this file except in compliance with 
 * the License. You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package opennlp.model;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.StringTokenizer;

import opennlp.maxent.GIS;
import opennlp.maxent.io.SuffixSensitiveGISModelWriter;

/** 
 * Class for using a file of events as an event stream.  The format of the file is one event perline with
 * each line consisting of outcome followed by contexts (space delimited).
 * @author Tom Morton
 *
 */
public class FileEventStream extends  AbstractEventStream {

  BufferedReader reader;
  String line;
  
  /**
   * Creates a new file event stream from the specified file name.
   * @param fileName the name fo the file containing the events.
   * @throws IOException When the specified file can not be read.
   */
  public FileEventStream(String fileName, String encoding) throws IOException {
    if (encoding == null) {
      reader = new BufferedReader(new FileReader(fileName));
    }
    else {
      reader = new BufferedReader(new InputStreamReader(new FileInputStream(fileName),encoding));
    }
  }
  
  public FileEventStream(String fileName) throws IOException {
    this(fileName,null);
  }
    
  /**
   * Creates a new file event stream from the specified file.
   * @param file the file containing the events.
   * @throws IOException When the specified file can not be read.
   */
  public FileEventStream(File file) throws IOException {
    reader = new BufferedReader(new InputStreamReader(new FileInputStream(file),"UTF8"));
  }
  
  public boolean hasNext() {
    try {
      return (null != (line = reader.readLine()));
    }
    catch (IOException e) {
      System.err.println(e);
      return (false);
    }
  }
  
  public Event next() {
    StringTokenizer st = new StringTokenizer(line);
    String outcome = st.nextToken();
    int count = st.countTokens();
    String[] context = new String[count];
    for (int ci = 0; ci < count; ci++) {
      context[ci] = st.nextToken();
    }
    return (new Event(outcome, context));
  }
  
  /**
   * Generates a string representing the specified event.
   * @param event The event for which a string representation is needed.
   * @return A string representing the specified event.
   */
  public static String toLine(Event event) {
    StringBuffer sb = new StringBuffer();
    sb.append(event.getOutcome());
    String[] context = event.getContext();
    for (int ci=0,cl=context.length;ci<cl;ci++) {
      sb.append(" "+context[ci]);
    }
    sb.append(System.getProperty("line.separator"));
    return sb.toString();
  }
  
  /**
   * Trains and writes a model based on the events in the specified event file.
   * the name of the model created is based on the event file name.
   * @param args eventfile [iterations cuttoff]
   * @throws IOException when the eventfile can not be read or the model file can not be written.
   */
  public static void main(String[] args) throws IOException {
    if (args.length == 0) {
      System.err.println("Usage: FileEventStream eventfile [iterations cutoff]");
      System.exit(1);
    }
    int ai=0;
    String eventFile = args[ai++];
    EventStream es = new FileEventStream(eventFile);
    int iterations = 100;
    int cutoff = 5;
    if (ai < args.length) {
      iterations = Integer.parseInt(args[ai++]);
      cutoff = Integer.parseInt(args[ai++]);
    }
    AbstractModel model = GIS.trainModel(es,iterations,cutoff);
    new SuffixSensitiveGISModelWriter(model, new File(eventFile+".bin.gz")).persist();
  }
}


--- NEW FILE: OnePassRealValueDataIndexer.java ---
/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreemnets.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0 
 * (the "License"); you may not use this file except in compliance with 
 * the License. You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package opennlp.model;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;


/**
 * An indexer for maxent model data which handles cutoffs for uncommon
 * contextual predicates and provides a unique integer index for each of the
 * predicates and maintains event values.  
 * @author Tom Morton
 */
public class OnePassRealValueDataIndexer extends OnePassDataIndexer {

  float[][] values;
  
  public OnePassRealValueDataIndexer(EventStream eventStream, int cutoff, boolean sort) {
    super(eventStream,cutoff,sort);
  }
  
  /**
   * Two argument constructor for DataIndexer.
   * @param eventStream An Event[] which contains the a list of all the Events
   *               seen in the training data.
   * @param cutoff The minimum number of times a predicate must have been
   *               observed in order to be included in the model.
   */
  public OnePassRealValueDataIndexer(EventStream eventStream, int cutoff) {
    super(eventStream,cutoff);
  }
  
  public float[][] getValues() {
    return values;
  }

  protected int sortAndMerge(List eventsToCompare,boolean sort) {
    int numUniqueEvents = super.sortAndMerge(eventsToCompare,sort);
    values = new float[numUniqueEvents][];
    int numEvents = eventsToCompare.size();
    for (int i = 0, j = 0; i < numEvents; i++) {
      ComparableEvent evt = (ComparableEvent) eventsToCompare.get(i);
      if (null == evt) {
        continue; // this was a dupe, skip over it.
      }
      values[j++] = evt.values;
    }
    return numUniqueEvents;
  }
  
  protected List index(LinkedList<Event> events, Map<String,Integer> predicateIndex) {
    Map<String,Integer> omap = new HashMap<String,Integer>();
    
    int numEvents = events.size();
    int outcomeCount = 0;
    List eventsToCompare = new ArrayList(numEvents);
    List<Integer> indexedContext = new ArrayList<Integer>();
    
    for (int eventIndex=0; eventIndex<numEvents; eventIndex++) {
      Event ev = (Event)events.removeFirst();
      String[] econtext = ev.getContext();
      ComparableEvent ce;
      
      int ocID;
      String oc = ev.getOutcome();
      
      if (omap.containsKey(oc)) {
        ocID = omap.get(oc);
      } else {
        ocID = outcomeCount++;
        omap.put(oc, ocID);
      }
      
      for (int i=0; i<econtext.length; i++) {
        String pred = econtext[i];
        if (predicateIndex.containsKey(pred)) {
          indexedContext.add(predicateIndex.get(pred));
        }
      }
      
      //drop events with no active features
      if (indexedContext.size() > 0) {
        int[] cons = new int[indexedContext.size()];
        for (int ci=0;ci<cons.length;ci++) {
          cons[ci] = indexedContext.get(ci);
        }
        ce = new ComparableEvent(ocID, cons, ev.getValues());
        eventsToCompare.add(ce);
      }
      else {
        System.err.println("Dropped event "+ev.getOutcome()+":"+Arrays.asList(ev.getContext()));
      }
//    recycle the TIntArrayList
      indexedContext.clear();
    }
    outcomeLabels = toIndexedStringArray(omap);
    predLabels = toIndexedStringArray(predicateIndex);
    return eventsToCompare;
  }

}

--- NEW FILE: ComparablePredicate.java ---
/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreemnets.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0 
 * (the "License"); you may not use this file except in compliance with 
 * the License. You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package opennlp.model;

/**
 * A maxent predicate representation which we can use to sort based on the
 * outcomes. This allows us to make the mapping of features to their parameters
 * much more compact.
 *
 * @author      Jason Baldridge
 * @version $Revision: 1.1 $, $Date: 2009/01/22 23:23:33 $
 */
public class ComparablePredicate implements Comparable {
    public String name;
    public int[] outcomes;
    public double[] params;
    
    public ComparablePredicate(String n, int[] ocs, double[] ps) {
	name = n;
	outcomes = ocs;
	params = ps;
    }

    public int compareTo(Object o) {
	ComparablePredicate cp = (ComparablePredicate)o;

	int smallerLength = (outcomes.length > cp.outcomes.length?
			     cp.outcomes.length : outcomes.length);

	for (int i=0; i<smallerLength; i++) {
	    if (outcomes[i] < cp.outcomes[i]) return -1;
	    else if (outcomes[i] > cp.outcomes[i]) return 1;
	}

	if (outcomes.length < cp.outcomes.length) return -1;
	else if (outcomes.length > cp.outcomes.length) return 1;

	return 0;
    }

    public String toString() {
	String s = "";
	for (int i=0; i<outcomes.length; i++) s+= " "+outcomes[i];
	return s;
    }

}
 

--- NEW FILE: TwoPassDataIndexer.java ---
/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreemnets.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0 
 * (the "License"); you may not use this file except in compliance with 
 * the License. You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package opennlp.model;

import java.io.BufferedWriter;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.io.Writer;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;


/**
 * Collecting event and context counts by making two passes over the events.  The
 * first pass determines which contexts will be used by the model, and the
 * second pass creates the events in memory containing only the contexts which 
 * will be used.  This greatly reduces the amount of memory required for storing
 * the events.  During the first pass a temporary event file is created which
 * is read during the second pass.
 */
public class TwoPassDataIndexer extends AbstractDataIndexer{

  /**
   * One argument constructor for DataIndexer which calls the two argument
   * constructor assuming no cutoff.
   *
   * @param eventStream An Event[] which contains the a list of all the Events
   *               seen in the training data.
   */
  public TwoPassDataIndexer(EventStream eventStream) throws IOException {
    this(eventStream, 0);
  }

  public TwoPassDataIndexer(EventStream eventStream, int cutoff) throws IOException {
    this(eventStream,cutoff,true);
  }
  /**
   * Two argument constructor for DataIndexer.
   *
   * @param eventStream An Event[] which contains the a list of all the Events
   *               seen in the training data.
   * @param cutoff The minimum number of times a predicate must have been
   *               observed in order to be included in the model.
   */
  public TwoPassDataIndexer(EventStream eventStream, int cutoff, boolean sort) throws IOException {
    Map<String,Integer> predicateIndex = new HashMap<String,Integer>();
    List eventsToCompare;

    System.out.println("Indexing events using cutoff of " + cutoff + "\n");

    System.out.print("\tComputing event counts...  ");
    try {
      File tmp = File.createTempFile("events", null);
      tmp.deleteOnExit();
      Writer osw = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(tmp),"UTF8"));
      int numEvents = computeEventCounts(eventStream, osw, predicateIndex, cutoff);
      System.out.println("done. " + numEvents + " events");

      System.out.print("\tIndexing...  ");

      eventsToCompare = index(numEvents, new FileEventStream(tmp), predicateIndex);
      // done with predicates
      predicateIndex = null;
      tmp.delete();
      System.out.println("done.");

      System.out.print("Sorting and merging events... ");
      sortAndMerge(eventsToCompare,sort);
      System.out.println("Done indexing.");
    }
    catch(IOException e) {
      System.err.println(e);
    }
  }

  /**
      * Reads events from <tt>eventStream</tt> into a linked list.  The
      * predicates associated with each event are counted and any which
      * occur at least <tt>cutoff</tt> times are added to the
      * <tt>predicatesInOut</tt> map along with a unique integer index.
      *
      * @param eventStream an <code>EventStream</code> value
      * @param eventStore a writer to which the events are written to for later processing.
      * @param predicatesInOut a <code>TObjectIntHashMap</code> value
      * @param cutoff an <code>int</code> value
      */
  private int computeEventCounts(EventStream eventStream, Writer eventStore, Map<String,Integer> predicatesInOut, int cutoff) throws IOException {
    Map<String,Integer> counter = new HashMap<String,Integer>();
    int eventCount = 0;
    Set predicateSet = new HashSet();
    while (eventStream.hasNext()) {
      Event ev = eventStream.next();
      eventCount++;
      eventStore.write(FileEventStream.toLine(ev));
      String[] ec = ev.getContext();
      update(ec,predicateSet,counter,cutoff);
    }
    predCounts = new int[predicateSet.size()];
    int index = 0;
    for (Iterator pi=predicateSet.iterator();pi.hasNext();index++) {
      String predicate = (String) pi.next();
      predCounts[index] = counter.get(predicate);
      predicatesInOut.put(predicate,index);
    }
    eventStore.close();
    return eventCount;
  }

  private List index(int numEvents, EventStream es, Map<String,Integer> predicateIndex) {
    Map<String,Integer> omap = new HashMap<String,Integer>();
    int outcomeCount = 0;
    List eventsToCompare = new ArrayList(numEvents);
    List<Integer> indexedContext = new ArrayList<Integer>();
    while (es.hasNext()) {
      Event ev = es.next();
      String[] econtext = ev.getContext();
      ComparableEvent ce;

      int ocID;
      String oc = ev.getOutcome();

      if (omap.containsKey(oc)) {
        ocID = omap.get(oc);
      }
      else {
        ocID = outcomeCount++;
        omap.put(oc, ocID);
      }

      for (int i = 0; i < econtext.length; i++) {
        String pred = econtext[i];
        if (predicateIndex.containsKey(pred)) {
          indexedContext.add(predicateIndex.get(pred));
        }
      }

      // drop events with no active features
      if (indexedContext.size() > 0) {
        int[] cons = new int[indexedContext.size()];
        for (int ci=0;ci<cons.length;ci++) {
          cons[ci] = indexedContext.get(ci);
        }
        ce = new ComparableEvent(ocID, cons);
        eventsToCompare.add(ce);
      }
      else {
        System.err.println("Dropped event " + ev.getOutcome() + ":" + Arrays.asList(ev.getContext()));
      }
      // recycle the TIntArrayList
      indexedContext.clear();
    }
    outcomeLabels = toIndexedStringArray(omap);
    predLabels = toIndexedStringArray(predicateIndex);
    return eventsToCompare;
  }

}


--- NEW FILE: OnePassDataIndexer.java ---
/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreemnets.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0 
 * (the "License"); you may not use this file except in compliance with 
 * the License. You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package opennlp.model;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;


/**
 * An indexer for maxent model data which handles cutoffs for uncommon
 * contextual predicates and provides a unique integer index for each of the
 * predicates. 
 *
 * @author      Jason Baldridge
 * @version $Revision: 1.1 $, $Date: 2009/01/22 23:23:33 $
 */
public class OnePassDataIndexer extends AbstractDataIndexer  {

    /**
     * One argument constructor for DataIndexer which calls the two argument
     * constructor assuming no cutoff.
     *
     * @param eventStream An Event[] which contains the a list of all the Events
     *               seen in the training data.
     */     
    public OnePassDataIndexer(EventStream eventStream) {
        this(eventStream, 0);
    }

    public OnePassDataIndexer(EventStream eventStream, int cutoff) {
      this(eventStream,cutoff,true);
    }
    /**
     * Two argument constructor for DataIndexer.
     *
     * @param eventStream An Event[] which contains the a list of all the Events
     *               seen in the training data.
     * @param cutoff The minimum number of times a predicate must have been
     *               observed in order to be included in the model.
     */
    public OnePassDataIndexer(EventStream eventStream, int cutoff, boolean sort) {
        Map<String,Integer> predicateIndex = new HashMap<String,Integer>();
        LinkedList<Event> events;
        List eventsToCompare;

        System.out.println("Indexing events using cutoff of " + cutoff + "\n");

        System.out.print("\tComputing event counts...  ");
        events = computeEventCounts(eventStream,predicateIndex,cutoff);
        System.out.println("done. "+events.size()+" events");

        System.out.print("\tIndexing...  ");
        eventsToCompare = index(events,predicateIndex);
        // done with event list
        events = null;
        // done with predicates
        predicateIndex = null;

        System.out.println("done.");

        System.out.print("Sorting and merging events... ");
        sortAndMerge(eventsToCompare,sort);
        System.out.println("Done indexing.");
    }


    
    /**
     * Reads events from <tt>eventStream</tt> into a linked list.  The
     * predicates associated with each event are counted and any which
     * occur at least <tt>cutoff</tt> times are added to the
     * <tt>predicatesInOut</tt> map along with a unique integer index.
     *
     * @param eventStream an <code>EventStream</code> value
     * @param predicatesInOut a <code>TObjectIntHashMap</code> value
     * @param cutoff an <code>int</code> value
     * @return a <code>TLinkedList</code> value
     */
    private LinkedList<Event> computeEventCounts(EventStream eventStream,Map<String,Integer> predicatesInOut,
        int cutoff) {
      Set predicateSet = new HashSet();
      Map<String,Integer> counter = new HashMap<String,Integer>();
      LinkedList<Event> events = new Lin...
 
[truncated message content]

[Maxent-commit] CVS: maxent/src/main/java/opennlp/perceptron BinaryPerceptronModelReader.java, NONE, 1.1 SuffixSensitivePerceptronModelWriter.java, NONE, 1.1 PlainTextPerceptronModelWriter.java, NONE, 1.1 BinaryPerceptronModelWriter.java, NONE, 1.1 PerceptronModel.java, NONE, 1.1 PerceptronModelWriter.java, NONE, 1.1 PerceptronModelReader.java, NONE, 1.1 PlainTextPerceptronModelReader.java, NONE, 1.1 PerceptronTrainer.java, NONE, 1.1

[Joern K. <joe...@us...>]

Update of /cvsroot/maxent/maxent/src/main/java/opennlp/perceptron
In directory 23jxhf1.ch3.sourceforge.com:/tmp/cvs-serv21001/src/main/java/opennlp/perceptron

Added Files:
	BinaryPerceptronModelReader.java 
	SuffixSensitivePerceptronModelWriter.java 
	PlainTextPerceptronModelWriter.java 
	BinaryPerceptronModelWriter.java PerceptronModel.java 
	PerceptronModelWriter.java PerceptronModelReader.java 
	PlainTextPerceptronModelReader.java PerceptronTrainer.java 
Log Message:
Introduced maven build.

--- NEW FILE: BinaryPerceptronModelReader.java ---
package opennlp.perceptron;

import java.io.DataInputStream;
import java.io.File;
import java.io.IOException;

import opennlp.model.BinaryFileDataReader;

public class BinaryPerceptronModelReader extends PerceptronModelReader {
  

  /**
   * Constructor which directly instantiates the DataInputStream containing
   * the model contents.
   *
   * @param dis The DataInputStream containing the model information.
   */
  public BinaryPerceptronModelReader(DataInputStream dis) {
    super(new BinaryFileDataReader(dis));
  }

  /**
   * Constructor which takes a File and creates a reader for it. Detects
   * whether the file is gzipped or not based on whether the suffix contains
   * ".gz" 
   *
   * @param f The File in which the model is stored.
   */
  public BinaryPerceptronModelReader (File f) throws IOException {
    super(f);
  }  
}

--- NEW FILE: SuffixSensitivePerceptronModelWriter.java ---
///////////////////////////////////////////////////////////////////////////////
// Copyright (C) 2001 Jason Baldridge and Gann Bierner
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; either
// version 2.1 of the License, or (at your option) any later version.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
//////////////////////////////////////////////////////////////////////////////   
package opennlp.perceptron;

import java.io.BufferedWriter;
import java.io.DataOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.util.zip.GZIPOutputStream;

import opennlp.model.AbstractModel;
import opennlp.model.AbstractModelWriter;

/**
 * A writer for GIS models which inspects the filename and invokes the
 * appropriate GISModelWriter depending on the filename's suffixes.
 *
 * <p>The following assumption are made about suffixes:
 *    <li>.gz  --> the file is gzipped (must be the last suffix)
 *    <li>.txt --> the file is plain text
 *    <li>.bin --> the file is binary
 *
 * @author      Jason Baldridge
 * @version     $Revision: 1.1 $, $Date: 2009/01/22 23:23:34 $
 */
public class SuffixSensitivePerceptronModelWriter extends PerceptronModelWriter {
    private final AbstractModelWriter suffixAppropriateWriter;

    /**
     * Constructor which takes a GISModel and a File and invokes the
     * GISModelWriter appropriate for the suffix.
     *
     * @param model The GISModel which is to be persisted.
     * @param f The File in which the model is to be stored.
     */
    public SuffixSensitivePerceptronModelWriter (AbstractModel model, File f)
	throws IOException {

	super (model);
	
	OutputStream output;
	String filename = f.getName();

	// handle the zipped/not zipped distinction
	if (filename.endsWith(".gz")) {
	    output = new GZIPOutputStream(new FileOutputStream(f));
	    filename = filename.substring(0,filename.length()-3);
	}
	else {
	    output = new DataOutputStream(new FileOutputStream(f));
	}

	// handle the different formats
	if (filename.endsWith(".bin")) {
	    suffixAppropriateWriter =
		new BinaryPerceptronModelWriter(model,
					 new DataOutputStream(output));
	}
	else { // default is ".txt"
	    suffixAppropriateWriter =
		new PlainTextPerceptronModelWriter(model,
		    new BufferedWriter(new OutputStreamWriter(output)));
	}    
    }

    public void writeUTF (String s) throws java.io.IOException {
      suffixAppropriateWriter.writeUTF(s);
    }

    public void writeInt (int i) throws java.io.IOException {
      suffixAppropriateWriter.writeInt(i);
    }
    
    public void writeDouble (double d) throws java.io.IOException {
      suffixAppropriateWriter.writeDouble(d);
    }

    public void close () throws java.io.IOException {
      suffixAppropriateWriter.close();
    }

}

--- NEW FILE: PlainTextPerceptronModelWriter.java ---
///////////////////////////////////////////////////////////////////////////////
// Copyright (C) 2001 Jason Baldridge and Gann Bierner
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; either
// version 2.1 of the License, or (at your option) any later version.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
//////////////////////////////////////////////////////////////////////////////   
package opennlp.perceptron;

import java.io.BufferedWriter;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.FileWriter;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.util.zip.GZIPOutputStream;

import opennlp.model.AbstractModel;

/**
 * Model writer that saves models in plain text format.
 *
 * @author      Jason Baldridge
 * @version     $Revision: 1.1 $, $Date: 2009/01/22 23:23:34 $
 */
public class PlainTextPerceptronModelWriter extends PerceptronModelWriter {
    BufferedWriter output;
    
   /**
     * Constructor which takes a PerceptronModel and a File and prepares itself to
     * write the model to that file. Detects whether the file is gzipped or not
     * based on whether the suffix contains ".gz".
     *
     * @param model The PerceptronModel which is to be persisted.
     * @param f The File in which the model is to be persisted.
     */
     public PlainTextPerceptronModelWriter (AbstractModel model, File f)
	throws IOException, FileNotFoundException {

	super(model);
	if (f.getName().endsWith(".gz")) {
	    output = new BufferedWriter(new OutputStreamWriter(
  	                 new GZIPOutputStream(new FileOutputStream(f))));
	}
	else {
	    output = new BufferedWriter(new FileWriter(f));
	}
    }

   /**
     * Constructor which takes a PerceptronModel and a BufferedWriter and prepares
     * itself to write the model to that writer.
     *
     * @param model The PerceptronModel which is to be persisted.
     * @param bw The BufferedWriter which will be used to persist the model.
     */
    public PlainTextPerceptronModelWriter (AbstractModel model, BufferedWriter bw) {
	super(model);
	output = bw;
    }

    public void writeUTF (String s) throws java.io.IOException {
	output.write(s);
	output.newLine();
    }

    public void writeInt (int i) throws java.io.IOException {
	output.write(Integer.toString(i));
	output.newLine();
    }
    
    public void writeDouble (double d) throws java.io.IOException {
	output.write(Double.toString(d));
	output.newLine();
    }

    public void close () throws java.io.IOException {
	output.flush();
	output.close();
    }
    
}

--- NEW FILE: BinaryPerceptronModelWriter.java ---
///////////////////////////////////////////////////////////////////////////////
//Copyright (C) 2001 Jason Baldridge and Gann Bierner

//This library is free software; you can redistribute it and/or
//modify it under the terms of the GNU Lesser General Public
//License as published by the Free Software Foundation; either
//version 2.1 of the License, or (at your option) any later version.

//This library is distributed in the hope that it will be useful,
//but WITHOUT ANY WARRANTY; without even the implied warranty of
//MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
//GNU General Public License for more details.

//You should have received a copy of the GNU Lesser General Public
//License along with this program; if not, write to the Free Software
//Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.

package opennlp.perceptron;

import java.io.DataOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.zip.GZIPOutputStream;

import opennlp.model.AbstractModel;

/**
 * Model writer that saves models in binary format.
 *
 * @author      Jason Baldridge
 * @version     $Revision: 1.1 $, $Date: 2009/01/22 23:23:34 $
 */
public class BinaryPerceptronModelWriter extends PerceptronModelWriter {
  DataOutputStream output;

  /**
   * Constructor which takes a GISModel and a File and prepares itself to
   * write the model to that file. Detects whether the file is gzipped or not
   * based on whether the suffix contains ".gz".
   *
   * @param model The GISModel which is to be persisted.
   * @param f The File in which the model is to be persisted.
   */
  public BinaryPerceptronModelWriter (AbstractModel model, File f) throws IOException {

    super(model);

    if (f.getName().endsWith(".gz")) {
      output = new DataOutputStream(
          new GZIPOutputStream(new FileOutputStream(f)));
    }
    else {
      output = new DataOutputStream(new FileOutputStream(f));
    }
  }

  /**
   * Constructor which takes a GISModel and a DataOutputStream and prepares
   * itself to write the model to that stream.
   *
   * @param model The GISModel which is to be persisted.
   * @param dos The stream which will be used to persist the model.
   */
  public BinaryPerceptronModelWriter (AbstractModel model, DataOutputStream dos) {
    super(model);
    output = dos;
  }

  public void writeUTF (String s) throws java.io.IOException {
    output.writeUTF(s);
  }

  public void writeInt (int i) throws java.io.IOException {
    output.writeInt(i);
  }

  public void writeDouble (double d) throws java.io.IOException {
    output.writeDouble(d);
  }

  public void close () throws java.io.IOException {
    output.flush();
    output.close();
  }

}

--- NEW FILE: PerceptronModel.java ---
package opennlp.perceptron;

import java.io.BufferedReader;
import java.io.File;
import java.io.InputStreamReader;
import java.text.DecimalFormat;

import opennlp.model.AbstractModel;
import opennlp.model.Context;
import opennlp.model.EvalParameters;

public class PerceptronModel extends AbstractModel {

  
  public PerceptronModel(Context[] params, String[] predLabels, String[] outcomeNames) {
    super(params,predLabels,outcomeNames);
    modelType = ModelType.Perceptron;
  }
  
  public double[] eval(String[] context) {
    return eval(context,new double[evalParams.getNumOutcomes()]);
  }
  
  public double[] eval(String[] context, float[] values) {
    return eval(context,values,new double[evalParams.getNumOutcomes()]);
  }

  public double[] eval(String[] context, double[] probs) {
    return eval(context,null,probs);
  }

  public double[] eval(String[] context, float[] values,double[] outsums) {
    int[] scontexts = new int[context.length];
    java.util.Arrays.fill(outsums, 0);
    for (int i=0; i<context.length; i++) {
      Integer ci = pmap.get(context[i]);
      scontexts[i] = ci == null ? -1 : ci;
    }
    return eval(scontexts,values,outsums,evalParams,true);
  }
  
  public static double[] eval(int[] context, double[] prior, EvalParameters model) {
    return eval(context,null,prior,model,true);
  }
  
  public static double[] eval(int[] context, float[] values, double[] prior, EvalParameters model, boolean normalize) {
    Context[] params = model.getParams();
    double[] activeParameters;
    int[] activeOutcomes;
    double value = 1;
    for (int ci = 0; ci < context.length; ci++) {
      if (context[ci] >= 0) {
        Context predParams = params[context[ci]];
        activeOutcomes = predParams.getOutcomes();
        activeParameters = predParams.getParameters();
        if (values != null) {
          value = values[ci];
        }
        for (int ai = 0; ai < activeOutcomes.length; ai++) {
          int oid = activeOutcomes[ai];
          prior[oid] += activeParameters[ai] * value;
        }
      }
    }    
    if (normalize) {
      double normal = 0.0;
      double min = prior[0];
      for (int oid = 0; oid < model.getNumOutcomes(); oid++) {
        if (prior[oid] < min) {
          min = prior[oid];
        }
      }
      for (int oid = 0; oid < model.getNumOutcomes(); oid++) {
        if (min < 0) {
          prior[oid]+=(-1*min);
        }
        normal += prior[oid];
      }
      if (normal == 0.0) {
        for (int oid = 0; oid < model.getNumOutcomes(); oid++) {
          prior[oid] = (double) 1/model.getNumOutcomes();
        }
      }
      else {
        for (int oid = 0; oid < model.getNumOutcomes(); oid++) {
          prior[oid] /= normal;
        }
      }
    }
    //System.err.println("Perceptron Model: "+java.util.Arrays.asList(prior));
    return prior;
  }
  
  public static void main(String[] args) throws java.io.IOException {
    if (args.length == 0) {
      System.err.println("Usage: PerceptronModel modelname < contexts");
      System.exit(1);
    }
    AbstractModel m = new PerceptronModelReader(new File(args[0])).getModel();
    BufferedReader in = new BufferedReader(new InputStreamReader(System.in));
    DecimalFormat df = new java.text.DecimalFormat(".###");
    for (String line = in.readLine(); line != null; line = in.readLine()) {
      String[] context = line.split(" ");
      double[] dist = m.eval(context);
      for (int oi=0;oi<dist.length;oi++) {
        System.out.print("["+m.getOutcome(oi)+" "+df.format(dist[oi])+"] ");
      }
      System.out.println();
    }
  }
}

--- NEW FILE: PerceptronModelWriter.java ---
package opennlp.perceptron;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Map;

import opennlp.model.AbstractModel;
import opennlp.model.AbstractModelWriter;
import opennlp.model.ComparablePredicate;
import opennlp.model.Context;

/**
 * Abstract parent class for Perceptron writers.  It provides the persist method
 * which takes care of the structure of a stored document, and requires an
 * extending class to define precisely how the data should be stored.
 *
 * @author      Jason Baldridge
 * @version     $Revision: 1.1 $, $Date: 2009/01/22 23:23:34 $
 */
public abstract class PerceptronModelWriter extends AbstractModelWriter {
    protected Context[] PARAMS;
    protected String[] OUTCOME_LABELS;
    protected String[] PRED_LABELS;
    int numOutcomes;

    public PerceptronModelWriter (AbstractModel model) {
      
      Object[] data = model.getDataStructures();
      this.numOutcomes = model.getNumOutcomes();
      PARAMS = (Context[]) data[0];
      Map<String,Integer> pmap = (Map<String,Integer>)data[1];
      OUTCOME_LABELS = (String[])data[2];
      
      PRED_LABELS = new String[pmap.size()];
      for (String pred : pmap.keySet()) {
        PRED_LABELS[pmap.get(pred)] = pred;
      }
    }

    protected ComparablePredicate[] sortValues () {
      ComparablePredicate[] sortPreds;
      ComparablePredicate[] tmpPreds = new ComparablePredicate[PARAMS.length];
      int[] tmpOutcomes = new int[numOutcomes];
      double[] tmpParams = new double[numOutcomes];
      int numPreds = 0;
      //remove parameters with 0 weight and predicates with no parameters 
      for (int pid=0; pid<PARAMS.length; pid++) {
        int numParams = 0;    
        double[] predParams = PARAMS[pid].getParameters();
        for (int pi=0;pi<predParams.length;pi++) {
          if (predParams[pi] != 0d) {
            tmpOutcomes[numParams]=pi;
            tmpParams[numParams]=predParams[pi];
            numParams++;
          }
        }

        int[] activeOutcomes = new int[numParams];
        double[] activeParams = new double[numParams];

        for (int pi=0;pi<numParams;pi++) {
          activeOutcomes[pi] = tmpOutcomes[pi];
          activeParams[pi] = tmpParams[pi];
        }
        if (numParams != 0) {
          tmpPreds[numPreds] = new ComparablePredicate(PRED_LABELS[pid],activeOutcomes,activeParams);
          numPreds++;
        }
      }
      sortPreds = new ComparablePredicate[numPreds];
      for (int pid=0;pid<numPreds;pid++) {
        sortPreds[pid] = tmpPreds[pid];
      }
      Arrays.sort(sortPreds);
      return sortPreds;
    }
    
        
    protected List compressOutcomes (ComparablePredicate[] sorted) {
      ComparablePredicate cp = sorted[0];
      List outcomePatterns = new ArrayList();
      List newGroup = new ArrayList();
      for (int i=0; i<sorted.length; i++) {
        if (cp.compareTo(sorted[i]) == 0) {
          newGroup.add(sorted[i]);
        } else {	    
          cp = sorted[i];
          outcomePatterns.add(newGroup);
          newGroup = new ArrayList();
          newGroup.add(sorted[i]);
        }	    
      }
      outcomePatterns.add(newGroup);
      return outcomePatterns;
    }

    /**
     * Writes the model to disk, using the <code>writeX()</code> methods
     * provided by extending classes.
     *
     * <p>If you wish to create a PerceptronModelWriter which uses a different
     * structure, it will be necessary to override the persist method in
     * addition to implementing the <code>writeX()</code> methods.
     */
    public void persist() throws IOException {
      
      // the type of model (Perceptron)
      writeUTF("Perceptron");
      
      // the mapping from outcomes to their integer indexes
      writeInt(OUTCOME_LABELS.length);
      
      for (int i=0; i<OUTCOME_LABELS.length; i++)
        writeUTF(OUTCOME_LABELS[i]); 
      
      // the mapping from predicates to the outcomes they contributed to.
      // The sorting is done so that we actually can write this out more
      // compactly than as the entire list.
      ComparablePredicate[] sorted = sortValues();
      List compressed = compressOutcomes(sorted);
      
      writeInt(compressed.size());
      
      for (int i=0; i<compressed.size(); i++) {
        List a = (List)compressed.get(i);
        writeUTF(a.size()
            + ((ComparablePredicate)a.get(0)).toString());
      } 
      
      // the mapping from predicate names to their integer indexes
      writeInt(sorted.length);
      
      for (int i=0; i<sorted.length; i++)
        writeUTF(sorted[i].name); 
      
      // write out the parameters
      for (int i=0; i<sorted.length; i++)
        for (int j=0; j<sorted[i].params.length; j++)
          writeDouble(sorted[i].params[j]);
      
      close();
    }
}

--- NEW FILE: PerceptronModelReader.java ---
package opennlp.perceptron;

import java.io.File;
import java.io.IOException;

import opennlp.model.AbstractModel;
import opennlp.model.AbstractModelReader;
import opennlp.model.Context;
import opennlp.model.DataReader;

/**
 * Abstract parent class for readers of GISModels.
 *
 * @author      Jason Baldridge
 * @version     $Revision: 1.1 $, $Date: 2009/01/22 23:23:34 $
 */
public class PerceptronModelReader extends AbstractModelReader {
  
    public PerceptronModelReader(File file) throws IOException {
      super(file);
    }
  
    public PerceptronModelReader(DataReader dataReader) {
      super(dataReader);
    }
    
    /**
     * Retrieve a model from disk. It assumes that models are saved in the
     * following sequence:
     * 
     * <br>Perceptron (model type identifier)
     * <br>1. # of parameters (int)
     * <br>2. # of outcomes (int)
     * <br>  * list of outcome names (String)
     * <br>3. # of different types of outcome patterns (int)
     * <br>   * list of (int int[])
     * <br>   [# of predicates for which outcome pattern is true] [outcome pattern]
     * <br>4. # of predicates (int)
     * <br>   * list of predicate names (String)
     *
     * <p>If you are creating a reader for a format which won't work with this
     * (perhaps a database or xml file), override this method and ignore the
     * other methods provided in this abstract class.
     *
     * @return The PerceptronModel stored in the format and location specified to
     *         this PerceptronModelReader (usually via its the constructor).
     */
    public AbstractModel constructModel() throws IOException {
      String[] outcomeLabels = getOutcomes();
      int[][] outcomePatterns = getOutcomePatterns();
      String[] predLabels = getPredicates();
      Context[] params = getParameters(outcomePatterns);
    
      return new PerceptronModel(params,
                          predLabels,
                          outcomeLabels);
    }

    public void checkModelType() throws java.io.IOException {
      String modelType = readUTF();
      if (!modelType.equals("Perceptron"))
          System.out.println("Error: attempting to load a "+modelType+
                             " model as a Perceptron model."+
                             " You should expect problems.");
    }
}

--- NEW FILE: PlainTextPerceptronModelReader.java ---
package opennlp.perceptron;

import java.io.BufferedReader;
import java.io.File;
import java.io.IOException;

import opennlp.model.PlainTextFileDataReader;

public class PlainTextPerceptronModelReader extends PerceptronModelReader {
  
  /**
   * Constructor which directly instantiates the BufferedReader containing
   * the model contents.
   *
   * @param br The BufferedReader containing the model information.
   */
  public PlainTextPerceptronModelReader(BufferedReader br) {
    super(new PlainTextFileDataReader(br));
  }

  /**
   * Constructor which takes a File and creates a reader for it. Detects
   * whether the file is gzipped or not based on whether the suffix contains
   * ".gz".
   *
   * @param f The File in which the model is stored.
   */
  public PlainTextPerceptronModelReader (File f) throws IOException {
    super(f);
  }
}

--- NEW FILE: PerceptronTrainer.java ---
package opennlp.perceptron;

import opennlp.model.AbstractModel;
import opennlp.model.DataIndexer;
import opennlp.model.EvalParameters;
import opennlp.model.EvalParameters;
import opennlp.model.MutableContext;

public class PerceptronTrainer {

  /** Number of unique events which occured in the event set. */
  private int numUniqueEvents;
  /** Number of events in the event set. */
  private int numEvents;
  
  /** Number of predicates. */
  private int numPreds; 
  /** Number of outcomes. */
  private int numOutcomes; 
  /** Records the array of predicates seen in each event. */
  private int[][] contexts;

  /** The value associates with each context. If null then context values are assumes to be 1. */
  private float[][] values;

  /** List of outcomes for each event i, in context[i]. */
  private int[] outcomeList;

  /** Records the num of times an event has been seen for each event i, in context[i]. */
  private int[] numTimesEventsSeen;
  
  /** Stores the String names of the outcomes.  The GIS only tracks outcomes
  as ints, and so this array is needed to save the model to disk and
  thereby allow users to know what the outcome was in human
  understandable terms. */
  private String[] outcomeLabels;

  /** Stores the String names of the predicates. The GIS only tracks
  predicates as ints, and so this array is needed to save the model to
  disk and thereby allow users to know what the outcome was in human
  understandable terms. */
  private String[] predLabels;

  /** Stores the estimated parameter value of each predicate during iteration. */
  private MutableContext[] params; 

  private int[][][] updates;
  private int VALUE = 0;
  private int ITER = 1;
  private int EVENT = 2;
  
  /** Stores the average parameter values of each predicate during iteration. */
  private MutableContext[] averageParams;

  private EvalParameters evalParams;

  private boolean printMessages = true;
    
  double[] modelDistribution;
  
  private int iterations;
  private boolean useAverage;
  
  public AbstractModel trainModel(int iterations, DataIndexer di, int cutoff) {
    this.iterations = iterations;
    return trainModel(iterations,di,cutoff,true);
  }
  
  public AbstractModel trainModel(int iterations, DataIndexer di, int cutoff, boolean useAverage) {
    display("Incorporating indexed data for training...  \n");
    this.useAverage = useAverage;
    contexts = di.getContexts();
    values = di.getValues();
    numTimesEventsSeen = di.getNumTimesEventsSeen();
    numEvents = di.getNumEvents();
    numUniqueEvents = contexts.length;

    this.iterations = iterations;
    outcomeLabels = di.getOutcomeLabels();
    outcomeList = di.getOutcomeList();

    predLabels = di.getPredLabels();
    numPreds = predLabels.length;
    numOutcomes = outcomeLabels.length;
    if (useAverage) updates = new int[numPreds][numOutcomes][3];
    
    display("done.\n");
    
    display("\tNumber of Event Tokens: " + numUniqueEvents + "\n");
    display("\t    Number of Outcomes: " + numOutcomes + "\n");
    display("\t  Number of Predicates: " + numPreds + "\n");
    

    params = new MutableContext[numPreds];
    if (useAverage) averageParams = new MutableContext[numPreds];
    evalParams = new EvalParameters(params,numOutcomes);
    
    int[] allOutcomesPattern= new int[numOutcomes];
    for (int oi = 0; oi < numOutcomes; oi++) {
      allOutcomesPattern[oi] = oi;
    }
    
    int numActiveOutcomes = numOutcomes;
    for (int pi = 0; pi < numPreds; pi++) {
      params[pi] = new MutableContext(allOutcomesPattern,new double[numActiveOutcomes]);
      if (useAverage) averageParams[pi] = new MutableContext(allOutcomesPattern,new double[numActiveOutcomes]);
      for (int aoi=0;aoi<numActiveOutcomes;aoi++) {
        params[pi].setParameter(aoi, 0.0);
        if (useAverage) averageParams[pi].setParameter(aoi, 0.0);
      }
    }
    modelDistribution = new double[numOutcomes];

    display("Computing model parameters...\n");
    findParameters(iterations);
    display("...done.\n");

    /*************** Create and return the model ******************/
    if (useAverage) {
      return new PerceptronModel(averageParams, predLabels, outcomeLabels);
    }
    else {
      return new PerceptronModel(params, predLabels, outcomeLabels);
    }
  }

  private void display(String s) {
    if (printMessages)
      System.out.print(s);
  }
  
  private void findParameters(int iterations) {
    display("Performing " + iterations + " iterations.\n");
    for (int i = 1; i <= iterations; i++) {
      if (i < 10)
        display("  " + i + ":  ");
      else if (i < 100)
        display(" " + i + ":  ");
      else
        display(i + ":  ");
      nextIteration(i);
    }
    // kill a bunch of these big objects now that we don't need them
    numTimesEventsSeen = null;
    contexts = null;
  }
  
  /* Compute one iteration of Perceptron and retutn log-likelihood.*/
  private void nextIteration(int iteration) {
    iteration--; //move to 0-based index
    int numCorrect = 0;
    for (int ei = 0; ei < numUniqueEvents; ei++) {
      for (int ni=0;ni<this.numTimesEventsSeen[ei];ni++) {
        for (int oi = 0; oi < numOutcomes; oi++) {
          modelDistribution[oi] = 0;
        }
        if (values != null) {
          PerceptronModel.eval(contexts[ei], values[ei], modelDistribution, evalParams,false);
        }
        else {
          PerceptronModel.eval(contexts[ei], null, modelDistribution, evalParams, false);
        }
        int max = 0;
        for (int oi = 1; oi < numOutcomes; oi++) {
          if (modelDistribution[oi] > modelDistribution[max]) {
            max = oi;
          }
        }
        if (max == outcomeList[ei]) {
          numCorrect += numTimesEventsSeen[ei];
        }
        for (int oi = 0;oi<numOutcomes;oi++) {
          if (oi == outcomeList[ei]) {
            if (modelDistribution[oi] <= 0) {
              for (int ci = 0; ci < contexts[ei].length; ci++) {
                int pi = contexts[ei][ci];
                if (values == null) {
                  params[pi].updateParameter(oi, 1);
                }
                else {
                  params[pi].updateParameter(oi, values[ei][ci]);
                }
                if (useAverage) {
                  if (updates[pi][oi][VALUE] != 0) {
                    averageParams[pi].updateParameter(oi,updates[pi][oi][VALUE]*(numEvents*(iteration-updates[pi][oi][ITER])+(ei-updates[pi][oi][EVENT])));
                  }
                  //System.err.println("updates["+pi+"]["+oi+"]=("+updates[pi][oi][ITER]+","+updates[pi][oi][EVENT]+","+updates[pi][oi][VALUE]+") + ("+iteration+","+ei+","+params[pi].getParameters()[oi]+") -> "+averageParams[pi].getParameters()[oi]);
                  updates[pi][oi][VALUE] = (int) params[pi].getParameters()[oi];
                  updates[pi][oi][ITER] = iteration;
                  updates[pi][oi][EVENT] = ei;
                }
              }
            }
          }
          else {
            if (modelDistribution[oi] > 0) {
              for (int ci = 0; ci < contexts[ei].length; ci++) {
                int pi = contexts[ei][ci];
                if (values == null) {
                  params[pi].updateParameter(oi,-1);
                }
                else {
                  params[pi].updateParameter(oi, values[ei][ci]*-1);
                }
                if (useAverage) {
                  if (updates[pi][oi][VALUE] != 0) {
                    averageParams[pi].updateParameter(oi,updates[pi][oi][VALUE]*(numEvents*(iteration-updates[pi][oi][ITER])+(ei-updates[pi][oi][EVENT])));
                  }
                  //System.err.println("updates["+pi+"]["+oi+"]=("+updates[pi][oi][ITER]+","+updates[pi][oi][EVENT]+","+updates[pi][oi][VALUE]+") + ("+iteration+","+ei+","+params[pi].getParameters()[oi]+") -> "+averageParams[pi].getParameters()[oi]);
                  updates[pi][oi][VALUE] = (int) params[pi].getParameters()[oi];
                  updates[pi][oi][ITER] = iteration;
                  updates[pi][oi][EVENT] = ei;
                }
              }
            }
          }
        }
      }
    }
    //finish average computation
    double totIterations = (double) iterations*numEvents;
    if (useAverage && iteration == iterations-1) {
      for (int pi = 0; pi < numPreds; pi++) {
        double[] predParams = averageParams[pi].getParameters();
        for (int oi = 0;oi<numOutcomes;oi++) {
          if (updates[pi][oi][VALUE] != 0) {
            predParams[oi] +=  updates[pi][oi][VALUE]*(numEvents*(iterations-updates[pi][oi][ITER])-updates[pi][oi][EVENT]);
          }
          if (predParams[oi] != 0) {
            predParams[oi] /=totIterations;  
            averageParams[pi].setParameter(oi, predParams[oi]);
            //System.err.println("updates["+pi+"]["+oi+"]=("+updates[pi][oi][ITER]+","+updates[pi][oi][EVENT]+","+updates[pi][oi][VALUE]+") + ("+iterations+","+0+","+params[pi].getParameters()[oi]+") -> "+averageParams[pi].getParameters()[oi]);
          }
        }
      }
    }
    display(". "+((double) numCorrect / numEvents) + "\n");
  }  
}