Menu
▾
▴
[DL-Learner SVN] SF.net SVN: dl-learner:[2052] trunk/src/dl-learner/org/dllearner
|
From: <ku...@us...> - 2010-02-17 00:06:26
|
Revision: 2052
http://dl-learner.svn.sourceforge.net/dl-learner/?rev=2052&view=rev
Author: kurzum
Date: 2010-02-17 00:06:10 +0000 (Wed, 17 Feb 2010)
Log Message:
-----------
server commit
Modified Paths:
--------------
trunk/src/dl-learner/org/dllearner/scripts/tiger/TestIterativeLearning.java
trunk/src/dl-learner/org/dllearner/utilities/experiments/IteratedConfig.java
trunk/src/dl-learner/org/dllearner/utilities/experiments/Table.java
trunk/src/dl-learner/org/dllearner/utilities/experiments/TableRowColumn.java
Modified: trunk/src/dl-learner/org/dllearner/scripts/tiger/TestIterativeLearning.java
===================================================================
--- trunk/src/dl-learner/org/dllearner/scripts/tiger/TestIterativeLearning.java 2010-02-16 20:04:11 UTC (rev 2051)
+++ trunk/src/dl-learner/org/dllearner/scripts/tiger/TestIterativeLearning.java 2010-02-17 00:06:10 UTC (rev 2052)
@@ -88,9 +88,9 @@
static MonKeyImp logIterationTime = new MonKeyImp("Iteration Time", Jamon.MS);
static List<MonKeyImp> mks = new ArrayList<MonKeyImp>(Arrays.asList(new MonKeyImp[] { logPrecision,
- logRecall, logFMeasure, logLearningTime, logIterationTime }));
+ logRecall, logFMeasure, logAccuracy, logLearningTime, logIterationTime }));
- static int iterations = 5;
+ static int iterations = 7;
static int folds = 10;
static int printSentences = 3;
@@ -158,14 +158,11 @@
conductExperiment(examples, experimentConfig);
}
Table expTable = new Table();
- expTable.addTableRowColumn(experimentConfig.getTableRows());
- expTable.write(resultFolder, experimentConfig.experimentName);
+ expTable.addTableRowColumns(experimentConfig.getTableRows());
+ expTable.write(resultFolder, "passiveNoZu_"+experimentConfig.experimentName);
masterTable.addTable(expTable);
- masterTable.sortByExperimentName();
- masterTable.write(resultFolder, "passiveNoZu_by_expname");
- masterTable.sortByLabel();
- masterTable.write(resultFolder, "passiveNoZu_by_label");
-
+ masterTable.write(resultFolder, "passiveNoZu_master");
+
JamonMonitorLogger.writeHTMLReport("/tmp/tiger.html");
logger.info(experimentConfig);
@@ -189,19 +186,25 @@
List<Examples> runs = new ArrayList<Examples>();
runs.add(new ExMakerRandomizer(allExamples).split(0.7d));
-// runs.add(new ExMakerRandomizer(allExamples).split(0.7d));
-// runs.add(new ExMakerRandomizer(allExamples).split(0.7d));
-// runs.add(new ExMakerRandomizer(allExamples).split(0.7d));
-// runs.add(new ExMakerRandomizer(allExamples).split(0.7d));
+ runs.add(new ExMakerRandomizer(allExamples).split(0.7d));
+ runs.add(new ExMakerRandomizer(allExamples).split(0.7d));
+ runs.add(new ExMakerRandomizer(allExamples).split(0.7d));
+ runs.add(new ExMakerRandomizer(allExamples).split(0.7d));
+ runs.add(new ExMakerRandomizer(allExamples).split(0.7d));
+ runs.add(new ExMakerRandomizer(allExamples).split(0.7d));
+ runs.add(new ExMakerRandomizer(allExamples).split(0.7d));
+ runs.add(new ExMakerRandomizer(allExamples).split(0.7d));
+ runs.add(new ExMakerRandomizer(allExamples).split(0.7d));
//
/*CLEANUP*/
positives = null;
negatives = null;
allExamples = null;
- List<IteratedConfig> configs = getConfigs();
+ List<IteratedConfig> configs = getConfigsZU();
Table masterTable = new Table();
for (IteratedConfig experimentConfig : configs) {
+ experimentConfig.init(mks);
logger.info("next: passiveWithZu." + experimentConfig.experimentName);
int i = 1;
for (Examples examples : runs) {
@@ -210,13 +213,10 @@
}
Table expTable = new Table();
- expTable.addTableRowColumn(experimentConfig.getTableRows());
- expTable.write(resultFolder, "passiveWithZu"+experimentConfig.experimentName);
+ expTable.addTableRowColumns(experimentConfig.getTableRows());
+ expTable.write(resultFolder, "passiveWithZu_"+experimentConfig.experimentName);
masterTable.addTable(expTable);
- masterTable.sortByExperimentName();
- masterTable.write(resultFolder, "passiveWithZu_by_expname");
- masterTable.sortByLabel();
- masterTable.write(resultFolder, "passiveWithZu_by_label");
+ masterTable.write(resultFolder, "passiveWithZu_master");
JamonMonitorLogger.writeHTMLReport("/tmp/tiger.html");
logger.info(experimentConfig);
@@ -229,13 +229,12 @@
List<IteratedConfig> l = new ArrayList<IteratedConfig>();
IteratedConfig baseline = new IteratedConfig("baseline_5_5", iterations);
-
+
IteratedConfig reducedExamples = new IteratedConfig("reducedExamples_2_2", iterations);
reducedExamples.initialsplits = 2;
reducedExamples.splits = 2;
- // reducedExamples.adaptMaxRuntime=false;
- // reducedExamples.maxExecutionTime = 20;
reducedExamples.factor = 6.0d;
+
IteratedConfig fixRuntime = new IteratedConfig("fixRuntime_20s", iterations);
fixRuntime.adaptMaxRuntime = false;
@@ -245,24 +244,56 @@
useLemma.useDataHasValue = false;
l.add(baseline);
-// l.add(reducedExamples);
-// l.add(fixRuntime);
-// l.add(useLemma);
+ l.add(reducedExamples);
+ l.add(fixRuntime);
+ l.add(useLemma);
return l;
}
+ public static List<IteratedConfig> getConfigsZU() {
+
+ List<IteratedConfig> l = new ArrayList<IteratedConfig>();
+ IteratedConfig baseline = new IteratedConfig("baseline", iterations);
+
+
+ IteratedConfig increasedNegativeExamples = new IteratedConfig("increasedNegativeExamples", iterations);
+ increasedNegativeExamples.negativeSplitAdd = 10;
+
+
+ IteratedConfig noNoise = new IteratedConfig("noNoise", iterations);
+ noNoise.factor = 4.0d;
+ noNoise.noise = 0;
+ noNoise.noiseIterationFactor = 0;
+
+
+
+ IteratedConfig useLemma = new IteratedConfig("useLemma_false", iterations);
+ useLemma.useDataHasValue = false;
+
+ l.add(baseline);
+ l.add(increasedNegativeExamples);
+ l.add(noNoise);
+ l.add(useLemma);
+
+ return l;
+ }
public static void conductExperiment(Examples allExamples, IteratedConfig config) {
Examples tmp = new Examples();
tmp.addPosTrain(allExamples.getPosTrain());
tmp.addNegTrain(allExamples.getNegTrain());
- Examples learn = new ExMakerFixedSize(tmp).select(config.initialsplits, config.initialsplits);
+ Examples learn = new ExMakerFixedSize(tmp).select(config.initialsplits, config.initialsplits+config.negativeSplitAdd);
tmp = null;
logger.debug("Total set \n" + allExamples);
logger.debug("Initial training set \n" + learn);
SortedSet<String> posAsPos = new TreeSet<String>();
+ SortedSet<String> posAsNeg = new TreeSet<String>();
+ SortedSet<String> negAsNeg = new TreeSet<String>();
+ SortedSet<String> negAsPos = new TreeSet<String>();
+
+
SortedSet<String> retrieved = new TreeSet<String>();
SortedSet<String> newTestRetrieved = new TreeSet<String>();
SortedSet<String> newTrainRetrieved = new TreeSet<String>();
@@ -271,6 +302,7 @@
double precision = 0.0;
double recall = 0.0;
double fmeasure = 0.0;
+ double accuracy = 0.0;
for (int i = 0; config.stopCondition(i, precision, recall, fmeasure, lastConcept); i++) {
Monitor iterationTime = JamonMonitorLogger.getTimeMonitor(TestIterativeLearning.class,
@@ -288,21 +320,27 @@
Monitor queryTime = JamonMonitorLogger.getTimeMonitor(TestIterativeLearning.class, "queryTime")
.start();
retrieved = getSentences(ed, config.resultLimit);
+ logger.debug("retrieved: "+retrieved.size());
queryTime.stop();
// remove all that are not to be tested
newTestRetrieved = Helper.intersection(allExamples.getTestExamples(), retrieved);
newTrainRetrieved = Helper.intersection(allExamples.getTrainExamples(), retrieved);
-
- SortedSet<String> posAsNegInformative = Helper.difference(allExamples.getPositiveExamples(),
- retrieved);
+ logger.debug("intersection with testset: "+newTestRetrieved.size());
- retrieved = null;
+
// logger.debug("Retrieved "+retrieved.size()+" sentences");
/* MASHING */
// Menge aller positiven geschn. mit den gefundenen
+
+
+
posAsPos = Helper.intersection(newTestRetrieved, allExamples.getPosTest());
+ negAsPos = Helper.intersection(newTestRetrieved, allExamples.getNegTest());
+ posAsNeg = Helper.difference(allExamples.getPosTest(), newTestRetrieved);
+ negAsNeg = Helper.difference(allExamples.getNegTest(), newTestRetrieved );
+// logger.debug("" + posAsPos.size()+"|"+negAsPos.size()+"|"+posAsNeg.size()+"|"+negAsPos.size());
logger.debug("Number of retrieved positives: " + posAsPos.size());
logger.debug("Number of total positives: " + allExamples.getPosTest().size());
@@ -312,11 +350,13 @@
config.add(logRecall, i, recall);
fmeasure = fmeasure(precision, recall);
config.add(logFMeasure, i, fmeasure);
+ accuracy = accuracy(posAsPos.size(), negAsNeg.size(), posAsNeg.size(), negAsPos.size());
+ config.add(logAccuracy, i, accuracy);
// Menge aller positiven geschn. mit den gefundenen
- SortedSet<String> negAsPos = Helper.intersection(newTestRetrieved, allExamples.getNegTest());
+
logger.debug("Number of retrieved negatives: " + negAsPos.size());
- logger.debug("Number of total negatives: " + allExamples.getNegTest().size());
+ logger.debug("Number of total negatives in test set: " + allExamples.getNegTest().size());
logger.debug("Total: " + posAsPos.size() + " + " + negAsPos.size() + " = "
+ newTestRetrieved.size());
@@ -336,11 +376,11 @@
logger.debug("Misclassified: " + misclassifiedNegInStore.size()
+ " negative sentences in store (printing " + printSentences + "):");
_getLabels(misclassifiedNegInStore, printSentences);
- logger.debug("Not found positives: " + posAsNegInformative.size()
+ logger.debug("Not found positives: " + posAsNeg.size()
+ " positive sentences in store (printing " + printSentences + "):");
- _getLabels(posAsNegInformative, printSentences);
+ _getLabels(posAsNeg, printSentences);
- newlyFound = new ExMakerFixedSize(newlyFound).select(config.splits, config.splits);
+ newlyFound = new ExMakerFixedSize(newlyFound).select(config.splits, config.splits+config.negativeSplitAdd);
learn.addPosTrain(newlyFound.getPosTrain());
learn.addNegTrain(newlyFound.getNegTrain());
@@ -361,7 +401,11 @@
}
public static double accuracy(int posAsPos, int negAsNeg, int posAsNeg, int negAsPos) {
- return 0.0d;
+ int upper = posAsPos+negAsNeg;
+ int lower = posAsPos+negAsNeg + posAsNeg + negAsPos;
+ double accuracy = ((double)upper)/((double)lower);
+ logger.debug("Accuracy: " + df.format(accuracy));
+ return accuracy;
}
public static double fmeasure(double precision, double recall) {
@@ -549,7 +593,7 @@
int maxExecutionTime = config.maxExecutionTime;
int valueFrequencyThreshold = ex.getPosTrain().size();
- int noise = config.noise + (iteration);
+ int noise = config.noise + (config.noiseIterationFactor * iteration);
if (config.adaptMaxRuntime) {
maxExecutionTime = (int) Math.floor(config.factor * (double) ex.sizeOfTrainingSets());
// valueFrequencyThreshold = (int)
Modified: trunk/src/dl-learner/org/dllearner/utilities/experiments/IteratedConfig.java
===================================================================
--- trunk/src/dl-learner/org/dllearner/utilities/experiments/IteratedConfig.java 2010-02-16 20:04:11 UTC (rev 2051)
+++ trunk/src/dl-learner/org/dllearner/utilities/experiments/IteratedConfig.java 2010-02-17 00:06:10 UTC (rev 2052)
@@ -11,14 +11,17 @@
public int splits = 5;
public int initialsplits = 5;
+ public int negativeSplitAdd = 0;
public boolean useStartClass = true;
public boolean searchTree = false;
public int noise = 5;
+ public int noiseIterationFactor = 1;
//sets ValueFrequency treshold and maxExecution time
public boolean adaptMaxRuntime = true;
public int maxExecutionTime = 20;
public double factor = 2.0d ;//1.5d;
+
public boolean useDataHasValue = true;
// private String highestPrecision = "";
Modified: trunk/src/dl-learner/org/dllearner/utilities/experiments/Table.java
===================================================================
--- trunk/src/dl-learner/org/dllearner/utilities/experiments/Table.java 2010-02-16 20:04:11 UTC (rev 2051)
+++ trunk/src/dl-learner/org/dllearner/utilities/experiments/Table.java 2010-02-17 00:06:10 UTC (rev 2052)
@@ -13,7 +13,6 @@
import org.dllearner.utilities.JamonMonitorLogger;
import com.jamonapi.Monitor;
-import com.jamonapi.MonitorFactory;
public class Table implements Serializable{
private static final long serialVersionUID = -7191672899557577952L;
@@ -25,9 +24,7 @@
enum Formats {
LATEX, GNUPLOT
};
-
-// private Map<String, TableRowColumn> m = new HashMap<String, TableRowColumn>();
private SortedSet<String> experimentNames = new TreeSet<String>();
private SortedSet<String> labels = new TreeSet<String>();
@@ -38,10 +35,6 @@
int tablesize = 10;
int nrOfMonitors = 10;
Table t = new Table();
-// ITable st = (ITable) MonProxyFactory.monitor(new Table());
-// Table t = (Table)t;
- MonitorFactory.getMonitor(new MyMonKey("aa","aa"));
- MonitorFactory.getTimeMonitor("ss");
Random r = new Random();
for (int i = 0; i < tablesize; i++) {
@@ -53,18 +46,33 @@
m[a].add(r.nextDouble());
}
-// System.out.println("avg: " + m[a].getAvg());
}
TableRowColumn trc = new TableRowColumn(m, "Test","entry_" + i);
-// trc.deleteAll();
trc.useStdDev=false;
t.addTableRowColumn(trc);
}
+ for (int i = 0; i < tablesize; i++) {
+ Monitor[] m = new Monitor[nrOfMonitors];
+ for (int a = 0; a < nrOfMonitors; a++) {
+ m[a] = JamonMonitorLogger.getStatisticMonitor("test" + i + "" + a,
+ (a==0)?"":JamonMonitorLogger.PERCENTAGE);
+ for (int b = 0; b < 2; b++) {
+ m[a].add(r.nextDouble());
+
+ }
+ }
+ TableRowColumn trc = new TableRowColumn(m, "Whatever","bentry_" + i);
+ trc.useStdDev=false;
+ t.addTableRowColumn(trc);
+ }
+
+
System.out.println(t.getLatexAsColumn(true));
- t.serialize("test.ser");
- Table n = deserialize("test.ser");
- System.out.println(n.getLatexAsColumn(true));
+ t.sortByLabel();
+ System.out.println(t.getLatexAsColumn(true));
+ t.sortByExperimentName();
+ System.out.println(t.getGnuPlotAsColumn(true));
// System.out.println(t.getLatexAsRows());
// System.out.println(t.getGnuPlotAsColumn(true));
// System.out.println(t.getGnuPlotAsRows());
@@ -73,21 +81,31 @@
// JamonMonitorLogger.writeHTMLReport("log/tiger.html");
}
- public void addTableRowColumn(List<TableRowColumn> trcs) {
- for (TableRowColumn tableRowColumn : trcs) {
- labels.add(tableRowColumn.getLabel());
- experimentNames.add(tableRowColumn.getExperimentName());
- addTableRowColumn(tableRowColumn);
- }
- }
+
+ /**
+ * passes each it TableRowColumn one by one to addTableRowColumn
+ * @param t
+ */
public void addTable(Table t){
for (TableRowColumn trc : t.tableRowColumns) {
addTableRowColumn(trc);
}
}
+ /**
+ * passes it one by one to addTableRowColumn
+ * @param trcs
+ */
+ public void addTableRowColumns(List<TableRowColumn> trcs) {
+ for (TableRowColumn tableRowColumn : trcs) {
+ addTableRowColumn(tableRowColumn);
+ }
+ }
+
public void addTableRowColumn(TableRowColumn trc) {
+ labels.add(trc.getLabel());
+ experimentNames.add(trc.getExperimentName());
try{
trc.toLatexRow();
}catch (NullPointerException e) {
@@ -139,21 +157,23 @@
for (int a = 0; a < tableRowColumns.size(); a++) {
TableRowColumn trc = tableRowColumns.get(a);
+ String header = trc.getExperimentName()+" "+trc.getLabel();
boolean firstColumn = (a==0);
+ boolean lastColumn = (a + 1 == tableRowColumns.size());
for (int i = 1; i < length+1; i++) {
- boolean last = (a + 1 == tableRowColumns.size());
- boolean first = (i==1);
+
+ boolean firstRow = (i==1);
switch (f) {
case LATEX:
- rows[0] += (first?trc.getHeader()+TableRowColumn.latexSep:"");
- rows[i] += (firstColumn&&addNumbersInFront?i+TableRowColumn.latexSep:"");
- rows[i] += trc.getLatexEntry(i-1)
- + ((last) ? TableRowColumn.latexSep : TableRowColumn.latexSep);
+ rows[0] += ((firstColumn&&firstRow&&addNumbersInFront)?TableRowColumn.latexSep:"");
+ rows[0] += (firstRow?header+TableRowColumn.latexSep:"");
+ rows[i] += ((firstColumn&&addNumbersInFront)?i+TableRowColumn.latexSep:"");
+ rows[i] += trc.getLatexEntry(i-1)+ ((lastColumn) ? TableRowColumn.latexSep : TableRowColumn.latexSep);
break;
case GNUPLOT:
- rows[0] += (first?"#"+trc.getHeader()+"\t":"");
+ rows[0] += (firstRow?"#"+header+"\t":"");
rows[i] += (firstColumn&&addNumbersInFront?i+"\t":"");
- rows[i] += trc.getGnuPlotEntry(i-1) + ((last) ?"" : "\t");
+ rows[i] += trc.getGnuPlotEntry(i-1) + ((lastColumn) ?"" : "\t");
break;
}
}
@@ -198,7 +218,7 @@
for (String s : l) {
for (TableRowColumn trc : tableRowColumns) {
- if(trc.getLabel().equals(s)){
+ if(trc.getLabel().startsWith(s) && trc.getLabel().contains(s)){
newTrc.add(trc);
}
}
@@ -220,6 +240,7 @@
}
public void write(String folder, String fileprefix){
+ logger.info("Writing results to "+folder+fileprefix);
Files.mkdir(folder);
Files.createFile(new File(folder+fileprefix+"_GNU_ROWS"), getGnuPlotAsRows());
Files.createFile(new File(folder+fileprefix+"_GNU_COLUMNS_I"), getGnuPlotAsColumn(true));
Modified: trunk/src/dl-learner/org/dllearner/utilities/experiments/TableRowColumn.java
===================================================================
--- trunk/src/dl-learner/org/dllearner/utilities/experiments/TableRowColumn.java 2010-02-16 20:04:11 UTC (rev 2051)
+++ trunk/src/dl-learner/org/dllearner/utilities/experiments/TableRowColumn.java 2010-02-17 00:06:10 UTC (rev 2052)
@@ -60,10 +60,6 @@
return experimentName+" "+label+" "+toGnuPlotRow();
}
- public String getHeader(){
- return experimentName+" "+label;
- }
-
public int size() {
return monitors.length;
}
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
