From: Sasa M. <sa...@us...> - 2004-12-15 14:55:50
|
Update of /cvsroot/jrobin/src/org/jrobin/data In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv9264/org/jrobin/data Modified Files: CDef.java DataProcessor.java Def.java PDef.java RpnCalculator.java SDef.java Source.java Added Files: Aggregates.java Aggregator.java Normalizer.java Log Message: MAJOR changes to the DataProcessor. This class MUST be used as a data source for all graphing purposes: - it calculates all DEF, CDEF, SDEF and PDEF values by choosing the optimal time step (not equal to the pixel width as before); this makes DataProcessor (almost) completely independent from the graphing engine. Calculated timestamps and values are compatible with RRDXPORT command. - it calculates all aggregates (gprint-ed values) independently from the actual graph width (in other words, calulcated aggregates will not change if the graph width is changed) - it calculates 95-percentile values - it calculates per-pixel datasource and timestamp values once the number of pixels (graph width) is supplied The graph.* package should no longer perform *any* kind of data calculation without the DataProcessor class (RpnCalculator, for example, should be removed from the graph.* package). Index: DataProcessor.java =================================================================== RCS file: /cvsroot/jrobin/src/org/jrobin/data/DataProcessor.java,v retrieving revision 1.4 retrieving revision 1.5 diff -C2 -d -r1.4 -r1.5 *** DataProcessor.java 8 Dec 2004 14:02:37 -0000 1.4 --- DataProcessor.java 15 Dec 2004 14:55:41 -0000 1.5 *************** *** 44,54 **** * DataProcessor dp = new DataProcessor(t1, t2); * // DEF datasource ! * dp.addDatasource("x", "demo.rrd", "sun", "AVERAGE"); * // DEF datasource ! * dp.addDatasource("y", "demo.rrd", "shade", "AVERAGE"); * // CDEF datasource, z = (x + y) / 2 * dp.addDatasource("z", "x,y,+,2,/"); * // ACTION! * dp.processData(); * System.out.println(dp.dump()); [...1207 lines suppressed...] + long[] t = dp.getTimestampsPerPixel(); + double[] v = dp.getValuesPerPixel("X"); + for(int i = 0; i < t.length; i++) { + System.out.println(t[i] + " " + Util.formatDouble(v[i])); + } + + // aggregates + System.out.println("\nAggregates for X"); + Aggregates agg = dp.getAggregates("X"); + System.out.println(agg.dump()); + System.out.println("\nAggregates for Y"); + agg = dp.getAggregates("Y"); + System.out.println(agg.dump()); + + // 95-percentile + System.out.println("\n95-percentile for X: " + Util.formatDouble(dp.get95Percentile("X"))); + System.out.println("95-percentile for Y: " + Util.formatDouble(dp.get95Percentile("Y"))); } } + Index: RpnCalculator.java =================================================================== RCS file: /cvsroot/jrobin/src/org/jrobin/data/RpnCalculator.java,v retrieving revision 1.2 retrieving revision 1.3 diff -C2 -d -r1.2 -r1.3 *** RpnCalculator.java 25 Nov 2004 12:14:54 -0000 1.2 --- RpnCalculator.java 15 Dec 2004 14:55:41 -0000 1.3 *************** *** 84,87 **** --- 84,89 ---- private static final byte TKN_SECOND = 48; private static final byte TKN_WEEK = 49; + private static final byte TKN_SIGN = 50; + private static final byte TKN_RND = 51; private String rpnExpression; *************** *** 92,96 **** private RpnStack stack = new RpnStack(); private double[] calculatedValues; ! private double[] timestamps; private double timeStep; --- 94,98 ---- private RpnStack stack = new RpnStack(); private double[] calculatedValues; ! private long[] timestamps; private double timeStep; *************** *** 266,269 **** --- 268,277 ---- token.id = TKN_WEEK; } + else if(parsedText.equals("SIGN")) { + token.id = TKN_SIGN; + } + else if(parsedText.equals("RND")) { + token.id = TKN_RND; + } else { token.id = TKN_VAR; *************** *** 464,467 **** --- 472,482 ---- push(getCalendarField(pop(), Calendar.WEEK_OF_YEAR)); break; + case TKN_SIGN: + x1 = pop(); + push(Double.isNaN(x1)? Double.NaN: x1 > 0? +1: x1 < 0? -1: 0); + break; + case TKN_RND: + push(Math.floor(pop() * Math.random())); + break; default: throw new RrdException("Unexpected RPN token encountered, token.id=" + token.id); --- NEW FILE: Aggregator.java --- /* ============================================================ * JRobin : Pure java implementation of RRDTool's functionality * ============================================================ * * Project Info: http://www.jrobin.org * Project Lead: Sasa Markovic (sa...@jr...); * * (C) Copyright 2003, by Sasa Markovic. * * Developers: Sasa Markovic (sa...@jr...) * Arne Vandamme (cob...@jr...) * * This library is free software; you can redistribute it and/or modify it under the terms * of the GNU Lesser General Public License as published by the Free Software Foundation; * either version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. * See the GNU Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public License along with this * library; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, * Boston, MA 02111-1307, USA. */ package org.jrobin.data; import org.jrobin.core.ConsolFuns; import org.jrobin.core.Util; import java.util.List; import java.util.ArrayList; import java.util.Arrays; /** * Class used to calculate aggregated values (MIN, MAX, LAST, FIRST, AVERAGE, TOTAL and 95-PERCENTILE) for * the given arrays of timestamps and associated datasource values. */ public class Aggregator implements ConsolFuns { private long timestamps[], step; private double[] values; /** * Constructs Aggregator object. Note that you have to supply two arrays to the constructor: an array * of timestamps and an array of datasource values. Aggregator assumes that:<p> * <ul> * <li>these two arrays have the same length * <li>the time step between consecutive timestamps is constant. * </ul> * @param timestamps Array of timestamps (in seconds, without milliseconds) * @param values Array of corresponding datasource values */ public Aggregator(long[] timestamps, double[] values) { assert timestamps.length == values.length: "Incompatible timestamps/values arrays (unequal lengths)"; assert timestamps.length >= 2: "At least two timestamps must be supplied"; this.timestamps = timestamps; this.values = values; this.step = timestamps[1] - timestamps[0]; } /** * Returns an object representing all aggregated values (MIN, MAX, LAST, FIRST, AVERAGE, TOTAL) * calculated from arrays supplied in the constructor. * @param tStart Starting timestamp of aggregation interval * @param tEnd Ending timestamp of aggregation interval * @return Object containing all aggregated values. */ public Aggregates getAggregates(long tStart, long tEnd) { Aggregates agg = new Aggregates(); long totalSeconds = 0; boolean firstFound = false; for (int i = 0; i < timestamps.length; i++) { long left = Math.max(timestamps[i] - step, tStart); long right = Math.min(timestamps[i], tEnd); long delta = right - left; if (delta > 0) { double value = values[i]; agg.min = Util.min(agg.min, value); agg.max = Util.max(agg.max, value); if (!firstFound) { agg.first = value; firstFound = true; } agg.last = value; if (!Double.isNaN(value)) { agg.total = Util.sum(agg.total, delta * value); totalSeconds += delta; } } } agg.average = totalSeconds > 0 ? (agg.total / totalSeconds) : Double.NaN; return agg; } /** * Used by ISPs which charge for bandwidth utilization on a "95th percentile" basis.<p> * * The 95th percentile is the highest source value left when the top 5% of a numerically sorted set * of source data is discarded. It is used as a measure of the peak value used when one discounts * a fair amount for transitory spikes. This makes it markedly different from the average.<p> * * Read more about this topic at:<p> * <a href="http://www.red.net/support/resourcecentre/leasedline/percentile.php">Rednet</a> or<br> * <a href="http://www.bytemark.co.uk/support/tech/95thpercentile.html">Bytemark</a>. * * @param tStart Starting timestamp of aggregation interval * @param tEnd Ending timestamp of aggregation interval * @return 95-percentile value */ public double get95Percentile(long tStart, long tEnd) { List valueList = new ArrayList(); // create a list of included datasource values (different from NaN) for (int i = 0; i < timestamps.length; i++) { long left = Math.max(timestamps[i] - step, tStart); long right = Math.min(timestamps[i], tEnd); if (right > left && !Double.isNaN(values[i])) { valueList.add(new Double(values[i])); } } // create an array to work with int count = valueList.size(); if (count > 1) { double[] valuesCopy = new double[count]; for (int i = 0; i < count; i++) { valuesCopy[i] = ((Double) valueList.get(i)).doubleValue(); } // sort array Arrays.sort(valuesCopy); // skip top 5% values count -= (int) Math.ceil(count * 0.05); // if we have anything left... if (count > 0) { return valuesCopy[count - 1]; } } // not enough data available return Double.NaN; } /* public static void main(String[] args) { long[] t = {10, 20, 30, 40}; double[] v = {2, Double.NaN, 3, 1}; Aggregator agg = new Aggregator(t, v); System.out.println(agg.getAggregates(0, 40).dump()); } */ } Index: PDef.java =================================================================== RCS file: /cvsroot/jrobin/src/org/jrobin/data/PDef.java,v retrieving revision 1.1 retrieving revision 1.2 diff -C2 -d -r1.1 -r1.2 *** PDef.java 25 Nov 2004 12:14:54 -0000 1.1 --- PDef.java 15 Dec 2004 14:55:41 -0000 1.2 *************** *** 54,60 **** } ! double getValue(double timestamp) { ! long t = (long) Math.round(timestamp); ! return plottable.getValue(t); } } --- 54,64 ---- } ! void calculateValues() { ! long[] times = getTimestamps(); ! double[] vals = new double[times.length]; ! for(int i = 0; i < times.length; i++) { ! vals[i] = plottable.getValue(times[i]); ! } ! setValues(vals); } } --- NEW FILE: Aggregates.java --- /* ============================================================ * JRobin : Pure java implementation of RRDTool's functionality * ============================================================ * * Project Info: http://www.jrobin.org * Project Lead: Sasa Markovic (sa...@jr...); * * (C) Copyright 2003, by Sasa Markovic. * * Developers: Sasa Markovic (sa...@jr...) * Arne Vandamme (cob...@jr...) * * This library is free software; you can redistribute it and/or modify it under the terms * of the GNU Lesser General Public License as published by the Free Software Foundation; * either version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. * See the GNU Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public License along with this * library; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, * Boston, MA 02111-1307, USA. */ package org.jrobin.data; import org.jrobin.core.ConsolFuns; import org.jrobin.core.RrdException; import org.jrobin.core.Util; /** * Simple class which holds aggregated values (MIN, MAX, FIRST, LAST, AVERAGE and TOTAL). You * don't need to create objects of this class directly. Objects of this class are returned from * <code>getAggregates()</code> method in * {@link org.jrobin.core.FetchData#getAggregates(String) FetchData} and * {@link DataProcessor#getAggregates(String)} DataProcessor} classes. */ public class Aggregates implements ConsolFuns { double min = Double.NaN, max = Double.NaN; double first = Double.NaN, last = Double.NaN; double average = Double.NaN, total = Double.NaN; Aggregates() { // NOP; } /** * Returns the minimal value * @return Minimal value */ public double getMin() { return min; } /** * Returns the maximum value * @return Maximum value */ public double getMax() { return max; } /** * Returns the first falue * @return First value */ public double getFirst() { return first; } /** * Returns the last value * @return Last value */ public double getLast() { return last; } /** * Returns average * @return Average value */ public double getAverage() { return average; } /** * Returns total value * @return Total value */ public double getTotal() { return total; } /** * Returns single aggregated value for the give consolidation function * @param consolFun Consolidation function: MIN, MAX, FIRST, LAST, AVERAGE, TOTAL. These constanst * are conveniently defined in the {@link org.jrobin.core.ConsolFuns ConsolFuns} interface. * @return Aggregated value * @throws RrdException Thrown if unsupported consolidation function is supplied */ public double getAggregate(String consolFun) throws RrdException { if(consolFun.equals(CF_AVERAGE)) { return average; } else if(consolFun.equals(CF_FIRST)) { return first; } else if(consolFun.equals(CF_LAST)) { return last; } else if(consolFun.equals(CF_MAX)) { return max; } else if(consolFun.equals(CF_MIN)) { return min; } else if(consolFun.equals(CF_TOTAL)) { return total; } else { throw new RrdException("Unknown consolidation function: " + consolFun); } } /** * Returns String representing all aggregated values. Just for debugging purposes. * @return String containing all aggregated values */ public String dump() { return "MIN=" + Util.formatDouble(min) + ", MAX=" + Util.formatDouble(max) + "\n" + "FIRST=" + Util.formatDouble(first) + ", LAST=" + Util.formatDouble(last) + "\n" + "AVERAGE=" + Util.formatDouble(average) + ", TOTAL=" + Util.formatDouble(total); } } Index: SDef.java =================================================================== RCS file: /cvsroot/jrobin/src/org/jrobin/data/SDef.java,v retrieving revision 1.1 retrieving revision 1.2 diff -C2 -d -r1.1 -r1.2 *** SDef.java 25 Nov 2004 12:14:54 -0000 1.1 --- SDef.java 15 Dec 2004 14:55:41 -0000 1.2 *************** *** 26,32 **** --- 26,35 ---- package org.jrobin.data; + import org.jrobin.core.RrdException; + class SDef extends Source { private String defName; private String consolFun; + private double value; SDef(String name, String defName, String consolFun) { *************** *** 44,53 **** } ! void setValue(double value, int count) { double[] values = new double[count]; ! for(int i = 0; i < values.length; i++) { values[i] = value; } setValues(values); } } --- 47,69 ---- } ! void setValue(double value) { ! this.value = value; ! int count = getTimestamps().length; double[] values = new double[count]; ! for(int i = 0; i < count; i++) { values[i] = value; } setValues(values); } + + Aggregates getAggregates(long tStart, long tEnd) throws RrdException { + Aggregates agg = new Aggregates(); + agg.first = agg.last = agg.min = agg.max = agg.average = value; + agg.total = value * (tEnd - tStart); + return agg; + } + + double get95Percentile(long tStart, long tEnd) throws RrdException { + return value; + } } Index: Source.java =================================================================== RCS file: /cvsroot/jrobin/src/org/jrobin/data/Source.java,v retrieving revision 1.2 retrieving revision 1.3 diff -C2 -d -r1.2 -r1.3 *** Source.java 8 Dec 2004 14:02:37 -0000 1.2 --- Source.java 15 Dec 2004 14:55:41 -0000 1.3 *************** *** 28,38 **** import org.jrobin.core.ConsolFuns; import org.jrobin.core.RrdException; - import org.jrobin.core.Util; - - import java.util.Arrays; abstract class Source implements ConsolFuns { final private String name; ! private double[] values; Source(String name) { --- 28,36 ---- import org.jrobin.core.ConsolFuns; import org.jrobin.core.RrdException; abstract class Source implements ConsolFuns { final private String name; ! protected double[] values; ! protected long[] timestamps; Source(String name) { *************** *** 40,154 **** } ! final String getName() { return name; } ! final void setValues(double[] values) { this.values = values; } ! final double[] getValues() { ! return values; ! } ! ! final double getAggregate(String consolFun, double secondsPerPixel) throws RrdException { ! if(values == null) { ! throw new RrdException("Could not calculate " + consolFun + ! " for datasource [" + name + "], datasource values are still not available"); ! } ! if(consolFun.equals(ConsolFuns.CF_FIRST)) { ! return values[1]; ! } ! else if(consolFun.equals(ConsolFuns.CF_LAST)) { ! return values[values.length - 1]; ! } ! else if(consolFun.equals(ConsolFuns.CF_MIN)) { ! return getMin(); ! } ! else if(consolFun.equals(ConsolFuns.CF_MAX)) { ! return getMax(); ! } ! else if(consolFun.equals(ConsolFuns.CF_AVERAGE)) { ! return getAverage(); ! } ! else if(consolFun.equals(ConsolFuns.CF_TOTAL)) { ! return getTotal(secondsPerPixel); ! } ! else { ! throw new RrdException("Unsupported consolidation function: " + consolFun); ! } ! } ! ! final double getPercentile(int percentile) throws RrdException { ! if(values == null) { ! throw new RrdException("Could not calculate 95th percentile for datasource [" + ! name + "], datasource values are still not available"); ! } ! if(percentile > 100 || percentile <= 0) { ! throw new RrdException("Invalid percentile specified: " + percentile); ! } ! return getPercentile(values, percentile); ! } ! ! private static double getPercentile(double[] values, int percentile) { ! int count = values.length; ! double[] valuesCopy = new double[count]; ! for(int i = 0; i < count; i++) { ! valuesCopy[i] = values[i]; ! } ! Arrays.sort(valuesCopy); ! // NaN values are at the end, eliminate them from consideration ! while(count > 0 && Double.isNaN(valuesCopy[count - 1])) { ! count--; ! } ! // skip top [percentile]% values ! int skipCount = (int) Math.ceil(((100 - percentile) * count) / 100.0); ! count -= skipCount; ! // if we have anything left... ! if(count > 0) { ! return valuesCopy[count - 1]; ! } ! else { ! // not enough data available ! return Double.NaN; ! } } ! private double getTotal(double secondsPerPixel) { ! double sum = 0; ! for(int i = 1; i < values.length; i++) { ! if(!Double.isNaN(values[i])) { ! sum += values[i]; ! } ! } ! return sum * secondsPerPixel; } ! private double getAverage() { ! double sum = 0; ! int count = 0; ! for(int i = 1; i < values.length; i++) { ! if(!Double.isNaN(values[i])) { ! sum += values[i]; ! count++; ! } ! } ! return sum / count; } ! private double getMax() { ! double max = Double.NaN; ! for(int i = 1; i < values.length; i++) { ! max = Util.max(max, values[i]); ! } ! return max; } ! private double getMin() { ! double min = Double.NaN; ! for(int i = 1; i < values.length; i++) { ! min = Util.min(min, values[i]); ! } ! return min; } } --- 38,69 ---- } ! String getName() { return name; } ! void setValues(double[] values) { this.values = values; } ! void setTimestamps(long[] timestamps) { ! this.timestamps = timestamps; } ! double[] getValues() { ! return values; } ! long[] getTimestamps() { ! return timestamps; } ! Aggregates getAggregates(long tStart, long tEnd) throws RrdException { ! Aggregator agg = new Aggregator(timestamps, values); ! return agg.getAggregates(tStart, tEnd); } ! double get95Percentile(long tStart, long tEnd) throws RrdException { ! Aggregator agg = new Aggregator(timestamps, values); ! return agg.get95Percentile(tStart, tEnd); } } Index: Def.java =================================================================== RCS file: /cvsroot/jrobin/src/org/jrobin/data/Def.java,v retrieving revision 1.2 retrieving revision 1.3 diff -C2 -d -r1.2 -r1.3 *** Def.java 26 Nov 2004 13:47:51 -0000 1.2 --- Def.java 15 Dec 2004 14:55:41 -0000 1.3 *************** *** 35,39 **** private String path, dsName, consolFun, backend; private FetchData fetchData; ! private long endingFetchTimestamp; Def(String name, String path, String dsName, String consolFunc) { --- 35,39 ---- private String path, dsName, consolFun, backend; private FetchData fetchData; ! private long lastValidTimestamp, fetchStep; Def(String name, String path, String dsName, String consolFunc) { *************** *** 78,82 **** void setFetchData(FetchData fetchData) throws IOException { this.fetchData = fetchData; ! this.endingFetchTimestamp = fetchData.getMatchingArchive().getEndTime(); } --- 78,83 ---- void setFetchData(FetchData fetchData) throws IOException { this.fetchData = fetchData; ! this.lastValidTimestamp = fetchData.getMatchingArchive().getEndTime(); ! this.fetchStep = fetchData.getMatchingArchive().getArcStep(); } *************** *** 89,94 **** } ! long getEndingFetchTimestamp() throws IOException { ! return endingFetchTimestamp; } } --- 90,113 ---- } ! long getLastValidTimestamp() { ! return lastValidTimestamp; ! } ! ! long getFetchStep() { ! return fetchStep; ! } ! ! Aggregates getAggregates(long tStart, long tEnd) throws RrdException { ! long[] t = getRrdTimestamps(); ! double[] v = getRrdValues(); ! Aggregator agg = new Aggregator(t, v); ! return agg.getAggregates(tStart, tEnd); ! } ! ! double get95Percentile(long tStart, long tEnd) throws RrdException { ! long[] t = getRrdTimestamps(); ! double[] v = getRrdValues(); ! Aggregator agg = new Aggregator(t, v); ! return agg.get95Percentile(tStart, tEnd); } } --- NEW FILE: Normalizer.java --- /* ============================================================ * JRobin : Pure java implementation of RRDTool's functionality * ============================================================ * * Project Info: http://www.jrobin.org * Project Lead: Sasa Markovic (sa...@jr...); * * (C) Copyright 2003, by Sasa Markovic. * * Developers: Sasa Markovic (sa...@jr...) * Arne Vandamme (cob...@jr...) * * This library is free software; you can redistribute it and/or modify it under the terms * of the GNU Lesser General Public License as published by the Free Software Foundation; * either version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. * See the GNU Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public License along with this * library; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, * Boston, MA 02111-1307, USA. */ package org.jrobin.data; import org.jrobin.core.Util; import java.util.Arrays; class Normalizer { private long[] timestamps; int count; long step; Normalizer(long[] timestamps) { this.timestamps = timestamps; this.step = timestamps[1] - timestamps[0]; this.count = timestamps.length; } double[] normalize(long[] rawTimestamps, double[] rawValues) { int rawCount = rawTimestamps.length; long rawStep = rawTimestamps[1] - rawTimestamps[0]; // check if we have a simple match if(rawCount == count && rawStep == step && rawTimestamps[0] == timestamps[0]) { return getCopyOf(rawValues); } // reset all normalized values to NaN double[] values = new double[count]; Arrays.fill(values, Double.NaN); for (int rawSeg = 0, seg = 0; rawSeg < rawCount && seg < count; rawSeg++) { double rawValue = rawValues[rawSeg]; if (!Double.isNaN(rawValue)) { long rawLeft = rawTimestamps[rawSeg] - rawStep; while (seg < count && rawLeft >= timestamps[seg]) { seg++; } boolean overlap = true; for (int fillSeg = seg; overlap && fillSeg < count; fillSeg++) { long left = timestamps[fillSeg] - step; long t1 = Math.max(rawLeft, left); long t2 = Math.min(rawTimestamps[rawSeg], timestamps[fillSeg]); if (t1 < t2) { values[fillSeg] = Util.sum(values[fillSeg], (t2 - t1) * rawValues[rawSeg]); } else { overlap = false; } } } } for (int seg = 0; seg < count; seg++) { values[seg] /= step; } return values; } private static double[] getCopyOf(double[] rawValues) { int n = rawValues.length; double[] values = new double[n]; for(int i = 0; i < n; i++) { values[i] = rawValues[i]; } return values; } private static void dump(long[] t, double[] v) { for(int i = 0; i < v.length; i++) { System.out.print("[" + t[i] + "," + v[i] + "] "); } System.out.println(""); } public static void main(String[] args) { long rawTime[] = {100, 120, 140, 160, 180, 200}; double rawValues[] = {10, 30, 20, Double.NaN, 50, 40}; long time[] = {60, 100, 140, 180, 220, 260, 300}; Normalizer n = new Normalizer(time); double[] values = n.normalize(rawTime, rawValues); dump(rawTime, rawValues); dump(time, values); } } |