Menu
▾
▴
[Bigdata-commit] SF.net SVN: bigdata:[4920] branches
|
From: <tho...@us...> - 2011-07-15 13:33:17
|
Revision: 4920
http://bigdata.svn.sourceforge.net/bigdata/?rev=4920&view=rev
Author: thompsonbry
Date: 2011-07-15 13:33:05 +0000 (Fri, 15 Jul 2011)
Log Message:
-----------
Added support for collecting OSX performance counters to both the 1.0.0 release branch for the 1.0.1 dot release and to the current development branch. This closes out [1]. However, note that we are not able to collect IO Wait under OSX. Nor are we able to report the IO statistics broken out by reads and writes. See [1] for more details on this. It is a problem with the OSX command line utilities for reporting performance counters.
[1] https://sourceforge.net/apps/trac/bigdata/ticket/225#comment:12
Modified Paths:
--------------
branches/BIGDATA_RELEASE_1_0_0/bigdata/src/java/com/bigdata/counters/AbstractStatisticsCollector.java
branches/BIGDATA_RELEASE_1_0_0/bigdata/src/java/com/bigdata/counters/IHostCounters.java
branches/BIGDATA_RELEASE_1_0_0/bigdata/src/java/com/bigdata/counters/IRequiredHostCounters.java
branches/BIGDATA_RELEASE_1_0_0/bigdata/src/java/com/bigdata/counters/linux/KernelVersion.java
branches/BIGDATA_RELEASE_1_0_0/bigdata/src/java/com/bigdata/counters/linux/VMStatCollector.java
branches/BIGDATA_RELEASE_1_0_0/bigdata/src/test/com/bigdata/counters/TestAll.java
branches/BIGDATA_RELEASE_1_0_0/bigdata/src/test/com/bigdata/counters/linux/TestParsing.java
branches/TERMS_REFACTOR_BRANCH/bigdata/src/java/com/bigdata/counters/AbstractStatisticsCollector.java
branches/TERMS_REFACTOR_BRANCH/bigdata/src/java/com/bigdata/counters/IHostCounters.java
branches/TERMS_REFACTOR_BRANCH/bigdata/src/java/com/bigdata/counters/IRequiredHostCounters.java
branches/TERMS_REFACTOR_BRANCH/bigdata/src/java/com/bigdata/counters/linux/KernelVersion.java
branches/TERMS_REFACTOR_BRANCH/bigdata/src/java/com/bigdata/counters/linux/VMStatCollector.java
branches/TERMS_REFACTOR_BRANCH/bigdata/src/test/com/bigdata/counters/TestAll.java
branches/TERMS_REFACTOR_BRANCH/bigdata/src/test/com/bigdata/counters/linux/TestParsing.java
Added Paths:
-----------
branches/BIGDATA_RELEASE_1_0_0/bigdata/src/java/com/bigdata/counters/osx/
branches/BIGDATA_RELEASE_1_0_0/bigdata/src/java/com/bigdata/counters/osx/IOStatCollector.java
branches/BIGDATA_RELEASE_1_0_0/bigdata/src/java/com/bigdata/counters/osx/StatisticsCollectorForOSX.java
branches/BIGDATA_RELEASE_1_0_0/bigdata/src/java/com/bigdata/counters/osx/VMStatCollector.java
branches/BIGDATA_RELEASE_1_0_0/bigdata/src/java/com/bigdata/counters/osx/package.html
branches/BIGDATA_RELEASE_1_0_0/bigdata/src/java/org/apache/system/MacOSX.java
branches/BIGDATA_RELEASE_1_0_0/bigdata/src/test/com/bigdata/counters/osx/
branches/BIGDATA_RELEASE_1_0_0/bigdata/src/test/com/bigdata/counters/osx/AbstractParserTestCase.java
branches/BIGDATA_RELEASE_1_0_0/bigdata/src/test/com/bigdata/counters/osx/TestAll.java
branches/BIGDATA_RELEASE_1_0_0/bigdata/src/test/com/bigdata/counters/osx/TestParse_iostat.java
branches/BIGDATA_RELEASE_1_0_0/bigdata/src/test/com/bigdata/counters/osx/TestParse_vm_stat.java
branches/TERMS_REFACTOR_BRANCH/bigdata/src/java/com/bigdata/counters/osx/
branches/TERMS_REFACTOR_BRANCH/bigdata/src/java/com/bigdata/counters/osx/IOStatCollector.java
branches/TERMS_REFACTOR_BRANCH/bigdata/src/java/com/bigdata/counters/osx/StatisticsCollectorForOSX.java
branches/TERMS_REFACTOR_BRANCH/bigdata/src/java/com/bigdata/counters/osx/VMStatCollector.java
branches/TERMS_REFACTOR_BRANCH/bigdata/src/java/com/bigdata/counters/osx/package.html
branches/TERMS_REFACTOR_BRANCH/bigdata/src/java/org/apache/system/MacOSX.java
branches/TERMS_REFACTOR_BRANCH/bigdata/src/test/com/bigdata/counters/osx/
branches/TERMS_REFACTOR_BRANCH/bigdata/src/test/com/bigdata/counters/osx/AbstractParserTestCase.java
branches/TERMS_REFACTOR_BRANCH/bigdata/src/test/com/bigdata/counters/osx/TestAll.java
branches/TERMS_REFACTOR_BRANCH/bigdata/src/test/com/bigdata/counters/osx/TestParse_iostat.java
branches/TERMS_REFACTOR_BRANCH/bigdata/src/test/com/bigdata/counters/osx/TestParse_vm_stat.java
Modified: branches/BIGDATA_RELEASE_1_0_0/bigdata/src/java/com/bigdata/counters/AbstractStatisticsCollector.java
===================================================================
--- branches/BIGDATA_RELEASE_1_0_0/bigdata/src/java/com/bigdata/counters/AbstractStatisticsCollector.java 2011-07-14 20:11:44 UTC (rev 4919)
+++ branches/BIGDATA_RELEASE_1_0_0/bigdata/src/java/com/bigdata/counters/AbstractStatisticsCollector.java 2011-07-15 13:33:05 UTC (rev 4920)
@@ -44,6 +44,7 @@
import com.bigdata.LRUNexus;
import com.bigdata.counters.httpd.CounterSetHTTPD;
import com.bigdata.counters.linux.StatisticsCollectorForLinux;
+import com.bigdata.counters.osx.StatisticsCollectorForOSX;
import com.bigdata.counters.win.StatisticsCollectorForWindows;
import com.bigdata.io.DirectBufferPool;
import com.bigdata.rawstore.Bytes;
@@ -568,16 +569,25 @@
throw new IllegalArgumentException(
"Required option not specified: " + Options.PROCESS_NAME);
- final String osname = System.getProperty("os.name").toLowerCase();
-
- if(osname.equalsIgnoreCase("linux")) {
+// final String osname = System.getProperty("os.name").toLowerCase();
+//
+// if(osname.equalsIgnoreCase("linux")) {
+ if(SystemUtil.isLinux()) {
return new StatisticsCollectorForLinux(interval, processName);
- } else if(osname.contains("windows")) {
+// } else if(osname.contains("windows")) {
+ } else if(SystemUtil.isWindows()) {
+
return new StatisticsCollectorForWindows(interval);
+// } else if(osname.contains("os x")) {
+
+ } else if(SystemUtil.isOSX()) {
+
+ return new StatisticsCollectorForOSX(interval, processName);
+
} else {
throw new UnsupportedOperationException(
@@ -641,7 +651,7 @@
if (count < 0)
throw new RuntimeException("count must be non-negative");
- Properties properties = new Properties(System.getProperties());
+ final Properties properties = new Properties(System.getProperties());
if (nargs != 0) {
Modified: branches/BIGDATA_RELEASE_1_0_0/bigdata/src/java/com/bigdata/counters/IHostCounters.java
===================================================================
--- branches/BIGDATA_RELEASE_1_0_0/bigdata/src/java/com/bigdata/counters/IHostCounters.java 2011-07-14 20:11:44 UTC (rev 4919)
+++ branches/BIGDATA_RELEASE_1_0_0/bigdata/src/java/com/bigdata/counters/IHostCounters.java 2011-07-15 13:33:05 UTC (rev 4920)
@@ -59,21 +59,21 @@
/**
* Percentage of the time the processor is not idle that it is executing
- * at the user level (normalized to 100% in single CPU and SMP
+ * at the user level in [0:1] (normalized to 100% in single CPU and SMP
* environments).
*/
String CPU_PercentUserTime = CPU + ps + "% User Time";
- /**
- * Percentage of the time the processor is not idle that it is executing
- * at the system (aka kernel) level (normalized to 100% in single CPU
- * and SMP environments).
- */
+ /**
+ * Percentage of the time the processor is not idle that it is executing at
+ * the system (aka kernel) level in [0:1] (normalized to 100% in single CPU
+ * and SMP environments).
+ */
String CPU_PercentSystemTime = CPU + ps + "% System Time";
/**
* Percentage of the time the CPU(s) were idle while the system had an
- * outstanding IO.
+ * outstanding IO in [0:1].
* <p>
* Note: The Windows platform does not appear to be able to report this
* counter. If it did I would move this into the "required" category. The
@@ -136,4 +136,17 @@
String PhysicalDisk_WritesPerSec = PhysicalDisk + ps
+ "Writes Per Second";
+ /**
+ * Disk bytes per second for the host (total of bytes read per second and
+ * bytes written per second).
+ */
+ String PhysicalDisk_BytesPerSec = PhysicalDisk + ps + "Bytes Per Second";
+
+ /**
+ * Disk operations per second for the host (total of disk reads per second
+ * and disk writes per second).
+ */
+ String PhysicalDisk_TransfersPerSec = PhysicalDisk + ps
+ + "Transfers Per Second";
+
}
Modified: branches/BIGDATA_RELEASE_1_0_0/bigdata/src/java/com/bigdata/counters/IRequiredHostCounters.java
===================================================================
--- branches/BIGDATA_RELEASE_1_0_0/bigdata/src/java/com/bigdata/counters/IRequiredHostCounters.java 2011-07-14 20:11:44 UTC (rev 4919)
+++ branches/BIGDATA_RELEASE_1_0_0/bigdata/src/java/com/bigdata/counters/IRequiredHostCounters.java 2011-07-15 13:33:05 UTC (rev 4920)
@@ -73,9 +73,14 @@
/*
* CPU
+ *
+ * TODO It would be great to have IO Wait as a required counter, but we
+ * do not have access to it under Windows (last I checked).
+ *
+ * See IHostCounters#CPU_PercentIOWait
*/
- /** Percentage of the time the processor is not idle. */
+ /** Percentage of the time the processor is not idle in [0:1]. */
String CPU_PercentProcessorTime = CPU + ps
+ "% Processor Time";
Modified: branches/BIGDATA_RELEASE_1_0_0/bigdata/src/java/com/bigdata/counters/linux/KernelVersion.java
===================================================================
--- branches/BIGDATA_RELEASE_1_0_0/bigdata/src/java/com/bigdata/counters/linux/KernelVersion.java 2011-07-14 20:11:44 UTC (rev 4919)
+++ branches/BIGDATA_RELEASE_1_0_0/bigdata/src/java/com/bigdata/counters/linux/KernelVersion.java 2011-07-15 13:33:05 UTC (rev 4920)
@@ -9,6 +9,7 @@
import java.util.regex.Pattern;
import org.apache.log4j.Logger;
+import org.apache.system.SystemUtil;
/**
* Reports on the kernel version for a linux host.
@@ -48,6 +49,13 @@
minor = Integer.parseInt(m.group(3));
}
+
+ public String toString() {
+
+ return "{version=" + version + ", major=" + major + ", minor=" + minor
+ + "}";
+
+ }
/**
* Return the version of the Linux kernel as reported by
@@ -66,11 +74,27 @@
try {
- commands.add("/bin/uname");
+ if (SystemUtil.isLinux()) {
+
+ commands.add("/bin/uname");
+
+ } else if (SystemUtil.isOSX()) {
+
+ commands.add("/usr/bin/uname");
+
+ } else {
+
+ /*
+ * Attempt uname -r at with this path, but this is an unknown
+ * OS.
+ */
+ commands.add("/bin/uname");
+
+ }
commands.add("-r");
- ProcessBuilder pb = new ProcessBuilder(commands);
+ final ProcessBuilder pb = new ProcessBuilder(commands);
pr = pb.start();
@@ -117,11 +141,25 @@
} else {
- throw new RuntimeException("Could not get PID: exitValue="
- + pr.exitValue());
+ throw new RuntimeException(
+ "Could not get kernal version: exitValue=" + pr.exitValue());
}
}
+ /**
+ * Prints the data reported by {@link KernelVersion#get()}.
+ *
+ * @param args
+ * ignored.
+ */
+ public static void main(final String[] args) {
+
+ final KernelVersion kver = KernelVersion.get();
+
+ System.out.println("kernel version: "+kver);
+
+ }
+
}
\ No newline at end of file
Modified: branches/BIGDATA_RELEASE_1_0_0/bigdata/src/java/com/bigdata/counters/linux/VMStatCollector.java
===================================================================
--- branches/BIGDATA_RELEASE_1_0_0/bigdata/src/java/com/bigdata/counters/linux/VMStatCollector.java 2011-07-14 20:11:44 UTC (rev 4919)
+++ branches/BIGDATA_RELEASE_1_0_0/bigdata/src/java/com/bigdata/counters/linux/VMStatCollector.java 2011-07-15 13:33:05 UTC (rev 4920)
@@ -139,7 +139,7 @@
* are paths into the {@link CounterSet}. The values are the data most
* recently read from <code>vmstat</code>.
*/
- private Map<String, Object> vals = new HashMap<String, Object>();
+ final private Map<String, Object> vals = new HashMap<String, Object>();
/**
* The timestamp associated with the most recently collected values.
@@ -193,91 +193,85 @@
/**
* Declares the counters that we will collect
*/
- /*synchronized*/ public CounterSet getCounters() {
-
-// if(root == null) {
-
- final CounterSet root = new CounterSet();
-
- inst = new LinkedList<I>();
-
- /*
- * Note: Counters are all declared as Double to facilitate
- * aggregation.
- */
+ public CounterSet getCounters() {
- /*
- * Note: [si] is "the #of blocks swapped in per second."
- */
- inst.add(new DI(IRequiredHostCounters.Memory_majorFaultsPerSecond,
- 1d));
+ final CounterSet root = new CounterSet();
- /*
- * Note: [swpd] is "the amount of virtual memory used". The counter
- * is reported in 1024 byte blocks, so we convert to bytes using a
- * scaling factor.
- *
- * @todo where do I get the amount of swap space available?
- */
- inst.add(new DI(IHostCounters.Memory_SwapBytesUsed, 1024d));
+ inst = new LinkedList<I>();
- /*
- * Note: [free] is "the amount of idle memory". The counter is
- * reported in 1024 byte blocks, so we convert to bytes using a
- * scaling factor.
- */
- inst.add(new DI(IHostCounters.Memory_Bytes_Free, 1024d));
+ /*
+ * Note: Counters are all declared as Double to facilitate aggregation.
+ */
- /*
- * Note: [bi] is "the blocks received from a device / second". The
- * counter is reported in 1024 byte blocks, so we convert to bytes
- * using a scaling factor.
- */
- inst.add(new DI(IRequiredHostCounters.PhysicalDisk_BytesReadPerSec,
- 1024d));
-
- /*
- * Note: [bo] is "the blocks sent to a device / second". The counter
- * is reported in 1024 byte blocks, so we convert to bytes using a
- * scaling factor.
- */
- inst.add(new DI(IRequiredHostCounters.PhysicalDisk_BytesWrittenPerSec,
- 1024d));
-
- if (cpuStats) {
+ /*
+ * Note: [si] is "the #of blocks swapped in per second."
+ */
+ inst.add(new DI(IRequiredHostCounters.Memory_majorFaultsPerSecond, 1d));
- /*
- * Note: vmstat reports percentages in [0:100] so we convert
- * them to [0:1] using a scaling factor.
- */
+ /*
+ * Note: [swpd] is "the amount of virtual memory used". The counter is
+ * reported in 1024 byte blocks, so we convert to bytes using a scaling
+ * factor.
+ *
+ * @todo where do I get the amount of swap space available?
+ */
+ inst.add(new DI(IHostCounters.Memory_SwapBytesUsed, 1024d));
- // Note: processor time = (100-idle), converted to [0:1].
- inst.add(new DI(IRequiredHostCounters.CPU_PercentProcessorTime,.01d));
- // Note: column us
- inst.add(new DI(IHostCounters.CPU_PercentUserTime, .01d));
- // Note: column sy
- inst.add(new DI(IHostCounters.CPU_PercentSystemTime, .01d));
- // Note: column wa
- inst.add(new DI(IHostCounters.CPU_PercentIOWait, .01d));
-
- }
-
- for(Iterator<I> itr = inst.iterator(); itr.hasNext(); ) {
-
- final I i = itr.next();
-
- root.addCounter(i.getPath(), i);
-
- }
-
-// }
-
- return root;
-
- }
- private List<I> inst = null;
-// private CounterSet root = null;
+ /*
+ * Note: [free] is "the amount of idle memory". The counter is reported
+ * in 1024 byte blocks, so we convert to bytes using a scaling factor.
+ */
+ inst.add(new DI(IHostCounters.Memory_Bytes_Free, 1024d));
+ /*
+ * Note: [bi] is "the blocks received from a device / second". The
+ * counter is reported in 1024 byte blocks, so we convert to bytes using
+ * a scaling factor.
+ */
+ inst.add(new DI(IRequiredHostCounters.PhysicalDisk_BytesReadPerSec,
+ 1024d));
+
+ /*
+ * Note: [bo] is "the blocks sent to a device / second". The counter is
+ * reported in 1024 byte blocks, so we convert to bytes using a scaling
+ * factor.
+ */
+ inst.add(new DI(IRequiredHostCounters.PhysicalDisk_BytesWrittenPerSec,
+ 1024d));
+
+ if (cpuStats) {
+
+ /*
+ * Note: vmstat reports percentages in [0:100] so we convert them to
+ * [0:1] using a scaling factor.
+ */
+
+ // Note: processor time = (100-idle), converted to [0:1].
+ inst.add(new DI(IRequiredHostCounters.CPU_PercentProcessorTime,
+ .01d));
+ // Note: column us
+ inst.add(new DI(IHostCounters.CPU_PercentUserTime, .01d));
+ // Note: column sy
+ inst.add(new DI(IHostCounters.CPU_PercentSystemTime, .01d));
+ // Note: column wa
+ inst.add(new DI(IHostCounters.CPU_PercentIOWait, .01d));
+
+ }
+
+ for (Iterator<I> itr = inst.iterator(); itr.hasNext();) {
+
+ final I i = itr.next();
+
+ root.addCounter(i.getPath(), i);
+
+ }
+
+ return root;
+
+ }
+
+ private List<I> inst = null;
+
public AbstractProcessReader getProcessReader() {
return new VMStatReader();
@@ -319,10 +313,6 @@
}
- /**
- *
- * @see TestParsing#test_vmstat_header_and_data_parse()
- */
@Override
protected void readProcess() throws Exception {
Added: branches/BIGDATA_RELEASE_1_0_0/bigdata/src/java/com/bigdata/counters/osx/IOStatCollector.java
===================================================================
--- branches/BIGDATA_RELEASE_1_0_0/bigdata/src/java/com/bigdata/counters/osx/IOStatCollector.java (rev 0)
+++ branches/BIGDATA_RELEASE_1_0_0/bigdata/src/java/com/bigdata/counters/osx/IOStatCollector.java 2011-07-15 13:33:05 UTC (rev 4920)
@@ -0,0 +1,527 @@
+/*
+
+Copyright (C) SYSTAP, LLC 2006-2008. All rights reserved.
+
+Contact:
+ SYSTAP, LLC
+ 4501 Tower Road
+ Greensboro, NC 27410
+ lic...@bi...
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+*/
+/*
+ * Created on Dec 9, 2008
+ */
+
+package com.bigdata.counters.osx;
+
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.regex.Pattern;
+
+import com.bigdata.counters.AbstractProcessCollector;
+import com.bigdata.counters.AbstractProcessReader;
+import com.bigdata.counters.ActiveProcess;
+import com.bigdata.counters.CounterSet;
+import com.bigdata.counters.ICounterHierarchy;
+import com.bigdata.counters.ICounterSet;
+import com.bigdata.counters.IHostCounters;
+import com.bigdata.counters.IInstrument;
+import com.bigdata.counters.IRequiredHostCounters;
+import com.bigdata.counters.ProcessReaderHelper;
+import com.bigdata.rawstore.Bytes;
+
+/**
+ * Collects some counters using <code>iostat</code>. Unfortunately,
+ * <code>iostat</code> does not break down the reads and writes and does not
+ * report IO Wait. This information is obviously available from OSX as it is
+ * provided by the ActivityMonitor, but we can not get it from
+ * <code>iostat</code>.
+ *
+ * @author <a href="mailto:tho...@us...">Bryan Thompson</a>
+ * @version $Id: VMStatCollector.java 4289 2011-03-10 21:22:30Z thompsonbry $
+ */
+public class IOStatCollector extends AbstractProcessCollector implements
+ ICounterHierarchy, IRequiredHostCounters, IHostCounters{
+
+ /**
+ * Inner class integrating the current values with the {@link ICounterSet}
+ * hierarchy.
+ *
+ * @author <a href="mailto:tho...@us...">Bryan
+ * Thompson</a>
+ */
+ abstract class I<T> implements IInstrument<T> {
+
+ protected final String path;
+
+ public String getPath() {
+
+ return path;
+
+ }
+
+ public I(String path) {
+
+ assert path != null;
+
+ this.path = path;
+
+ }
+
+ public long lastModified() {
+
+ return lastModified;
+
+ }
+
+ /**
+ * @throws UnsupportedOperationException
+ * always.
+ */
+ public void setValue(T value, long timestamp) {
+
+ throw new UnsupportedOperationException();
+
+ }
+
+ }
+
+ /**
+ * Double precision counter with scaling factor.
+ *
+ * @author <a href="mailto:tho...@us...">Bryan Thompson</a>
+ */
+ class DI extends I<Double> {
+
+ protected final double scale;
+
+ DI(final String path) {
+
+ this(path,1d);
+
+ }
+
+ DI(final String path, final double scale) {
+
+ super( path );
+
+ this.scale = scale;
+
+ }
+
+
+ public Double getValue() {
+
+ final Double value = (Double) vals.get(path);
+
+ // no value is defined.
+ if (value == null)
+ return 0d;
+
+ final double d = value.doubleValue() * scale;
+
+ return d;
+
+ }
+
+ }
+
+ /**
+ * Map containing the current values for the configured counters. The keys
+ * are paths into the {@link CounterSet}. The values are the data most
+ * recently read from <code>vmstat</code>.
+ */
+ final private Map<String, Object> vals = new HashMap<String, Object>();
+
+ /**
+ * The timestamp associated with the most recently collected values.
+ */
+ private long lastModified = System.currentTimeMillis();
+
+ /**
+ * The {@link Pattern} used to split apart the rows read from
+ * <code>iostat</code>.
+ */
+ final static Pattern pattern = Pattern.compile("\\s+");
+
+ final private boolean cpuStats;
+
+ /**
+ *
+ * @param interval
+ * The interval for the collected statistics.
+ * @param cpuStats
+ * <code>true</code> if the collector should report on the CPU
+ * stats (us, sy, id).
+ */
+ public IOStatCollector(final int interval, final boolean cpuStats) {
+
+ super(interval);
+
+ this.cpuStats = cpuStats;
+
+ }
+
+ public List<String> getCommand() {
+
+ final List<String> command = new LinkedList<String>();
+
+ command.add("/usr/sbin/iostat");
+
+ // report only device stats.
+ command.add("-d");
+
+ // report on ALL devices
+ command.add("-n");
+ command.add("999");
+
+ // report CPU stats (explicitly request, so in addition to device stats).
+ command.add("-C");
+
+ // Note: The configured interval in seconds between reports.
+ command.add("-w");
+ command.add("" + getInterval());
+
+ return command;
+
+ }
+
+ /**
+ * Declares the counters that we will collect
+ */
+ public CounterSet getCounters() {
+
+ final CounterSet root = new CounterSet();
+
+ inst = new LinkedList<I>();
+
+ /*
+ * Note: Counters are all declared as Double to facilitate aggregation.
+ */
+
+ /*
+ * This reports CPU (us, sy, id), bytes read/written per second, and
+ * transfers per second.
+ *
+ * Note: We could also report KB/t (kilobytes per transfer), but it can
+ * be derived from MB/s and tps.
+ */
+
+ /*
+ * Note: The counter is reported in MB/s, so we convert to bytes using a
+ * scaling factor.
+ */
+ inst.add(new DI(IHostCounters.PhysicalDisk_BytesPerSec,
+ (double) Bytes.megabyte32));
+
+ /*
+ * Note: The counter is reported in transfers/s.
+ */
+ inst.add(new DI(IHostCounters.PhysicalDisk_TransfersPerSec));
+
+ if (cpuStats) {
+
+ /*
+ * Note: iostats reports percentages in [0:100] so we convert them
+ * to [0:1] using a scaling factor.
+ */
+
+ // Note: processor time = (100-idle), converted to [0:1].
+ inst.add(new DI(IRequiredHostCounters.CPU_PercentProcessorTime,.01d));
+ // Note: column us
+ inst.add(new DI(IHostCounters.CPU_PercentUserTime, .01d));
+ // Note: column sy
+ inst.add(new DI(IHostCounters.CPU_PercentSystemTime, .01d));
+// // Note: IO Wait is NOT reported by vmstat.
+// inst.add(new DI(IHostCounters.CPU_PercentIOWait, .01d));
+
+ }
+
+ for (Iterator<I> itr = inst.iterator(); itr.hasNext();) {
+
+ final I i = itr.next();
+
+ root.addCounter(i.getPath(), i);
+
+ }
+
+ return root;
+
+ }
+ private List<I> inst = null;
+
+ public AbstractProcessReader getProcessReader() {
+
+ return new IOStatReader();
+
+ }
+
+ /**
+ * Sample output for <code>iostat -d -C -n 999 -w 60</code>, where
+ * <code>60</code> is the interval. There is no option to suppress the
+ * periodic repeat of the header. The header repeats in its entirety every
+ * "page" full.
+ *
+ * <pre>
+ * disk0 disk1 disk2 disk3 disk4 cpu
+ * KB/t tps MB/s KB/t tps MB/s KB/t tps MB/s KB/t tps MB/s KB/t tps MB/s us sy id
+ * 197.14 26 5.10 79.10 1 0.07 3.74 0 0.00 41.31 0 0.00 13.03 0 0.00 31 6 63
+ * 0.00 0 0.00 0.00 0 0.00 0.00 0 0.00 0.00 0 0.00 0.00 0 0.00 53 4 43
+ * 0.00 0 0.00 0.00 0 0.00 0.00 0 0.00 0.00 0 0.00 0.00 0 0.00 56 8 37
+ * 0.00 0 0.00 0.00 0 0.00 0.00 0 0.00 0.00 0 0.00 0.00 0 0.00 54 9 37
+ * 0.00 0 0.00 0.00 0 0.00 0.00 0 0.00 0.00 0 0.00 0.00 0 0.00 57 8 35
+ * </pre>
+ *
+ * Note that the #of device columns will vary for this format. The #of
+ * columns for the report can presumably vary as devices are connected and
+ * disconnected!
+ * <p>
+ * Note: The first data line is ignored as it will be a historical average.
+ *
+ * @author <a href="mailto:tho...@us...">Bryan
+ * Thompson</a>
+ */
+ private class IOStatReader extends ProcessReaderHelper {
+
+ protected ActiveProcess getActiveProcess() {
+
+ if (activeProcess == null)
+ throw new IllegalStateException();
+
+ return activeProcess;
+
+ }
+
+ public IOStatReader() {
+
+ super();
+
+ }
+
+ @Override
+ protected void readProcess() throws Exception {
+
+ if(log.isInfoEnabled())
+ log.info("begin");
+
+ for (int i = 0; i < 10 && !getActiveProcess().isAlive(); i++) {
+
+ if(log.isInfoEnabled())
+ log.info("waiting for the readerFuture to be set.");
+
+ Thread.sleep(100/*ms*/);
+
+ }
+
+ if(log.isInfoEnabled())
+ log.info("running");
+
+ // 1st header. this has the "disk#" metadata, which we are ignoring.
+ {
+
+ final String h0;
+ h0 = readLine();
+
+ if (log.isInfoEnabled())
+ log.info("header: " + h0);
+
+ }
+
+ /*
+ * Read 2nd header. this is also ignored, but we spot check a few
+ * things.
+ *
+ * Note: We have to recompute the #of devices for each data line as
+ * devices may be connected and disconnected at any time.
+ */
+ final int ncpuFields = cpuStats ? 3 : 0; // us sy id
+ final int nfieldsPerDevice = 3; // KB/t tps MB/s
+ String header1 = null;
+ {
+
+ final String h1;
+ header1 = h1 = readLine();
+
+ if (log.isInfoEnabled())
+ log.info("header: " + h1);
+
+ final String[] fields = pattern
+ .split(h1.trim(), 0/* limit */);
+
+ final int nfields = fields.length;
+ final int ndeviceFields = fields.length - ncpuFields;
+ final int ndevices = ndeviceFields / nfieldsPerDevice;
+
+ if (ndevices * nfieldsPerDevice != ndeviceFields) {
+ // There should be an even multiple of fields per device,
+ // otherwise the format has changed.
+ throw new RuntimeException("Unexpected #of device fields: "
+ + h1);
+ }
+
+ if (ndevices > 0) {
+ // at least one device.
+ assertField(0, fields, "KB/t");
+ assertField(1, fields, "tps");
+ assertField(2, fields, "MB/s");
+ }
+
+ if (cpuStats) {
+ assertField(nfields - 3, fields, "us");
+ assertField(nfields - 2, fields, "sy");
+ assertField(nfields - 1, fields, "id");
+ }
+
+ if (log.isInfoEnabled()) {
+ log.info("ndevices=" + ndevices);
+ }
+
+ }
+
+// * disk0 disk1 disk2 disk3 disk4 cpu
+// * KB/t tps MB/s KB/t tps MB/s KB/t tps MB/s KB/t tps MB/s KB/t tps MB/s us sy id
+// * 197.14 26 5.10 79.10 1 0.07 3.74 0 0.00 41.31 0 0.00 13.03 0 0.00 31 6 63
+// * 0.00 0 0.00 0.00 0 0.00 0.00 0 0.00 0.00 0 0.00 0.00 0 0.00 53 4 43
+
+ // read lines until interrupted.
+ boolean first = true;
+ while(true) {
+
+ // read the next line of data.
+ final String data;
+ {
+ String s = readLine();
+ if (s.contains("disk") || s.contains("cpu")) {
+ // We just read the 1st header
+ header1 = s = readLine(); // 2nd header line.
+ s = readLine(); // data line.
+ if(log.isInfoEnabled())
+ log.info("Skipped headers.");
+ }
+ data = s;
+ }
+
+ if(first) {
+ // Skip the first data line. It is a historical summary.
+ first = false;
+ continue;
+ }
+
+ try {
+
+ // timestamp
+ lastModified = System.currentTimeMillis();
+
+ final String[] fields = pattern
+ .split(data.trim(), 0/* limit */);
+
+ final int nfields = fields.length;
+ final int ndeviceFields = fields.length - ncpuFields;
+ final int ndevices = ndeviceFields / nfieldsPerDevice;
+
+ if (cpuStats) {
+
+ final String us = fields[nfields - 3];
+ final String sy = fields[nfields - 2];
+ final String id = fields[nfields - 1];
+ if (cpuStats) {
+
+ vals.put(IHostCounters.CPU_PercentUserTime,
+ Double.parseDouble(us));
+
+ vals.put(IHostCounters.CPU_PercentSystemTime,
+ Double.parseDouble(sy));
+
+ // Note: NOT reported by iostat under OSX.
+// vals.put(IHostCounters.CPU_PercentIOWait,
+// Double.parseDouble(iowait));
+
+ vals.put(
+ IRequiredHostCounters.CPU_PercentProcessorTime,
+ (100d - Double.parseDouble(id)));
+
+ }
+
+ }
+
+ // Aggregate across all devices.
+ double totalKBPerXFer = 0;
+ double totalxferPerSec = 0;
+ double totalMBPerSec = 0;
+ for (int i = 0; i < ndevices; i++) {
+
+ final int off = i * nfieldsPerDevice;
+
+ final String kbPerXfer = fields[off + 0]; // KB/t
+ final String xferPerSec = fields[off + 1]; // tps
+ final String mbPerSec = fields[off + 2]; // MB/s
+
+ final double _kbPerXFer = Double.parseDouble(kbPerXfer);
+ final double _xferPerSec = Double
+ .parseDouble(xferPerSec);
+ final double _mbPerSec = Double.parseDouble(mbPerSec);
+
+ totalKBPerXFer += _kbPerXFer;
+ totalxferPerSec += _xferPerSec;
+ totalMBPerSec += _mbPerSec;
+
+ if (log.isInfoEnabled())
+ log.info("\ntotalKBPerXfer=" + totalKBPerXFer
+ + ", totalXFerPerSec=" + totalxferPerSec
+ + ", totalMBPerSec=" + totalMBPerSec + "\n"
+ + header1 + "\n" + data);
+
+ vals.put(IHostCounters.PhysicalDisk_TransfersPerSec,
+ totalxferPerSec);
+
+ vals.put(IHostCounters.PhysicalDisk_BytesPerSec,
+ totalMBPerSec);
+
+ }
+
+ } catch (Exception ex) {
+
+ /*
+ * Issue warning for parsing problems.
+ */
+
+ log.warn(ex.getMessage() //
+ + "\nheader: " + header1 //
+ + "\n data: " + data //
+ , ex);
+
+ }
+
+ } // while(true)
+
+ } // readProcess()
+
+ } // class IOStatReader
+
+ private static void assertField(final int index, final String[] fields,
+ final String expected) {
+
+ if (!expected.equals(fields[index]))
+ throw new RuntimeException("Expecting '" + expected + "', found: '"
+ + fields[0] + "'");
+
+ }
+
+}
Added: branches/BIGDATA_RELEASE_1_0_0/bigdata/src/java/com/bigdata/counters/osx/StatisticsCollectorForOSX.java
===================================================================
--- branches/BIGDATA_RELEASE_1_0_0/bigdata/src/java/com/bigdata/counters/osx/StatisticsCollectorForOSX.java (rev 0)
+++ branches/BIGDATA_RELEASE_1_0_0/bigdata/src/java/com/bigdata/counters/osx/StatisticsCollectorForOSX.java 2011-07-15 13:33:05 UTC (rev 4920)
@@ -0,0 +1,283 @@
+package com.bigdata.counters.osx;
+
+import java.util.UUID;
+
+import com.bigdata.counters.AbstractStatisticsCollector;
+import com.bigdata.counters.CounterSet;
+import com.bigdata.counters.IHostCounters;
+import com.bigdata.counters.IRequiredHostCounters;
+import com.bigdata.counters.PIDUtil;
+
+/**
+ * Collection of host performance data using OSX native utilities (curent
+ * vm_stat and iostat).
+ * <p>
+ * Note: Unfortunately, the OSX command line reporting utilities do NOT provide
+ * access to IO Wait and do NOT break down IO into reads and writes. This means
+ * that we can not provide reporting for the following counters:
+ * <ul>
+ * <li> {@link IRequiredHostCounters#PhysicalDisk_BytesReadPerSec}</li>
+ * <li> {@link IRequiredHostCounters#PhysicalDisk_BytesWrittenPerSec}</li>
+ * <li> {@link IHostCounters#CPU_PercentIOWait}</li>
+ * </ul>
+ * However, we are collecting the following which may be substituted to some
+ * extent:
+ * <ul>
+ * <li> {@link IHostCounters#PhysicalDisk_BytesPerSec}</li>
+ * <li> {@link IHostCounters#PhysicalDisk_TransfersPerSec}</li>
+ * </ul>
+ * However, there is no substitute for IO Wait.
+ *
+ * @see https://sourceforge.net/apps/trac/bigdata/ticket/225
+ *
+ * @author <a href="mailto:tho...@us...">Bryan Thompson</a>
+ * @version $Id: StatisticsCollectorForLinux.java 4290 2011-03-11 15:12:26Z
+ * thompsonbry $
+ */
+public class StatisticsCollectorForOSX extends AbstractStatisticsCollector {
+
+ /**
+ * The process identifier for this process (the JVM).
+ */
+ static protected int pid;
+ static {
+
+ pid = PIDUtil.getLinuxPIDWithBash();
+
+ }
+
+// /**
+// * The Linux {@link KernelVersion}.
+// */
+// static protected KernelVersion kernelVersion;
+// static {
+//
+// kernelVersion = KernelVersion.get();
+//
+// }
+
+ /**
+ * The name of the process (or more typically its service {@link UUID})
+ * whose per-process performance counters are to be collected.
+ */
+ private final String processName;
+
+ /**
+ * Reports on the host disk and CPU utilization (but not IOWait and does not
+ * break out disk read versus write activity).
+ */
+ private IOStatCollector iostat;
+
+ /**
+ * <code>vm_stat</code> reports on the host page faults, buffers, etc., but
+ * it does NOT report on either CPU stats or IO Wait.
+ */
+ private VMStatCollector vmstat;
+
+ /**
+ * TODO Under OSX, <code>top</code> provides more full featured reporting of
+ * OS performance counters, including some IO counters and process specific
+ * counters. Unfortunately, it does NOT provide IO Wait and it does NOT
+ * break out IO by reads and writes. It is also more work to parse its
+ * output.
+ * <p>
+ * <code>-l 0</code> turns on "logging" mode. The ZERO (0) indicates
+ * infinite samples, rather than some finite number of samples.
+ * <p>
+ * <code>-s [delay]</code> sets the delay between reports.
+ * <p>
+ * <code>-S</code> adds reporting on swap space into the headers.
+ *
+ * <pre>
+ * Swap: 545M + 479M free.
+ * Purgeable: 31M 21336(0) pages purged.
+ * </pre>
+ *
+ * Here is an example where the pid is NOT specified.
+ *
+ * <pre>
+ * top -l 0 -s 60 -S
+ * Processes: 107 total, 3 running, 104 sleeping, 587 threads
+ * 2011/07/13 20:17:36
+ * Load Avg: 2.24, 1.77, 1.53
+ * CPU usage: 63.15% user, 36.84% sys, 0.0% idle
+ * SharedLibs: 5292K resident, 6092K data, 0B linkedit.
+ * MemRegions: 18245 total, 4813M resident, 30M private, 429M shared.
+ * PhysMem: 1027M wired, 5472M active, 671M inactive, 7171M used, 1019M free.
+ * VM: 263G vsize, 1042M framework vsize, 2911706(0) pageins, 2092534(0) pageouts.
+ * Swap: 545M + 479M free.
+ * Purgeable: 31M 21336(0) pages purged.
+ * Networks: packets: 20768691/24G in, 22255115/23G out.
+ * Disks: 3335765/133G read, 8511536/2051G written.
+ *
+ * PID COMMAND %CPU TIME #TH #WQ #PORTS #MREGS RPRVT RSHRD RSIZE VPRVT VSIZE PGRP PPID STATE UID FAULTS COW MSGSENT MSGRECV SYSBSD SYSMACH CSW PAGEINS USER
+ * 94188 mdworker 0.0 00:06.04 3 1 48+ 60+ 836K+ 16M+ 2488K+ 31M+ 2408M+ 94188 1 sleeping 89 32402+ 147+ 115066+ 37438+ 263505+ 79593+ 55853+ 42+ _spotlight
+ * 79486 Image Capture Ex 0.0 00:01.19 3 1 100+ 73+ 3052K+ 17M+ 7004K+ 40M+ 2672M+ 79486 865 sleeping 501 11662+ 174+ 126963+ 63971+ 26666+ 71795+ 16995+ 55+ bryan
+ * 79438 Preview 0.0 00:31.59 3 1 207+ 453+ 30M+ 67M+ 65M+ 77M+ 2851M+ 79438 865 sleeping 501 96609+ 1466+ 330566+ 162122+ 709661+ 553728+ 129001+ 801+ bryan
+ * 79114 TextEdit 0.0 00:04.80 3 2 121+ 193+ 6616K+ 28M+ 18M+ 44M+ 2751M+ 79114 865 sleeping 501 24588+ 870+ 211100+ 104015+ 52253+ 142782+ 43400+ 238+ bryan
+ * 66520- Skype 0.0 89:18.16 21 2 538+ 658+ 161M+ 78M+ 208M+ 270M+ 1266M+ 66520 865 sleeping 501 115403971+ 2498+ 12493857+ 6119421+ 10995806+ 28285072+ 17521056+ 3683+ bryan
+ * </pre>
+ *
+ * Top can be used to go after per-process statistics by specifying
+ * <code>-pid [pid]</code>. It is also possible to request all processes
+ * owned by a user with <code>-user [user]</code>. Here is an example where
+ * the pid is specified.
+ *
+ * <pre>
+ * top -l 0 -s 60 -S -pid 26866
+ * Processes: 107 total, 3 running, 104 sleeping, 649 threads
+ * 2011/07/13 20:19:29
+ * Load Avg: 1.14, 1.49, 1.45
+ * CPU usage: 56.25% user, 43.75% sys, 0.0% idle
+ * SharedLibs: 5292K resident, 6060K data, 0B linkedit.
+ * MemRegions: 18439 total, 4806M resident, 30M private, 404M shared.
+ * PhysMem: 969M wired, 4768M active, 1366M inactive, 7102M used, 1089M free.
+ * VM: 263G vsize, 1042M framework vsize, 2911717(0) pageins, 2092534(0) pageouts.
+ * Swap: 545M + 479M free.
+ * Purgeable: 380K 22126(0) pages purged.
+ * Networks: packets: 20876610/24G in, 22362893/23G out.
+ * Disks: 3336157/133G read, 8512107/2051G written.
+ *
+ * PID COMMAND %CPU TIME #TH #WQ #PORTS #MREGS RPRVT RSHRD RSIZE VPRVT VSIZE PGRP PPID STATE UID FAULTS COW MSGSENT MSGRECV SYSBSD SYSMACH CSW PAGEINS USER
+ * 26866 java 0.0 20:29.90 137/1 1 5890+ 703+ 2823M+ 7944K+ 2863M+ 3281M+ 7450M+ 21051 21052 running 501 1975814+ 2352+ 9530469+ 138939+ 5625664+ 25326440+ 10828748+ 8030+ bryan
+ * </pre>
+ *
+ */
+// private TOPCollector pidstat;
+
+ public void start() {
+
+ if (log.isInfoEnabled())
+ log.info("starting collectors");
+
+ super.start();
+
+ if (iostat != null)
+ try {
+ iostat.start();
+ } catch (Throwable t) {
+ log.error(t, t);
+ }
+
+ if (vmstat != null)
+ try {
+ vmstat.start();
+ } catch (Throwable t) {
+ log.error(t, t);
+ }
+
+// if (pidstat != null)
+// try {
+// pidstat.start();
+// } catch (Throwable t) {
+// log.error(t, t);
+// }
+
+ }
+
+ public void stop() {
+
+ if (log.isInfoEnabled())
+ log.info("stopping collectors");
+
+ super.stop();
+
+ if (iostat != null)
+ try {
+ iostat.stop();
+ } catch (Throwable t) {
+ log.error(t, t);
+ }
+
+ if (vmstat != null)
+ try {
+ vmstat.stop();
+ } catch (Throwable t) {
+ log.error(t, t);
+ }
+
+// if (pidstat != null)
+// try {
+// pidstat.stop();
+// } catch (Throwable t) {
+// log.error(t, t);
+// }
+
+ }
+
+ @Override
+ public CounterSet getCounters() {
+
+ final CounterSet root = super.getCounters();
+
+ if (iostat != null) {
+
+ /*
+ * These are per-host counters. We attach them under the fully
+ * qualified hostname.
+ */
+
+ root.makePath(fullyQualifiedHostName).attach(iostat.getCounters());
+
+ }
+
+ if (vmstat != null) {
+
+ /*
+ * These are per-host counters. We attach them under the fully
+ * qualified hostname.
+ */
+
+ root.makePath(fullyQualifiedHostName).attach(vmstat.getCounters());
+
+ }
+
+// if (pidstat != null) {
+//
+// /*
+// * These are per-process counters. We attach them under a path
+// * described by the fully qualified hostname followed by the process
+// * name.
+// */
+//
+// root.makePath(fullyQualifiedHostName + ps + processName)
+// .attach(pidstat.getCounters());
+//
+// }
+
+ return root;
+
+ }
+
+ /**
+ *
+ * @param interval
+ * The interval at which the performance counters will be
+ * collected in seconds.
+ * @param processName
+ * The name of the process (or more typically its service
+ * {@link UUID}) whose per-process performance counters are to
+ * be collected.
+ */
+ public StatisticsCollectorForOSX(final int interval,
+ final String processName) {
+
+ super(interval);
+
+ if (processName == null)
+ throw new IllegalArgumentException();
+
+ this.processName = processName;
+
+ // host wide collection
+ iostat = new IOStatCollector(interval, true/* cpuStats */);
+
+ // Enable supported collectors.
+ vmstat = new VMStatCollector(interval);
+
+// // process specific collection.
+// pidstat = new PIDStatCollector(pid, interval, kernelVersion);
+
+ }
+
+}
Added: branches/BIGDATA_RELEASE_1_0_0/bigdata/src/java/com/bigdata/counters/osx/VMStatCollector.java
===================================================================
--- branches/BIGDATA_RELEASE_1_0_0/bigdata/src/java/com/bigdata/counters/osx/VMStatCollector.java (rev 0)
+++ branches/BIGDATA_RELEASE_1_0_0/bigdata/src/java/com/bigdata/counters/osx/VMStatCollector.java 2011-07-15 13:33:05 UTC (rev 4920)
@@ -0,0 +1,474 @@
+/*
+
+Copyright (C) SYSTAP, LLC 2006-2008. All rights reserved.
+
+Contact:
+ SYSTAP, LLC
+ 4501 Tower Road
+ Greensboro, NC 27410
+ lic...@bi...
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+*/
+/*
+ * Created on Dec 9, 2008
+ */
+
+package com.bigdata.counters.osx;
+
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.regex.Pattern;
+
+import com.bigdata.counters.AbstractProcessCollector;
+import com.bigdata.counters.AbstractProcessReader;
+import com.bigdata.counters.ActiveProcess;
+import com.bigdata.counters.CounterSet;
+import com.bigdata.counters.ICounterHierarchy;
+import com.bigdata.counters.ICounterSet;
+import com.bigdata.counters.IHostCounters;
+import com.bigdata.counters.IInstrument;
+import com.bigdata.counters.IRequiredHostCounters;
+import com.bigdata.counters.ProcessReaderHelper;
+
+/**
+ * Collects some counters using <code>vmstat</code>.
+ *
+ * @author <a href="mailto:tho...@us...">Bryan Thompson</a>
+ * @version $Id: VMStatCollector.java 4289 2011-03-10 21:22:30Z thompsonbry $
+ */
+public class VMStatCollector extends AbstractProcessCollector implements
+ ICounterHierarchy, IRequiredHostCounters, IHostCounters{
+
+ /**
+ * Inner class integrating the current values with the {@link ICounterSet}
+ * hierarchy.
+ *
+ * @author <a href="mailto:tho...@us...">Bryan
+ * Thompson</a>
+ */
+ abstract class I<T> implements IInstrument<T> {
+
+ protected final String path;
+
+ public String getPath() {
+
+ return path;
+
+ }
+
+ public I(String path) {
+
+ assert path != null;
+
+ this.path = path;
+
+ }
+
+ public long lastModified() {
+
+ return lastModified;
+
+ }
+
+ /**
+ * @throws UnsupportedOperationException
+ * always.
+ */
+ public void setValue(T value, long timestamp) {
+
+ throw new UnsupportedOperationException();
+
+ }
+
+ }
+
+ /**
+ * Double precision counter with scaling factor.
+ *
+ * @author <a href="mailto:tho...@us...">Bryan Thompson</a>
+ */
+ class DI extends I<Double> {
+
+ protected final double scale;
+
+ DI(final String path) {
+
+ this(path,1d);
+
+ }
+
+ DI(final String path, final double scale) {
+
+ super( path );
+
+ this.scale = scale;
+
+ }
+
+
+ public Double getValue() {
+
+ final Double value = (Double) vals.get(path);
+
+ // no value is defined.
+ if (value == null)
+ return 0d;
+
+ final double d = value.doubleValue() * scale;
+
+ return d;
+
+ }
+
+ }
+
+ /**
+ * Map containing the current values for the configured counters. The keys
+ * are paths into the {@link CounterSet}. The values are the data most
+ * recently read from <code>vmstat</code>.
+ */
+ final private Map<String, Object> vals = new HashMap<String, Object>();
+
+ /**
+ * The timestamp associated with the most recently collected values.
+ */
+ private long lastModified = System.currentTimeMillis();
+
+ /**
+ * The {@link Pattern} used to split apart the rows read from
+ * <code>vmstat</code>.
+ */
+ final static Pattern pattern = Pattern.compile("\\s+");
+
+ /**
+ *
+ * @param interval
+ * The interval for the collected statistics.
+ */
+ public VMStatCollector(final int interval) {
+
+ super(interval);
+
+ }
+
+ public List<String> getCommand() {
+
+ final List<String> command = new LinkedList<String>();
+
+ command.add("/usr/bin/vm_stat");
+
+ // Note: The configured interval in seconds between reports.
+ command.add("" + getInterval());
+
+ return command;
+
+ }
+
+ /**
+ * Declares the counters that we will collect
+ */
+ public CounterSet getCounters() {
+
+ final CounterSet root = new CounterSet();
+
+ inst = new LinkedList<I>();
+
+ /*
+ * Note: Counters are all declared as Double to facilitate aggregation.
+ */
+
+ /*
+ * Note: [pageins] is "the #of blocks swapped in".
+ */
+ inst.add(new DI(IRequiredHostCounters.Memory_majorFaultsPerSecond));
+
+ /*
+ * This is an aggregation of the counters for "active", "wire", and
+ * "spec".
+ */
+ inst.add(new DI(IHostCounters.Memory_SwapBytesUsed));
+
+ /*
+ * Note: [free] is "the total number of free pages in the system".
+ */
+ inst.add(new DI(IHostCounters.Memory_Bytes_Free));
+
+ for (Iterator<I> itr = inst.iterator(); itr.hasNext();) {
+
+ final I i = itr.next();
+
+ root.addCounter(i.getPath(), i);
+
+ }
+
+ return root;
+
+ }
+ private List<I> inst = null;
+
+ public AbstractProcessReader getProcessReader() {
+
+ return new VMStatReader();
+
+ }
+
+ /**
+ * Sample output for <code>vm_stat 60</code>, where <code>60</code> is the
+ * interval. Unlike the linux <code>vmstat</code>, there is no option to
+ * suppress the periodic repeat of the header. The header repeats in its
+ * entirety every "page" full.
+ *
+ * <pre>
+ * Mach Virtual Memory Statistics: (page size of 4096 bytes, cache hits 0%)
+ * free active spec inactive wire faults copy 0fill reactive pageins pageout
+ * 346061 1246K 51002 196922 190727 1004215K 19835511 525442K 7555575 2897558 2092534
+ * 346423 1247K 50754 196922 189767 2707 0 1263 0 0 0
+ * 343123 1247K 52180 198906 189767 3885 0 1754 0 0 0
+ * 342474 1247K 52101 198978 190116 21739 2013 4651 0 0 0
+ * 342542 1247K 52016 199319 189799 5792 17 2561 0 0 0
+ * </pre>
+ *
+ * @author <a href="mailto:tho...@us...">Bryan
+ * Thompson</a>
+ */
+ private class VMStatReader extends ProcessReaderHelper {
+
+ protected ActiveProcess getActiveProcess() {
+
+ if (activeProcess == null)
+ throw new IllegalStateException();
+
+ return activeProcess;
+
+ }
+
+ public VMStatReader() {
+
+ super();
+
+ }
+
+ /**
+ *
+ * @see TestParse_vm_stat#test_vmstat_header_and_data_parse()
+ */
+ @Override
+ protected void readProcess() throws Exception {
+
+ if(log.isInfoEnabled())
+ log.info("begin");
+
+ for (int i = 0; i < 10 && !getActiveProcess().isAlive(); i++) {
+
+ if(log.isInfoEnabled())
+ log.info("waiting for the readerFuture to be set.");
+
+ Thread.sleep(100/*ms*/);
+
+ }
+
+ if(log.isInfoEnabled())
+ log.info("running");
+
+ // 1st header: we need the pageSize.
+ final String h0;
+ final int pageSize;
+ {
+
+ h0 = readLine();
+
+ final String[] fields = pattern.split(h0.trim(), 0/* limit */);
+
+ if (log.isInfoEnabled())
+ log.info("header: " + h0);
+
+ assertField(0, fields, "Mach");
+
+ pageSize = Integer.valueOf(fields[7]);
+
+ if (pageSize <= 0 || (pageSize % 512 != 0))
+ throw new RuntimeException("pageSize=" + pageSize);
+
+ if (log.isInfoEnabled())
+ log.info("pageSize: " + pageSize);
+
+ }
+
+ // read 2nd header and verify expected fields.
+ final String h1;
+ {
+
+ h1 = readLine();
+
+ if (log.isInfoEnabled())
+ log.info("header: " + h1);
+
+ final String[] fields = pattern
+ .split(h1.trim(), 0/* limit */);
+
+ assertField(0, fields, "free");
+ assertField(9, fields, "pageins");
+
+ }
+
+ /*
+ * Note: Some fields are reported with a 'K' suffix. Some are in
+ * "pages". The pageSize was extracted from the header above.
+ */
+ // Mach Virtual Memory Statistics: (page size of 4096 bytes, cache hits 0%)
+ // free active spec inactive wire faults copy 0fill reactive pageins pageout
+ //398649 1196K 47388 203418 185941 145234K 2036323 82012367 1353888 351301 149940
+ //400080 1197K 44784 205254 183886 1829 0 1046 0 0 0
+
+ // read lines until interrupted.
+ long pageout_tm1 = 0;
+ boolean first = true;
+ while(true) {
+
+ // read the next line of data.
+ final String data;
+ {
+ String s = readLine();
+ if(s.startsWith("Mach")) { // 1st header line.
+ s = readLine(); // 2nd header line.
+ s = readLine(); // data line.
+ if(log.isInfoEnabled())
+ log.info("Skipped headers.");
+ }
+ data = s;
+ }
+
+ try {
+
+ // timestamp
+ lastModified = System.currentTimeMillis();
+
+ final String[] fields = pattern.split(data.trim(), 0/* limit */);
+
+ final String free = fields[0]; // free
+ final String active = fields[1]; // in use and pageable
+ final String spec = fields[2]; // speculative
+// final String inactive = fields[3]; //
+ final String wire = fields[4]; // wired down
+// final String faults = fields[5]; // translation faults
+
+ final String pageins = fields[9]; // pageins
+ final String pageout = fields[10]; // pageout
+
+ if (log.isInfoEnabled())
+ log.info("\nfree=" + free + ", active=" + active
+ + ", spec=" + spec + ", wire=" + wire + ", si="
+ + pageins + ", so=" + pageout + "\n" + h1
+ + "\n" + data);
+
+ {
+
+ final double _free = parseDouble(free);
+ final double _active = parseDouble(active);
+ final double _spec = parseDouble(spec);
+ final double _wire = parseDouble(wire);
+
+ final double swapBytesUsed = _active + _spec + _wire;
+
+ vals.put(IHostCounters.Memory_Bytes_Free, _free
+ * pageSize);
+
+ vals.put(IHostCounters.Memory_SwapBytesUsed,
+ swapBytesUsed * pageSize);
+
+ }
+
+ /*
+ * pageout is reported as a total over time. we have to
+ * compute a delta, then divide through by the interval to
+ * get pages/second.
+ */
+ {
+ final double _pageout = parseDouble(pageout);
+
+ if (!first) {
+
+ final double delta = _pageout - pageout_tm1;
+
+ final double majorPageFaultsPerSec = delta
+ / getInterval();
+
+ vals.put(
+ IRequiredHostCounters.Memory_majorFaultsPerSecond,
+ majorPageFaultsPerSec);
+
+ }
+ }
+ first = false;
+
+ } catch (Exception ex) {
+
+ /*
+ * Issue warning for parsing problems.
+ */
+
+ log.warn(ex.getMessage() //
+ + "\nheader: " + h1 //
+ + "\n data: " + data //
+ , ex);
+
+ }
+
+ } // while(true)
+
+ } // readProcess()
+
+ } // class VMStatReader
+
+ /**
+ * Parse a string which may have a "K" suffix, returning a double. If the
+ * "K" suffix is present, then the returned value is scaled by 1000. This
+ * handles an OSX specific oddity for <code>vm_stat</code>
+ *
+ * @param s
+ * The string.
+ *
+ * @return The (possibly scaled) value.
+ */
+ private static double parseDouble(final String s) {
+
+ final int pos = s.indexOf("K");
+
+ if(pos == -1) {
+
+ return Long.valueOf(s);
+
+ }
+
+ final long val = Long.valueOf(s.substring(0, pos));
+
+ return val * 1000;
+
+ }
+
+ private static void assertField(final int index, final String[] fields,
+ final String expected) {
+
+ if (!expected.equals(fields[index]))
+ throw new RuntimeException("Expecting '" + expected + "', found: '"
+ + fields[0] + "'");
+
+ }
+
+}
Added: branches/BIGDATA_RELEASE_1_0_0/bigdata/src/java/com/bigdata/counters/osx/package.html
===================================================================
--- branches/BIGDATA_RELEASE_1_0_0/bigdata/src/java/com/bigdata/counters/osx/package.html (rev 0)
+++ branches/BIGDATA_RELEASE_1_0_0/bigdata/src/java/com/bigdata/counters/osx/package.html 2011-07-15 13:33:05 UTC (rev 4920)
@@ -0,0 +1,14 @@
+<html>
+<head>
+<title>Performance counter collection on OSX.</title>
+</head>
+<body>
+
+<p>
+
+Package provides performance counter collection on OSX.
+
+</p>
+
+</body>
+</html>
Added: branches/BIGDATA_RELEASE_1_0_0/bigdata/src/java/org/apache/system/MacOSX.java
===================================================================
--- branches/BIGDATA_RELEASE_1_0_0/bigdata/src/java/org/apache/system/MacOSX.java (rev 0)
+++ branches/BIGDATA_RELEASE_1_0_0/bigdata/src/java/org/apache/system/MacOSX.java 2011-07-15 13:33:05 UTC (rev 4920)
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied.
+ *
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.system;
+
+import java.lang.management.ManagementFactory;
+import java.lang.management.OperatingSystemMXBean;
+
+/**
+ * Generic version based on {@link OperatingSystemMXBean}.
+ *
+ * @author <a href="mailto:de...@av...">Avalon Development Team</a>
+ * @version $Id: Linux.java 2265 2009-10-26 12:51:06Z thompsonbry $
+ */
+public final class MacOSX implements CPUParser
+{
+ private fin...
[truncated message content] |
