[Assorted-commits] SF.net SVN: assorted:[1137] ydb/trunk

[<yan...@us...>]

Revision: 1137
          http://assorted.svn.sourceforge.net/assorted/?rev=1137&view=rev
Author:   yangzhang
Date:     2009-01-23 21:44:55 +0000 (Fri, 23 Jan 2009)

Log Message:
-----------
- added further breakdown of the xfer time
- upgraded readmsg
- mean -> median in analysis plot

Modified Paths:
--------------
    ydb/trunk/src/Makefile
    ydb/trunk/src/main.lzz.clamp
    ydb/trunk/tools/analysis.py
    ydb/trunk/tools/test.bash

Modified: ydb/trunk/src/Makefile
===================================================================
--- ydb/trunk/src/Makefile	2009-01-23 09:34:16 UTC (rev 1136)
+++ ydb/trunk/src/Makefile	2009-01-23 21:44:55 UTC (rev 1137)
@@ -27,6 +27,7 @@
 endif
 LDFLAGS  := -pthread -lstx -lst -lresolv -lprotobuf -lgtest \
             -lboost_program_options-gcc43-mt -lboost_thread-gcc43-mt $(GPROF)
+# The -Wno- warnings are for boost.
 CXXFLAGS := -g3 -pthread $(GPROF) -Wall -Werror -Wextra -Woverloaded-virtual \
             -Wconversion -Wno-conversion -Wno-ignored-qualifiers \
             -Wpointer-arith -Wcast-qual -Wcast-align -Wwrite-strings \

Modified: ydb/trunk/src/main.lzz.clamp
===================================================================
--- ydb/trunk/src/main.lzz.clamp	2009-01-23 09:34:16 UTC (rev 1136)
+++ ydb/trunk/src/main.lzz.clamp	2009-01-23 21:44:55 UTC (rev 1137)
@@ -282,17 +282,31 @@
 }
 
 /**
- * Read a message.
+ * Read a message.  This is done in two steps: first by reading the length
+ * prefix, then by reading the actual body.
+ *
+ * \param[in] src The socket from which to read.
+ *
+ * \param[in] msg The protobuf to read into.
+ *
+ * \param[in] timed Whether to make a note of the time at which the first piece of the
+ * message (the length) was received.  Such measurement only makes sense for
+ * large messages which take a long time to receive.
+ *
+ * \param[in] timeout on each of the two read operations (first one is on
+ * length, second one is on the rest).
  */
 template <typename T>
-void
-readmsg(st_netfd_t src, T & msg, st_utime_t timeout = ST_UTIME_NO_TIMEOUT)
+long long
+readmsg(st_netfd_t src, T & msg, bool timed = false, st_utime_t timeout =
+        ST_UTIME_NO_TIMEOUT)
 {
   // Read the message length.
   uint32_t len;
   checkeqnneg(st_read_fully(src, static_cast<void*>(&len), sizeof len,
                             timeout),
               static_cast<ssize_t>(sizeof len));
+  long long start_receive = timed ? current_time_millis() : -1;
   len = ntohl(len);
 
 #define GETMSG(buf) \
@@ -308,6 +322,8 @@
     scoped_array<char> buf(new char[len]);
     GETMSG(buf.get());
   }
+
+  return start_receive;
 }
 
 /**
@@ -320,7 +336,7 @@
 readmsg(st_netfd_t src, st_utime_t timeout = ST_UTIME_NO_TIMEOUT)
 {
   T msg;
-  readmsg(src, msg, timeout);
+  readmsg(src, msg, false, timeout);
   return msg;
 }
 
@@ -943,13 +959,15 @@
         recovery_builders.push_back(my_spawn(lambda() {
           // Read the recovery message.
           Recovery recovery;
+          long long receive_start = -1;
           {
             st_intr intr(stop_hub);
-            readmsg(__ref(replicas)[__ctx(i)], recovery);
+            receive_start = readmsg(__ref(replicas)[__ctx(i)], recovery, true);
           }
           long long build_start = current_time_millis();
           cout << "got recovery message in "
-               << build_start - __ref(before_recv) << " ms" << endl;
+               << build_start - __ref(before_recv) << " ms (xfer took "
+               << build_start - receive_start << " ms)" << endl;
           for (int i = 0; i < recovery.pair_size(); i++) {
             const Recovery_Pair &p = recovery.pair(i);
             __ref(map)[p.key()] = p.value();

Modified: ydb/trunk/tools/analysis.py
===================================================================
--- ydb/trunk/tools/analysis.py	2009-01-23 09:34:16 UTC (rev 1136)
+++ ydb/trunk/tools/analysis.py	2009-01-23 21:44:55 UTC (rev 1137)
@@ -18,7 +18,7 @@
       yield list(tups)
   a = array(list(gen()))
   indexes = a[:,0,0]
-  means   = a.mean(1)
+  means   = median(a,1) #a.mean(1)
   stds    = a.std(1)
   tup = (indexes,)
   for i in range(1, len(a[0,0])):
@@ -62,25 +62,27 @@
     check(path)
     def getpairs():
       with file(path) as f:
-        seqno = recv = buildup = catchup = total = None
+        seqno = dump = recv = buildup = catchup = total = None
         for line in f:
           m = re.match( r'=== seqno=(?P<seqno>\d+) ', line )
           if m: seqno = int(m.group('seqno'))
-          m = re.search( r'got recovery message in (?P<time>\d+) ms', line )
-          if m: recv = float(m.group('time'))
+          m = re.search( r'got recovery message in (?P<dump>\d+) ms \(xfer took (?P<recv>\d+) ms\)', line )
+          if m: dump, recv = float(m.group('dump')), float(m.group('recv'))
           m = re.search( r'built up .* (?P<time>\d+) ms', line )
           if m: buildup = float(m.group('time'))
-          m = re.search( r'replayer caught up; from backlog replayed \d+ txns in (?P<time>\d+) ms', line )
+          m = re.search( r'replayer caught up; from backlog replayed \d+ txns .* in (?P<time>\d+) ms', line )
           if m: catchup = float(m.group('time'))
           m = re.match( r'.*: recovering node caught up; took (?P<time>\d+) ?ms', line )
           if m: total = float(m.group('time'))
-          tup = (seqno, recv, buildup, catchup, total)
+          tup = (seqno, dump, recv, buildup, catchup, total)
           if all(tup):
             yield tup
-            seqno = recv = buildup = catchup = total = None
-    seqnos, recvmeans, recvsds, buildmeans, buildsds, catchmeans, catchsds, totalmeans, totalsds, stacked, a = agg(getpairs())
+            seqno = dump = recv = buildup = catchup = total = None
+    seqnos, dumpmeans, dumpsds, recvmeans, recvsds, buildmeans, buildsds, \
+        catchmeans, catchsds, totalmeans, totalsds, stacked, a = \
+        agg(getpairs())
 
-    print 'max seqno, recv mean, recv sd, build mean, build sd, catch mean, catch sd, total mean, total sd'
+    print 'max seqno, dump mean, dump sd, recv mean, recv sd, build mean, build sd, catch mean, catch sd, total mean, total sd'
     print stacked
     print
 
@@ -93,13 +95,16 @@
                                (226,240,214),
                                (246,255,224)][i+1]))
     ehue = lambda i: hue(-1) # tuple(map(lambda x: min(1, x + .3), hue(i)))
-    a = bar(seqnos, recvmeans,  yerr = recvsds,  width = width, color = hue(0),
-            ecolor = ehue(0), label = 'State receive')
-    b = bar(seqnos, buildmeans, yerr = buildsds, width = width, color = hue(1),
-            ecolor = ehue(1), label = 'Build-up time', bottom = recvmeans)
-    c = bar(seqnos, catchmeans, yerr = catchsds, width = width, color = hue(2),
-            ecolor = ehue(2), label = 'Catch-up',
-            bottom = recvmeans + buildmeans)
+    bar(seqnos, dumpmeans,  yerr = dumpsds,  width = width, color = hue(0),
+        ecolor = ehue(0), label = 'State serialization')
+    bar(seqnos, recvmeans,  yerr = recvsds,  width = width, color = hue(0),
+        ecolor = ehue(0), label = 'State receive', bottom = dumpmeans)
+    bar(seqnos, buildmeans, yerr = buildsds, width = width, color = hue(1),
+        ecolor = ehue(1), label = 'Build-up',
+        bottom = dumpmeans + recvmeans)
+    bar(seqnos, catchmeans, yerr = catchsds, width = width, color = hue(2),
+        ecolor = ehue(2), label = 'Catch-up',
+        bottom = dumpmeans + recvmeans + buildmeans)
 
   title('Recovery time over number of transactions')
   xlabel('Transaction count (corresponds roughly to data size)')

Modified: ydb/trunk/tools/test.bash
===================================================================
--- ydb/trunk/tools/test.bash	2009-01-23 09:34:16 UTC (rev 1136)
+++ ydb/trunk/tools/test.bash	2009-01-23 21:44:55 UTC (rev 1137)
@@ -369,6 +369,10 @@
   hostargs kill-helper
 }
 
+times() {
+  parssh date +%s.%N
+}
+
 # Use mssh to log in with password as root to each machine.
 mssh-root() {
   : "${hosts:="$(hosts)"}"


This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.