assorted-commits

[Assorted-commits] SF.net SVN: assorted: [377] hash-join/trunk/tools

[<yan...@us...>]

Revision: 377
          http://assorted.svn.sourceforge.net/assorted/?rev=377&view=rev
Author:   yangzhang
Date:     2008-02-11 09:03:34 -0800 (Mon, 11 Feb 2008)

Log Message:
-----------
moved to simpler, more generic log processor

Modified Paths:
--------------
    hash-join/trunk/tools/LogProc.scala
    hash-join/trunk/tools/Makefile

Modified: hash-join/trunk/tools/LogProc.scala
===================================================================
--- hash-join/trunk/tools/LogProc.scala	2008-02-11 17:03:04 UTC (rev 376)
+++ hash-join/trunk/tools/LogProc.scala	2008-02-11 17:03:34 UTC (rev 377)
@@ -3,77 +3,65 @@
 import commons.Io._
 import commons.Debug._
 import commons.Path._
-// import commons.Plotting._
 import scala.collection.mutable._
+import scala.util._
 
 object LogProc {
   type FieldMap = Map[String,Double]
   type MutFieldMap = HashMap[String,Double]
-  case class Stats(
-    ncpus: Int,
-    values: FieldMap
-  )
-  val descriptors = Array(
-    ("movieLoading",         "loading movies"               ),
-    ("actressLoading",       "loading actresses"            ),
-    ("moviePartitioning",    "hash-partitioning movies"     ),
-    ("actressPartitioning",  "hash-partitioning actresses"  ),
-    ("movieBuilding",        "building with movies"         ),
-    ("actressProbing",       "probing with actresses"       ),
-    ("sum",                  "sum"                          )
-  )
-  val fieldNameToLabel = Map(descriptors: _*)
-  def fieldName(k: Int) = descriptors(k)._1
+  type MutStatMap = HashMap[Int,ArrayBuffer[FieldMap]]
   def main(args: Array[String]) {
-    val lines = using (TextReader(args(0))) (_.readLines.toArray)
+    val indexer = args(0)
+    val lines = using (TextReader(Console.in)) (_.readLines.toArray)
     val map = new MutFieldMap
-    var ncpus = 0
-    val stats = new ArrayBuffer[Stats]
-    var fieldIndex = Iterator from 0
+    var index: Option[Int] = None
+    val stats = new MutStatMap {
+      override def default(k: Int) = {
+        val buf = new ArrayBuffer[FieldMap]
+        this(k) = buf
+        buf
+      }
+    }
+    val names = new HashSet[String]
 
     // Parse logs into Stats.
     for (line <- lines) {
-      if (line contains " cpus") {
-        // Include sum.
-        if (ncpus != 0) {
-          map("sum") = sum(map.values) - map("actressLoading") - map("movieLoading")
-          stats += Stats(ncpus, map.clone)
+      if (line contains indexer) {
+        if (index != None) {
+          stats(index.get) += map.clone
         }
-        ncpus = line.split(" ")(1).toInt
-        fieldIndex = Iterator from 0
+        index = Some(line.split(" ")(1).toInt)
         map.clear
-      } else if (line contains "main time: ") {
-        map(fieldName(fieldIndex.next)) = line.split(" ").last.toDouble / 1000.0
+      } else {
+        val Seq(name, value) = line split ": "
+        names += name
+        map(name) = value.toDouble / 1000.0
       }
     }
 
-    // Build actual plot data.
-    val plotData = new HashMap[String,ArrayBuffer[Double]] {
-      override def default(k: String) = {
-        val buf = new ArrayBuffer[Double]
-        this(k) = buf
-        buf
+    // Build plot data.
+    val plotData = for (name <- names) yield {
+      val points = for ((index, buf) <- stats) yield {
+        val values = for (map <- buf) yield map(name)
+        (index, mean(values))
       }
+      val array = points.toStream.toArray
+      Sorting quickSort array
+      (name, array)
     }
-    val ncpuList = stats map (_.ncpus)
-    for (Stats(ncpus, map) <- stats) {
-      for (field <- map.keys) {
-        plotData(field) += map(field)
-      }
-    }
 
-    // Produce the time and speedup .dats.
-    for ((field,times) <- plotData) {
-      val baseline = times(0).toDouble
-      println(field + ": " + times)
-      using (TextWriter("data" / camelToHyphen(field) + "-time.dat")) { w =>
-        for ((time,ncpus) <- times zip ncpuList) {
-          w.println(ncpus + " " + time)
+    // Write plot data.
+    for ((name, points) <- plotData) {
+      println(name + ": " + points.mkString(", "))
+      using (TextWriter("data" / spacedToHyphen(name) + "-time.dat")) { w =>
+        for ((index, value) <- points) {
+          w println (index + " " + value)
         }
       }
-      using (TextWriter("data" / camelToHyphen(field) + "-speedup.dat")) { w =>
-        for ((time,ncpus) <- times map (baseline / _) zip ncpuList) {
-          w.println(ncpus + " " + time)
+      val baseline = points(0)._2
+      using (TextWriter("data" / spacedToHyphen(name) + "-speedup.dat")) { w =>
+        for ((index, value) <- points) {
+          w println (index + " " + baseline / value)
         }
       }
     }
@@ -81,8 +69,8 @@
     // Instruct gnuplot.
     def f(s:String) = {
       {
-        for ((field,_) <- plotData) yield (
-          "'data/" + camelToHyphen(field) + s + ".dat" + "' with linespoints title '" + fieldNameToLabel(field) + "'"
+        for (name <- names) yield (
+          "'data/" + spacedToHyphen(name) + s + ".dat' with linespoints title '" + name + "'"
         )
       } mkString ", "
     }

Modified: hash-join/trunk/tools/Makefile
===================================================================
--- hash-join/trunk/tools/Makefile	2008-02-11 17:03:04 UTC (rev 376)
+++ hash-join/trunk/tools/Makefile	2008-02-11 17:03:34 UTC (rev 377)
@@ -23,7 +23,9 @@
 
 proc: out/LogProc.class
 	mkdir -p data
-	scala -cp out LogProc $(log)
+	egrep 'cpus|time: ' $(log) | \
+		sed 's/ time: /: /' | \
+		scala -cp out LogProc " cpus"
 
 titles: out/Titles.class
 	cat ../src/movies.dat | tr '\0' '\n' | scala -cp out Titles > titles.txt


This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.

[Assorted-commits] SF.net SVN: assorted: [376] hash-join/trunk/src/bench.bash

[<yan...@us...>]

Revision: 376
          http://assorted.svn.sourceforge.net/assorted/?rev=376&view=rev
Author:   yangzhang
Date:     2008-02-11 09:03:04 -0800 (Mon, 11 Feb 2008)

Log Message:
-----------
added multiple trials

Modified Paths:
--------------
    hash-join/trunk/src/bench.bash

Modified: hash-join/trunk/src/bench.bash
===================================================================
--- hash-join/trunk/src/bench.bash	2008-02-11 17:02:55 UTC (rev 375)
+++ hash-join/trunk/src/bench.bash	2008-02-11 17:03:04 UTC (rev 376)
@@ -3,6 +3,8 @@
 set -o nounset
 set -o errexit
 
-for i in 1 `seq 2 2 16` `seq 24 8 64`
-do ./$TARGET-opt $i $MOVIEDATA/movies.dat $MOVIEDATA/actresses.dat
+for i in 1 `seq 2 2 16` `seq 24 8 64` ; do
+  for j in {1..3} ; do
+    ./$TARGET-opt $i $MOVIEDATA/movies.dat $MOVIEDATA/actresses.dat
+  done
 done >& log


This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.

[Assorted-commits] SF.net SVN: assorted: [375] hash-join/trunk/src/hashjoin.cc

[<yan...@us...>]

Revision: 375
          http://assorted.svn.sourceforge.net/assorted/?rev=375&view=rev
Author:   yangzhang
Date:     2008-02-11 09:02:55 -0800 (Mon, 11 Feb 2008)

Log Message:
-----------
moved to new output for more friendly log processing

Modified Paths:
--------------
    hash-join/trunk/src/hashjoin.cc

Modified: hash-join/trunk/src/hashjoin.cc
===================================================================
--- hash-join/trunk/src/hashjoin.cc	2008-02-11 15:53:17 UTC (rev 374)
+++ hash-join/trunk/src/hashjoin.cc	2008-02-11 17:02:55 UTC (rev 375)
@@ -151,38 +151,45 @@
 
   cout << "using " << ncpus << " cpus" << endl;
 
-  timer t("main time: ");
-
   // Load the data files.
 
   cout << "loading movies" << endl;
+  timer tldmov("loading movies time: ");
   movdb mdb(movies);
-  t.print();
+  tldmov.print();
 
   cout << "loading actresses" << endl;
+  timer tldact("loading actresses time: ");
   actdb adb(actresses);
-  t.print();
+  tldact.print();
 
+  timer ttotal("total time: ");
+
   // Hash-partition the data among the nodes.
 
   cout << "hash-partitioning movies into per-core buckets" << endl;
+  timer thpmov("hash-partitioning movies time: ");
   const bucket **movbucs = mdb.partition();
-  t.print();
+  thpmov.print();
 
   cout << "hash-partitioning actresses into per-core buckets" << endl;
+  timer thpact("hash-partitioning actresses time: ");
   const bucket **actbucs = adb.partition();
-  t.print();
+  thpact.print();
 
   // Perform the hash-join.
 
   cout << "building with movies" << endl;
+  timer tbuild("building with movies time: ");
   const hmap *hs = mdb.build(movbucs);
-  t.print();
+  tbuild.print();
 
   cout << "probing with actresses" << endl;
+  timer tprobe("probing with actresses time: ");
   adb.probe(hs, actbucs, true);
-  t.print();
+  tprobe.print();
 
+  ttotal.print();
   cout << "done" << endl;
 }
 


This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.

[Assorted-commits] SF.net SVN: assorted: [374] hash-join/trunk/src/hashjoin.cc

[<yan...@us...>]

Revision: 374
          http://assorted.svn.sourceforge.net/assorted/?rev=374&view=rev
Author:   yangzhang
Date:     2008-02-11 07:53:17 -0800 (Mon, 11 Feb 2008)

Log Message:
-----------
added debug output

Modified Paths:
--------------
    hash-join/trunk/src/hashjoin.cc

Modified: hash-join/trunk/src/hashjoin.cc
===================================================================
--- hash-join/trunk/src/hashjoin.cc	2008-02-11 15:45:31 UTC (rev 373)
+++ hash-join/trunk/src/hashjoin.cc	2008-02-11 15:53:17 UTC (rev 374)
@@ -427,6 +427,7 @@
             hits++;
             join(title, name);
           } else {
+	    if (misses == 0) cerr << "MISS: '" << title << '\'' << endl;
             misses++;
           }
           // End of a tuple? (Don't actually need this check, since the


This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.

[Assorted-commits] SF.net SVN: assorted: [373] hash-join/trunk/tools/LogProc.scala

[<yan...@us...>]

Revision: 373
          http://assorted.svn.sourceforge.net/assorted/?rev=373&view=rev
Author:   yangzhang
Date:     2008-02-11 07:45:31 -0800 (Mon, 11 Feb 2008)

Log Message:
-----------
fixed data output path; changed ms -> s

Modified Paths:
--------------
    hash-join/trunk/tools/LogProc.scala

Modified: hash-join/trunk/tools/LogProc.scala
===================================================================
--- hash-join/trunk/tools/LogProc.scala	2008-02-11 15:45:13 UTC (rev 372)
+++ hash-join/trunk/tools/LogProc.scala	2008-02-11 15:45:31 UTC (rev 373)
@@ -1,12 +1,14 @@
 import commons.Collections._
 import commons.Control._
 import commons.Io._
+import commons.Debug._
+import commons.Path._
 // import commons.Plotting._
 import scala.collection.mutable._
 
 object LogProc {
-  type FieldMap = Map[String,Int]
-  type MutFieldMap = HashMap[String,Int]
+  type FieldMap = Map[String,Double]
+  type MutFieldMap = HashMap[String,Double]
   case class Stats(
     ncpus: Int,
     values: FieldMap
@@ -33,20 +35,22 @@
     for (line <- lines) {
       if (line contains " cpus") {
         // Include sum.
-        map("sum") = sum(map.values)
-        if (ncpus != 0) stats += Stats(ncpus, map.clone)
+        if (ncpus != 0) {
+          map("sum") = sum(map.values) - map("actressLoading") - map("movieLoading")
+          stats += Stats(ncpus, map.clone)
+        }
         ncpus = line.split(" ")(1).toInt
         fieldIndex = Iterator from 0
         map.clear
       } else if (line contains "main time: ") {
-        map(fieldName(fieldIndex.next)) = line.split(" ").last.toInt
+        map(fieldName(fieldIndex.next)) = line.split(" ").last.toDouble / 1000.0
       }
     }
 
     // Build actual plot data.
-    val plotData = new HashMap[String,ArrayBuffer[Int]] {
+    val plotData = new HashMap[String,ArrayBuffer[Double]] {
       override def default(k: String) = {
-        val buf = new ArrayBuffer[Int]
+        val buf = new ArrayBuffer[Double]
         this(k) = buf
         buf
       }
@@ -62,12 +66,12 @@
     for ((field,times) <- plotData) {
       val baseline = times(0).toDouble
       println(field + ": " + times)
-      using (TextWriter(camelToHyphen(field) + "-time.dat")) { w =>
+      using (TextWriter("data" / camelToHyphen(field) + "-time.dat")) { w =>
         for ((time,ncpus) <- times zip ncpuList) {
           w.println(ncpus + " " + time)
         }
       }
-      using (TextWriter(camelToHyphen(field) + "-speedup.dat")) { w =>
+      using (TextWriter("data" / camelToHyphen(field) + "-speedup.dat")) { w =>
         for ((time,ncpus) <- times map (baseline / _) zip ncpuList) {
           w.println(ncpus + " " + time)
         }
@@ -87,7 +91,7 @@
       set xlabel 'number of threads'
 
       set output 'data/times.pdf'
-      set ylabel 'time (ms)'
+      set ylabel 'time (s)'
       plot """ + f("-time") + """
 
       set output 'data/speedups.pdf'


This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.

[Assorted-commits] SF.net SVN: assorted: [372] hash-join/trunk/tools/DbPrep.scala

[<yan...@us...>]

Revision: 372
          http://assorted.svn.sourceforge.net/assorted/?rev=372&view=rev
Author:   yangzhang
Date:     2008-02-11 07:45:13 -0800 (Mon, 11 Feb 2008)

Log Message:
-----------
fixed bug: not cleaning up titles on subsequent repetitions

Modified Paths:
--------------
    hash-join/trunk/tools/DbPrep.scala

Modified: hash-join/trunk/tools/DbPrep.scala
===================================================================
--- hash-join/trunk/tools/DbPrep.scala	2008-02-11 05:46:03 UTC (rev 371)
+++ hash-join/trunk/tools/DbPrep.scala	2008-02-11 15:45:13 UTC (rev 372)
@@ -55,11 +55,11 @@
                   }
                   if (body && line != "") {
                     val (actress, title) = extract(pActress, line)
-                    wa print (actress + "\0" + cleanTitle(xform(title)) + "\0")
+                    wa print (actress + "\0" + xform(cleanTitle(title)) + "\0")
                     while (line != "") {
                       line = r.readLine.trim
                       if (line != "") {
-                        wa print (cleanTitle(xform(title)) + "\0")
+                        wa print (xform(cleanTitle(title)) + "\0")
                       }
                     }
                     wa print "\0"


This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.

[Assorted-commits] SF.net SVN: assorted: [371] hash-join/trunk/src

[<yan...@us...>]

Revision: 371
          http://assorted.svn.sourceforge.net/assorted/?rev=371&view=rev
Author:   yangzhang
Date:     2008-02-10 21:46:03 -0800 (Sun, 10 Feb 2008)

Log Message:
-----------
moved benchmarking code out of makefile

Modified Paths:
--------------
    hash-join/trunk/src/Makefile

Added Paths:
-----------
    hash-join/trunk/src/bench.bash

Modified: hash-join/trunk/src/Makefile
===================================================================
--- hash-join/trunk/src/Makefile	2008-02-11 05:18:57 UTC (rev 370)
+++ hash-join/trunk/src/Makefile	2008-02-11 05:46:03 UTC (rev 371)
@@ -1,5 +1,6 @@
 TARGET := hashjoin
 SRCS   := hashjoin.cc $(wildcard commons/*.h)
+export TARGET
 
 ### begin common makefrag
 
@@ -42,6 +43,4 @@
 ### end common makefrag
 
 bench: $(TARGET)-opt
-	for i in 1 `seq 2 2 16` `seq 24 8 64` ; do \
-		./$(TARGET)-opt $$i $(MOVIEDATA)/movies.dat $(MOVIEDATA)/actresses.dat ; \
-	done > log 2>&1
+	./bench.bash

Added: hash-join/trunk/src/bench.bash
===================================================================
--- hash-join/trunk/src/bench.bash	                        (rev 0)
+++ hash-join/trunk/src/bench.bash	2008-02-11 05:46:03 UTC (rev 371)
@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+set -o nounset
+set -o errexit
+
+for i in 1 `seq 2 2 16` `seq 24 8 64`
+do ./$TARGET-opt $i $MOVIEDATA/movies.dat $MOVIEDATA/actresses.dat
+done >& log


Property changes on: hash-join/trunk/src/bench.bash
___________________________________________________________________
Name: svn:executable
   + *


This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.

[Assorted-commits] SF.net SVN: assorted: [370] hash-join/trunk/tools/Makefile

[<yan...@us...>]

Revision: 370
          http://assorted.svn.sourceforge.net/assorted/?rev=370&view=rev
Author:   yangzhang
Date:     2008-02-10 21:18:57 -0800 (Sun, 10 Feb 2008)

Log Message:
-----------
tweaks

Modified Paths:
--------------
    hash-join/trunk/tools/Makefile

Modified: hash-join/trunk/tools/Makefile
===================================================================
--- hash-join/trunk/tools/Makefile	2008-02-11 05:18:47 UTC (rev 369)
+++ hash-join/trunk/tools/Makefile	2008-02-11 05:18:57 UTC (rev 370)
@@ -1,24 +1,35 @@
-COMMONS_SRCS := $(wildcard commons/*.scala)
-DBPREP_SRCS  := DbPrep.scala $(COMMONS_SRCS)
-LOGPREP_SRCS := LogProc.scala $(COMMONS_SRCS)
+COMMONS_SRCS  := $(wildcard commons/*.scala)
+DBPREP_SRCS   := DbPrep.scala $(COMMONS_SRCS)
+LOGPROC_SRCS  := LogProc.scala $(COMMONS_SRCS)
+TITLES_SRCS   := Titles.scala $(COMMONS_SRCS)
+log           := log-josmp
+rep           := 1
 
+FSC            = mkdir -p out && fsc -deprecation -d out $^
+
 all: out/DbPrep.class out/LogProc.class
 
 out/DbPrep.class: $(DBPREP_SRCS)
-	mkdir -p out
-	fsc -deprecation -d out $^
+	$(FSC)
 
-out/LogProc.class: $(LOGPREP_SRCS)
-	mkdir -p out
-	fsc -deprecation -d out $^
+out/LogProc.class: $(LOGPROC_SRCS)
+	$(FSC)
 
-run: out/DbPrep.class
-	scala -cp out DbPrep
+out/Titles.class: $(TITLES_SRCS)
+	$(FSC)
 
+prep: out/DbPrep.class
+	scala -cp out DbPrep $(rep)
+
 proc: out/LogProc.class
-	scala -cp out LogProc log-opt
+	mkdir -p data
+	scala -cp out LogProc $(log)
 
+titles: out/Titles.class
+	cat ../src/movies.dat | tr '\0' '\n' | scala -cp out Titles > titles.txt
+	# head -c 1024 movies.dat | tr '\0' '\n' | scala -cp out Titles > titles.txt
+
 clean:
 	rm -rf out
 
-.PHONY: clean run
+.PHONY: clean run prep proc


This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.

[Assorted-commits] SF.net SVN: assorted: [369] hash-join/trunk/tools/DbPrep.scala

[<yan...@us...>]

Revision: 369
          http://assorted.svn.sourceforge.net/assorted/?rev=369&view=rev
Author:   yangzhang
Date:     2008-02-10 21:18:47 -0800 (Sun, 10 Feb 2008)

Log Message:
-----------
DbPrep can generate larger data sets by repeating n times and using rot encoding

Modified Paths:
--------------
    hash-join/trunk/tools/DbPrep.scala

Modified: hash-join/trunk/tools/DbPrep.scala
===================================================================
--- hash-join/trunk/tools/DbPrep.scala	2008-02-11 05:17:48 UTC (rev 368)
+++ hash-join/trunk/tools/DbPrep.scala	2008-02-11 05:18:47 UTC (rev 369)
@@ -1,3 +1,4 @@
+import commons.Collections._
 import commons.Control._
 import commons.Io._
 import java.util.regex._
@@ -15,59 +16,63 @@
     val pMovie = Pattern compile """^([^\t]+)\t+(.*)$"""
     val pActress = Pattern compile """^([^\t]+)\t+([^\t]+)$"""
     val (doMovies, doActresses) = (true, true)
-    if (doMovies) {
-      using (TextReader("movies.list")) { r =>
-        using (TextWriter("movies.dat")) { w =>
-          var line = r.readLine
-          try {
-            var body = false
-            while (line != null) {
-              if (body && (line contains "----------------")) {
-                body = false
+    val nreps = args(0).toInt
+    using (TextWriter("movies.dat")) { wm =>
+      using (TextWriter("actresses.dat")) { wa =>
+        for (i <- 0 until nreps) {
+          def xform(s: String) = if (i == 0) s else rot(i, s)
+          if (doMovies) {
+            using (TextReader("movies.list")) { r =>
+              var line = r.readLine
+              try {
+                var body = false
+                while (line != null) {
+                  if (body && (line contains "----------------")) {
+                    body = false
+                  }
+                  if (body && line != "") {
+                    val (title, release) = extract(pMovie, line)
+                    wm print (xform(title) + "\0" + release + "\0\0")
+                  }
+                  if (!body && (line contains "=======")) {
+                    body = true
+                  }
+                  line = r.readLine
+                }
+              } catch {
+                case e: Exception => { Console.err.println(line); throw e }
               }
-              if (body && line != "") {
-                val (title, release) = extract(pMovie, line)
-                w print (title + "\0" + release + "\0\0")
-              }
-              if (!body && (line contains "=======")) {
-                body = true
-              }
-              line = r.readLine
             }
-          } catch {
-            case e: Exception => { Console.err.println(line); throw e }
           }
-        }
-      }
-    }
-    if (doActresses) {
-      using (TextReader("actresses.list")) { r =>
-        using (TextWriter("actresses.dat")) { w =>
-          var line = r.readLine
-          try {
-            var body = false
-            while (line != null) {
-              if (body && (line contains "----------------")) {
-                body = false
-              }
-              if (body && line != "") {
-                val (actress, title) = extract(pActress, line)
-                w print (actress + "\0" + cleanTitle(title) + "\0")
-                while (line != "") {
-                  line = r.readLine.trim
-                  if (line != "") {
-                    w print (cleanTitle(title) + "\0")
+          if (doActresses) {
+            using (TextReader("actresses.list")) { r =>
+              var line = r.readLine
+              try {
+                var body = false
+                while (line != null) {
+                  if (body && (line contains "----------------")) {
+                    body = false
                   }
+                  if (body && line != "") {
+                    val (actress, title) = extract(pActress, line)
+                    wa print (actress + "\0" + cleanTitle(xform(title)) + "\0")
+                    while (line != "") {
+                      line = r.readLine.trim
+                      if (line != "") {
+                        wa print (cleanTitle(xform(title)) + "\0")
+                      }
+                    }
+                    wa print "\0"
+                  }
+                  if (!body && ((line contains "\t") && (line startsWith "----") && (line endsWith "----"))) {
+                    body = true
+                  }
+                  line = r.readLine
                 }
-                w print "\0"
+              } catch {
+                case e: Exception => { Console.err.println(line); throw e }
               }
-              if (!body && ((line contains "\t") && (line startsWith "----") && (line endsWith "----"))) {
-                body = true
-              }
-              line = r.readLine
             }
-          } catch {
-            case e: Exception => { Console.err.println(line); throw e }
           }
         }
       }


This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.

[Assorted-commits] SF.net SVN: assorted: [368] hash-join/trunk/tools/LogProc.scala

[<yan...@us...>]

Revision: 368
          http://assorted.svn.sourceforge.net/assorted/?rev=368&view=rev
Author:   yangzhang
Date:     2008-02-10 21:17:48 -0800 (Sun, 10 Feb 2008)

Log Message:
-----------
LogProc outputs to data/

Modified Paths:
--------------
    hash-join/trunk/tools/LogProc.scala

Modified: hash-join/trunk/tools/LogProc.scala
===================================================================
--- hash-join/trunk/tools/LogProc.scala	2008-02-11 05:17:25 UTC (rev 367)
+++ hash-join/trunk/tools/LogProc.scala	2008-02-11 05:17:48 UTC (rev 368)
@@ -78,7 +78,7 @@
     def f(s:String) = {
       {
         for ((field,_) <- plotData) yield (
-          "'" + camelToHyphen(field) + s + ".dat" + "' with linespoints title '" + fieldNameToLabel(field) + "'"
+          "'data/" + camelToHyphen(field) + s + ".dat" + "' with linespoints title '" + fieldNameToLabel(field) + "'"
         )
       } mkString ", "
     }
@@ -86,11 +86,11 @@
       set terminal pdf
       set xlabel 'number of threads'
 
-      set output 'times.pdf'
+      set output 'data/times.pdf'
       set ylabel 'time (ms)'
       plot """ + f("-time") + """
 
-      set output 'speedups.pdf'
+      set output 'data/speedups.pdf'
       set ylabel 'speedup (relative to 1 thread)'
       plot """ + f("-speedup")
     )


This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.

[Assorted-commits] SF.net SVN: assorted: [367] hash-join/trunk/tools/Titles.scala

[<yan...@us...>]

Revision: 367
          http://assorted.svn.sourceforge.net/assorted/?rev=367&view=rev
Author:   yangzhang
Date:     2008-02-10 21:17:25 -0800 (Sun, 10 Feb 2008)

Log Message:
-----------
added Titles filter

Added Paths:
-----------
    hash-join/trunk/tools/Titles.scala

Added: hash-join/trunk/tools/Titles.scala
===================================================================
--- hash-join/trunk/tools/Titles.scala	                        (rev 0)
+++ hash-join/trunk/tools/Titles.scala	2008-02-11 05:17:25 UTC (rev 367)
@@ -0,0 +1,13 @@
+import commons.Collections._
+object Titles {
+  def main(args: Array[String]) {
+    var newPar = true
+    for (line <- untilNull(Console.readLine)) {
+      if (newPar) {
+        println(line)
+        newPar = false
+      }
+      if (line isEmpty) newPar = true
+    }
+  }
+}


This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.

[Assorted-commits] SF.net SVN: assorted: [366] numa-bench/trunk/src/Makefile

[<yan...@us...>]

Revision: 366
          http://assorted.svn.sourceforge.net/assorted/?rev=366&view=rev
Author:   yangzhang
Date:     2008-02-10 20:57:56 -0800 (Sun, 10 Feb 2008)

Log Message:
-----------
just build .c; added more targets

Modified Paths:
--------------
    numa-bench/trunk/src/Makefile

Modified: numa-bench/trunk/src/Makefile
===================================================================
--- numa-bench/trunk/src/Makefile	2008-02-11 04:57:31 UTC (rev 365)
+++ numa-bench/trunk/src/Makefile	2008-02-11 04:57:56 UTC (rev 366)
@@ -1,5 +1,5 @@
 COMMONS := $(wildcard commons/*.h)
-CXX = g++-4.2 -I. -lnuma -lpthread -o $@ $^
+CXX = g++-4.2 -I. -lnuma -lpthread -o $@ $<
 
 # all: avail cache malloc threads
 all: malloc
@@ -19,6 +19,9 @@
 openmp: openmp.cc
 	$(CXX) -fopenmp
 
+bthreads: bthreads.cc
+	$(CXX) -lboost_thread-gcc41-mt
+
 clean:
 	rm -f avail cache
 


This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.

[Assorted-commits] SF.net SVN: assorted: [365] numa-bench/trunk/src/malloc.cc

[<yan...@us...>]

Revision: 365
          http://assorted.svn.sourceforge.net/assorted/?rev=365&view=rev
Author:   yangzhang
Date:     2008-02-10 20:57:31 -0800 (Sun, 10 Feb 2008)

Log Message:
-----------
added cpu pinning to malloc

Modified Paths:
--------------
    numa-bench/trunk/src/malloc.cc

Modified: numa-bench/trunk/src/malloc.cc
===================================================================
--- numa-bench/trunk/src/malloc.cc	2008-02-11 04:57:12 UTC (rev 364)
+++ numa-bench/trunk/src/malloc.cc	2008-02-11 04:57:31 UTC (rev 365)
@@ -5,17 +5,21 @@
 
 #include <sched.h>
 
+#include <boost/bind.hpp>
+
 #include <commons/check.h>
 #include <commons/threads.h>
 #include <commons/time.h>
+#include <commons/boost/threads.h>
 
+using namespace boost;
 using namespace commons;
 using namespace std;
 
 const size_t size = 10000000;
 
 void*
-chew(void* pp)
+chew(void* pp, int core)
 {
   char* p = (char*) pp;
   const int reps = 100;
@@ -25,17 +29,16 @@
   // Pin this thread to the right processor.
   cpu_set_t cs;
   CPU_ZERO(&cs);
-  CPU_SET(1, &cs);
+  CPU_SET(core, &cs);
   sched_setaffinity(pid, sizeof(cs), &cs);
 
-  // TODO: try shuffling indexes
   for (int c = 0; c < reps; c++) {
     for (size_t i = 0; i < size; i++) {
       p[i] = i;
     }
   }
 
-  // Print the elapsed time;
+  // Print the elapsed time.
   cout << pid;
   t.print();
   return NULL;
@@ -53,21 +56,23 @@
   void *p = malloc(size);
 
   // warmup
-  chew(p);
+  chew(p, 0);
   pthread_t ts[n];
 
   // start thread on each core
   for (int i = 0; i < n; i++) {
-    check(pthread_create(&ts[i], NULL, chew, p) == 0);
+    pthread_t t;
+    check((t = spawn(bind(chew, p, i))) != 0);
+    check(pthread_join(t, NULL) == 0);
   }
-  waitall(ts, n);
+  // waitall(ts, n);
   return 0;
 
   // THRASH
 
   // spawn workers
   for (int i = 0; i < n; i++) {
-    check(pthread_create(&ts[i], NULL, chew, p) == 0);
+    check((ts[i] = spawn(bind(chew, p, i))) == 0);
   }
   waitall(ts, n);
   return 0;


This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.

[Assorted-commits] SF.net SVN: assorted: [364] numa-bench/trunk/src/bthreads.cc

[<yan...@us...>]

Revision: 364
          http://assorted.svn.sourceforge.net/assorted/?rev=364&view=rev
Author:   yangzhang
Date:     2008-02-10 20:57:12 -0800 (Sun, 10 Feb 2008)

Log Message:
-----------
added boost thread test

Added Paths:
-----------
    numa-bench/trunk/src/bthreads.cc

Added: numa-bench/trunk/src/bthreads.cc
===================================================================
--- numa-bench/trunk/src/bthreads.cc	                        (rev 0)
+++ numa-bench/trunk/src/bthreads.cc	2008-02-11 04:57:12 UTC (rev 364)
@@ -0,0 +1,37 @@
+#include <iostream>
+
+#include <boost/thread/thread.hpp>
+#include <boost/bind.hpp>
+
+using namespace std;
+using namespace boost;
+
+void
+cnt(int i)
+{
+  cout << i << endl;
+}
+
+int
+main()
+{
+  const int n = 4;
+  {
+    thread* ts[n];
+    for (int i = 0; i < n; i++) {
+      ts[i] = new thread(bind(&cnt, i));
+    }
+    for (int i = 0; i < n; i++) {
+      ts[i]->join();
+    }
+  }
+
+  {
+    thread_group ts;
+    for (int i = 0; i < n; i++) {
+      ts.create_thread(bind(&cnt, i));
+    }
+    ts.join_all();
+  }
+  return 0;
+}


This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.

[Assorted-commits] SF.net SVN: assorted: [363] sandbox/trunk/src/cc/stl_seq_bench.cc

[<yan...@us...>]

Revision: 363
          http://assorted.svn.sourceforge.net/assorted/?rev=363&view=rev
Author:   yangzhang
Date:     2008-02-10 20:56:41 -0800 (Sun, 10 Feb 2008)

Log Message:
-----------
added stl sequence benchmark

Added Paths:
-----------
    sandbox/trunk/src/cc/stl_seq_bench.cc

Added: sandbox/trunk/src/cc/stl_seq_bench.cc
===================================================================
--- sandbox/trunk/src/cc/stl_seq_bench.cc	                        (rev 0)
+++ sandbox/trunk/src/cc/stl_seq_bench.cc	2008-02-11 04:56:41 UTC (rev 363)
@@ -0,0 +1,33 @@
+// A simple performance benchmark for the various sequence containers.
+//
+// TODO: add list
+
+#include <deque>
+#include <vector>
+
+#include <commons/time.h>
+
+using namespace std;
+using namespace commons;
+
+template<typename T> void
+doit(T xs)
+{
+  const int n = 1000000, t = 100;
+  timer timer("");
+  for (int j = 0; j < t; j++) {
+    for (int i = 0; i < n; i++)
+      xs.push_back(i);
+  }
+  timer.print();
+}
+
+int
+main()
+{
+  doit(deque<int>());
+  doit(deque<int>());
+  doit(vector<int>());
+  doit(vector<int>());
+  return 0;
+}


This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.

[Assorted-commits] SF.net SVN: assorted: [362] cpp-commons/trunk/src/commons

[<yan...@us...>]

Revision: 362
          http://assorted.svn.sourceforge.net/assorted/?rev=362&view=rev
Author:   yangzhang
Date:     2008-02-10 20:54:57 -0800 (Sun, 10 Feb 2008)

Log Message:
-----------
tweaks

Modified Paths:
--------------
    cpp-commons/trunk/src/commons/check.h
    cpp-commons/trunk/src/commons/cppcommons.cpp
    cpp-commons/trunk/src/commons/hash.h

Modified: cpp-commons/trunk/src/commons/check.h
===================================================================
--- cpp-commons/trunk/src/commons/check.h	2008-02-11 04:54:43 UTC (rev 361)
+++ cpp-commons/trunk/src/commons/check.h	2008-02-11 04:54:57 UTC (rev 362)
@@ -38,6 +38,8 @@
   /**
    * Similar to assert(), but is not conditionally compiled, so this is safe to
    * use as a guard against expected failures (such as checking return codes).
+   * This is a macro for two reasons: (1) the file and line, and (2) the lazy
+   * evaluation of msg.
    */
 #define checkmsg(cond, msg) \
     bool b = cond; \

Modified: cpp-commons/trunk/src/commons/cppcommons.cpp
===================================================================
--- cpp-commons/trunk/src/commons/cppcommons.cpp	2008-02-11 04:54:43 UTC (rev 361)
+++ cpp-commons/trunk/src/commons/cppcommons.cpp	2008-02-11 04:54:57 UTC (rev 362)
@@ -23,6 +23,7 @@
 #include <config.h>
 #endif
 
+#include <commons/boost/threads.h>
 #include <commons/check.h>
 #include <commons/cpuid.h>
 #include <commons/files.h>

Modified: cpp-commons/trunk/src/commons/hash.h
===================================================================
--- cpp-commons/trunk/src/commons/hash.h	2008-02-11 04:54:43 UTC (rev 361)
+++ cpp-commons/trunk/src/commons/hash.h	2008-02-11 04:54:57 UTC (rev 362)
@@ -1,6 +1,8 @@
 #ifndef _COMMONS_HASH_H
 #define _COMMONS_HASH_H
 
+#include <sys/types.h>
+
 namespace commons
 {
 


This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.

[Assorted-commits] SF.net SVN: assorted: [361] cpp-commons/trunk/src/commons/threads.h

[<yan...@us...>]

Revision: 361
          http://assorted.svn.sourceforge.net/assorted/?rev=361&view=rev
Author:   yangzhang
Date:     2008-02-10 20:54:43 -0800 (Sun, 10 Feb 2008)

Log Message:
-----------
cleaned up threads.h; added spawn, runner

Modified Paths:
--------------
    cpp-commons/trunk/src/commons/threads.h

Modified: cpp-commons/trunk/src/commons/threads.h
===================================================================
--- cpp-commons/trunk/src/commons/threads.h	2008-02-11 04:54:05 UTC (rev 360)
+++ cpp-commons/trunk/src/commons/threads.h	2008-02-11 04:54:43 UTC (rev 361)
@@ -1,133 +1,193 @@
-// TODO: use boost::bind?
-
 #ifndef _COMMONS_THREADS_H
 #define _COMMONS_THREADS_H
 
 #include <pthread.h>
+#include <sys/syscall.h>
+#include <sys/types.h>
+#include <unistd.h>
 
+#include <commons/check.h>
+
 namespace commons
 {
 
-  // non-rpc-specific utility to start a thread that runs
-  // an object method. returns a pthread_t on success, and
-  // zero on error.
+  /**
+   * Get the current thread ID. Glibc does not provide a wrapper for this
+   * system call.
+   */
+  inline pid_t
+  gettid()
+  {
+    return (pid_t) syscall(SYS_gettid);
+  }
+
+  /**
+   * Wait for all the given threads to join, discarding their return values.
+   */
+  inline void
+  waitall(pthread_t* ts, int n)
+  {
+    for (int i = 0; i < n; i++) {
+      check(pthread_join(ts[i], NULL) == 0);
+    }
+  }
+
+  /**
+   * Helper for launching new threads.
+   */
+  void*
+  runner(void *p)
+  {
+    void (*f)() = (void(*)()) p;
+    (*f)();
+    return NULL;
+  }
+
+  /**
+   * Run a function in pthread.
+   * \return The new pthread_t on success, 0 on failure.
+    * TODO: Is it safe to treat the pthread_t as an integral type?
+   */
+  pthread_t
+  spawn(void (*f)())
+  {
+    pthread_t t;
+    return pthread_create(&t, NULL, &runner, (void*) f) == 0 ? t : 0;
+  }
+
+  /**
+   * Run a method in pthread.
+   * \return The new pthread_t on success, 0 on failure.
+   */
   template <class C> pthread_t 
   method_thread(C *o, void (C::*m)())
   {
-    class XXX {
+    class runnable {
     public:
       C *o;
       void (C::*m)();
-      static void *yyy(void *vvv) {
-        XXX *x = (XXX*)vvv;
-        C *o = x->o;
-        void (C::*m)() = x->m;
-        delete x;
+      static void *run(void *vvv) {
+        runnable *r = (runnable*)vvv;
+        C *o = r->o;
+        void (C::*m)() = r->m;
+        delete r;
         (o->*m)();
         return 0;
       }
     };
-    XXX *x = new XXX;
-    x->o = o;
-    x->m = m;
+    runnable *r = new runnable;
+    r->o = o;
+    r->m = m;
     pthread_t th;
-    if(pthread_create(&th, NULL, &XXX::yyy, (void *) x) == 0){
+    if(pthread_create(&th, NULL, &runnable::run, (void *) r) == 0){
       return th;
     }
     return 0;
   }
 
+  /**
+   * Run a method in pthread.
+   * \return The new pthread_t on success, 0 on failure.
+   */
   template <class C, class A> pthread_t
   method_thread(C *o, void (C::*m)(A), A a)
   {
-    class XXX {
+    class runnable {
     public:
       C *o;
       void (C::*m)(A a);
       A a;
-      static void *yyy(void *vvv) {
-        XXX *x = (XXX*)vvv;
-        C *o = x->o;
-        void (C::*m)(A ) = x->m;
-        A a = x->a;
-        delete x;
+      static void *run(void *vvv) {
+        runnable *r = (runnable*)vvv;
+        C *o = r->o;
+        void (C::*m)(A ) = r->m;
+        A a = r->a;
+        delete r;
         (o->*m)(a);
         return 0;
       }
     };
-    XXX *x = new XXX;
-    x->o = o;
-    x->m = m;
-    x->a = a;
+    runnable *r = new runnable;
+    r->o = o;
+    r->m = m;
+    r->a = a;
     pthread_t th;
-    if(pthread_create(&th, NULL, &XXX::yyy, (void *) x) == 0){
+    if(pthread_create(&th, NULL, &runnable::run, (void *) r) == 0){
       return th;
     }
     return 0;
   }
 
+  /**
+   * Run a method in pthread.
+   * \return The new pthread_t on success, 0 on failure.
+   */
   template <class C, class A1, class A2> pthread_t
   method_thread(C *o, void (C::*m)(A1 , A2 ), A1 a1, A2 a2)
   {
-    class XXX {
+    class runnable {
     public:
       C *o;
       void (C::*m)(A1 a1, A2 a2);
       A1 a1;
-      A2  a2;
-      static void *yyy(void *vvv) {
-        XXX *x = (XXX*)vvv;
-        C *o = x->o;
-        void (C::*m)(A1 , A2 ) = x->m;
-        A1 a1 = x->a1;
-        A2 a2 = x->a2;
-        delete x;
+      A2 a2;
+      static void *run(void *vvv) {
+        runnable *r = (runnable*)vvv;
+        C *o = r->o;
+        void (C::*m)(A1 , A2 ) = r->m;
+        A1 a1 = r->a1;
+        A2 a2 = r->a2;
+        delete r;
         (o->*m)(a1, a2);
         return 0;
       }
     };
-    XXX *x = new XXX;
-    x->o = o;
-    x->m = m;
-    x->a1 = a1;
-    x->a2 = a2;
+    runnable *r = new runnable;
+    r->o = o;
+    r->m = m;
+    r->a1 = a1;
+    r->a2 = a2;
     pthread_t th;
-    if(pthread_create(&th, NULL, &XXX::yyy, (void *) x) == 0){
+    if(pthread_create(&th, NULL, &runnable::run, (void *) r) == 0){
       return th;
     }
     return 0;
   }
 
+  /**
+   * Run a method in pthread.
+   * \return The new pthread_t on success, 0 on failure.
+   */
   template <class C, class A1, class A2, class A3> pthread_t
   method_thread(C *o, void (C::*m)(A1 , A2, A3), A1 a1, A2 a2, A3 a3)
   {
-    class XXX {
+    class runnable {
     public:
       C *o;
       void (C::*m)(A1 a1, A2 a2, A3 a3);
       A1 a1;
       A2 a2;
       A3 a3;
-      static void *yyy(void *vvv) {
-        XXX *x = (XXX*)vvv;
-        C *o = x->o;
-        void (C::*m)(A1, A2, A3) = x->m;
-        A1 a1 = x->a1;
-        A2 a2 = x->a2;
-        A3 a3 = x->a3;
-        delete x;
+      static void *run(void *vvv) {
+        runnable *r = (runnable*)vvv;
+        C *o = r->o;
+        void (C::*m)(A1, A2, A3) = r->m;
+        A1 a1 = r->a1;
+        A2 a2 = r->a2;
+        A3 a3 = r->a3;
+        delete r;
         (o->*m)(a1, a2, a3);
         return 0;
       }
     };
-    XXX *x = new XXX;
-    x->o = o;
-    x->m = m;
-    x->a1 = a1;
-    x->a2 = a2;
-    x->a3 = a3;
+    runnable *r = new runnable;
+    r->o = o;
+    r->m = m;
+    r->a1 = a1;
+    r->a2 = a2;
+    r->a3 = a3;
     pthread_t th;
-    if(pthread_create(&th, NULL, &XXX::yyy, (void *) x) == 0){
+    if(pthread_create(&th, NULL, &runnable::run, (void *) r) == 0){
       return th;
     }
     return 0;


This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.

[Assorted-commits] SF.net SVN: assorted: [360] cpp-commons/trunk/src/commons

[<yan...@us...>]

Revision: 360
          http://assorted.svn.sourceforge.net/assorted/?rev=360&view=rev
Author:   yangzhang
Date:     2008-02-10 20:54:05 -0800 (Sun, 10 Feb 2008)

Log Message:
-----------
added boost and boost thread utils

Added Paths:
-----------
    cpp-commons/trunk/src/commons/boost/
    cpp-commons/trunk/src/commons/boost/threads.h

Added: cpp-commons/trunk/src/commons/boost/threads.h
===================================================================
--- cpp-commons/trunk/src/commons/boost/threads.h	                        (rev 0)
+++ cpp-commons/trunk/src/commons/boost/threads.h	2008-02-11 04:54:05 UTC (rev 360)
@@ -0,0 +1,45 @@
+#ifndef _COMMONS_BOOST_THREADS_H
+#define _COMMONS_BOOST_THREADS_H
+
+#include <pthread.h>
+#include <sys/syscall.h>
+#include <sys/types.h>
+#include <unistd.h>
+
+#include <boost/function.hpp>
+
+#include <commons/check.h>
+#include <commons/threads.h>
+
+namespace commons
+{
+
+  using namespace boost;
+
+  /**
+   * Helper for launching new threads.
+   */
+  void*
+  run_function0(void* p)
+  {
+    const function0<void>* pf = (const function0<void>*) p;
+    (*pf)();
+    delete pf;
+    return NULL;
+  }
+
+  /**
+   * Run a function in pthread.
+   * \return The new pthread_t on success, 0 on failure.
+    * TODO: Is it safe to treat the pthread_t as an integral type?
+   */
+  pthread_t
+  spawn(const function0<void>& f)
+  {
+    pthread_t t;
+    return pthread_create(&t, NULL, &run_function0, (void*) new function0<void>(f)) == 0 ? t : 0;
+  }
+
+}
+
+#endif


This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.

[Assorted-commits] SF.net SVN: assorted: [359] cpp-commons/trunk/src/commons/deque.h

[<yan...@us...>]

Revision: 359
          http://assorted.svn.sourceforge.net/assorted/?rev=359&view=rev
Author:   yangzhang
Date:     2008-02-10 20:53:49 -0800 (Sun, 10 Feb 2008)

Log Message:
-----------
added beginnings of lightweight deque

Added Paths:
-----------
    cpp-commons/trunk/src/commons/deque.h

Added: cpp-commons/trunk/src/commons/deque.h
===================================================================
--- cpp-commons/trunk/src/commons/deque.h	                        (rev 0)
+++ cpp-commons/trunk/src/commons/deque.h	2008-02-11 04:53:49 UTC (rev 359)
@@ -0,0 +1,39 @@
+#ifndef _COMMONS_DEQUE_H
+#define _COMMONS_DEQUE_H
+
+#include <list>
+#include <vector>
+
+namespace commons
+{
+  using namespace std;
+
+  template <typename T>
+  class deque
+  {
+  private:
+    class chunk
+    {
+    public:
+      chunk() : xs(node_size) {}
+    private:
+      vector<T> xs;
+    };
+
+    list<chunk> chunks;
+    size_t node_size;
+  public:
+    deque(size_t node_size = 8192) : node_size(node_size) {}
+
+    void push_back(const T& x)
+    {
+      chunk& last = chunks.back();
+      if (last.xs.size() == last.xs.capacity()) {
+        chunks.push_back(chunk());
+      }
+      last.push_back(x);
+    }
+  };
+}
+
+#endif


This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.

[Assorted-commits] SF.net SVN: assorted: [358] hash-join/trunk/src/hashjoin.cc

[<yan...@us...>]

Revision: 358
          http://assorted.svn.sourceforge.net/assorted/?rev=358&view=rev
Author:   yangzhang
Date:     2008-02-10 20:52:44 -0800 (Sun, 10 Feb 2008)

Log Message:
-----------
fixed bug omitting subsequent buckets in build/probe

Modified Paths:
--------------
    hash-join/trunk/src/hashjoin.cc

Modified: hash-join/trunk/src/hashjoin.cc
===================================================================
--- hash-join/trunk/src/hashjoin.cc	2008-02-11 04:52:20 UTC (rev 357)
+++ hash-join/trunk/src/hashjoin.cc	2008-02-11 04:52:44 UTC (rev 358)
@@ -13,11 +13,15 @@
 #include <pthread.h>
 
 #include <commons/check.h>
+#include <commons/deque.h>
 #include <commons/files.h>
+#include <commons/hash.h>
 #include <commons/strings.h>
 #include <commons/threads.h>
 #include <commons/time.h>
 
+// TODO: #include <boost/array.hpp>
+
 //
 // Hash Join
 //
@@ -25,6 +29,7 @@
 using namespace std;
 using namespace __gnu_cxx;
 using namespace commons;
+// TODO: using namespace boost;
 
 // TODO use dependency injection!
 unsigned int ncpus = 1;
@@ -51,9 +56,11 @@
   /**
    * The data that we hold.
    */
-  vector<char *> bufs;
+  vector<char*> bufs;
 };
 
+// TODO: typedef list< array<char, bucket_size> > bucket;
+
 /**
  * An abstract in-memory database that holds "tuples" in a contiguous buffer.
  * The format/interpretation of the buffers is up to the subclasses.
@@ -222,7 +229,7 @@
 unsigned int
 db::push_bucket(char **heads, bucket *bs, const char *s, const char *p, size_t nbytes)
 {
-  size_t h = __stl_hash_string(s);
+  size_t h = hash_djb2(s);
   unsigned int bucket = h % (map_size * ncpus) / map_size;
   size_t bucket_size = max(1000000UL,buflen / ncpus * 3);
   if (heads[bucket] + nbytes < bs[bucket].bufs.back() + bucket_size) {
@@ -357,14 +364,19 @@
   hmap &h = *ph;
   // Visit each bucket that's destined to us (visit each source).
   for (unsigned int i = 0; i < ncpus; i++) {
-    char *p = movbucs[i][pid].bufs[0],
-         *end = movbucs[i][pid].bufs[0] + movbucs[i][pid].sz[0];
-    while (p < end) {
-      char *title = p;
-      char *release = strchr(p, '\0') + 1;
-      p = strchr(release, '\0') + 2;
-      // Insert into hash map.
-      h[title] = release;
+    const vector<char*>& bufs = movbucs[i][pid].bufs;
+    const vector<size_t>& sz  = movbucs[i][pid].sz;
+    // Iterate over the bucket.
+    for (unsigned int j = 0; j < bufs.size(); j++) {
+      char *p = bufs[j], *end = bufs[j] + sz[j];
+      // Iterate over the chunk.
+      while (p < end) {
+        char *title = p;
+        char *release = strchr(p, '\0') + 1;
+        p = strchr(release, '\0') + 2;
+        // Insert into hash map.
+        h[title] = release;
+      }
     }
   }
 }
@@ -398,28 +410,32 @@
   int hits = 0, misses = 0;
   // For each source bucket.
   for (unsigned int i = 0; i < ncpus; i++) {
-    char *p = actbucs[i][pid].bufs[0],
-         *end = actbucs[i][pid].bufs[0] + actbucs[i][pid].sz[0];
+    const vector<char*>& bufs = actbucs[i][pid].bufs;
+    const vector<size_t>& sz  = actbucs[i][pid].sz;
     // Iterate over the bucket.
-    while (p < end) {
-      char *name = p;
-      p = strchr(p, '\0') + 1;
-      while (true) {
-        char *title = p;
+    for (unsigned int j = 0; j < bufs.size(); j++) {
+      char *p = bufs[j], *end = bufs[j] + sz[j];
+      // Iterate over the chunk.
+      while (p < end) {
+        char *name = p;
         p = strchr(p, '\0') + 1;
-        // Emit the joined tuple (if a join was possible).
-        if (h.find(title) != h.end()) {
-          hits++;
-          join(title, name);
-        } else {
-          misses++;
+        while (true) {
+          char *title = p;
+          p = strchr(p, '\0') + 1;
+          // Emit the joined tuple (if a join was possible).
+          if (h.find(title) != h.end()) {
+            hits++;
+            join(title, name);
+          } else {
+            misses++;
+          }
+          // End of a tuple? (Don't actually need this check, since the
+          // hash-partitioning "normalizes" the tuples from the actresses file.)
+          if (*p == '\0') {
+            p++;
+            break;
+          }
         }
-        // End of a tuple? (Don't actually need this check, since the
-        // hash-partitioning "normalizes" the tuples from the actresses file.)
-        if (*p == '\0') {
-          p++;
-          break;
-        }
       }
     }
   }


This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.

[Assorted-commits] SF.net SVN: assorted: [357] hash-join/trunk/src/Makefile

[<yan...@us...>]

Revision: 357
          http://assorted.svn.sourceforge.net/assorted/?rev=357&view=rev
Author:   yangzhang
Date:     2008-02-10 20:52:20 -0800 (Sun, 10 Feb 2008)

Log Message:
-----------
tweaks

Modified Paths:
--------------
    hash-join/trunk/src/Makefile

Modified: hash-join/trunk/src/Makefile
===================================================================
--- hash-join/trunk/src/Makefile	2008-02-10 21:48:17 UTC (rev 356)
+++ hash-join/trunk/src/Makefile	2008-02-11 04:52:20 UTC (rev 357)
@@ -5,23 +5,30 @@
 
 CFLAGS := -I. -Wall -lpthread # -lprofiler
 
-CXX     = g++ $(CFLAGS) -o $@ $^
+CXX     = g++ $(CFLAGS) -o $@ $<
 
 all: pg
 dbg: $(TARGET)-dbg
 opt: $(TARGET)-opt
 pg: $(TARGET)-pg
+pg-opt: $(TARGET)-pg-opt
 goo: $(TARGET)-goo
 
 $(TARGET)-pg: $(SRCS)
-	$(CXX) -g -pg
+	$(CXX) -pg
 
+$(TARGET)-pg-opt: $(SRCS)
+	$(CXX) -pg -O3
+
 $(TARGET)-dbg: $(SRCS)
-	$(CXX) -g3 -fno-omit-frame-pointer
+	$(CXX) -g3
 
 $(TARGET)-opt: $(SRCS)
 	$(CXX) -g -O3 -fno-omit-frame-pointer
 
+$(TARGET)-goo: $(SRCS)
+	$(CXX) -lprofile
+
 doc: doc/html/index.html
 
 doc/html/index.html: $(SRCS) Doxyfile


This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.

[Assorted-commits] SF.net SVN: assorted: [356] numa-bench/trunk/src

[<yan...@us...>]

Revision: 356
          http://assorted.svn.sourceforge.net/assorted/?rev=356&view=rev
Author:   yangzhang
Date:     2008-02-10 13:48:17 -0800 (Sun, 10 Feb 2008)

Log Message:
-----------
updates

Modified Paths:
--------------
    numa-bench/trunk/src/Makefile
    numa-bench/trunk/src/malloc.cc

Added Paths:
-----------
    numa-bench/trunk/src/openmp.cc
    numa-bench/trunk/src/threads.cc

Modified: numa-bench/trunk/src/Makefile
===================================================================
--- numa-bench/trunk/src/Makefile	2008-02-10 18:52:12 UTC (rev 355)
+++ numa-bench/trunk/src/Makefile	2008-02-10 21:48:17 UTC (rev 356)
@@ -1,7 +1,8 @@
 COMMONS := $(wildcard commons/*.h)
-CXX = g++ -I. -lnuma -o $@ $^
+CXX = g++-4.2 -I. -lnuma -lpthread -o $@ $^
 
-all: avail cache
+# all: avail cache malloc threads
+all: malloc
 
 avail: avail.cc $(COMMONS)
 	$(CXX)
@@ -9,6 +10,15 @@
 cache: cache.cc $(COMMONS)
 	$(CXX)
 
+malloc: malloc.cc $(COMMONS)
+	$(CXX)
+
+threads: threads.cc $(COMMONS)
+	$(CXX)
+
+openmp: openmp.cc
+	$(CXX) -fopenmp
+
 clean:
 	rm -f avail cache
 

Modified: numa-bench/trunk/src/malloc.cc
===================================================================
--- numa-bench/trunk/src/malloc.cc	2008-02-10 18:52:12 UTC (rev 355)
+++ numa-bench/trunk/src/malloc.cc	2008-02-10 21:48:17 UTC (rev 356)
@@ -2,32 +2,73 @@
 
 #include <cstdlib>
 #include <iostream>
-#include <time.h>
-#include <pthread.h>
 
+#include <sched.h>
+
+#include <commons/check.h>
+#include <commons/threads.h>
+#include <commons/time.h>
+
+using namespace commons;
 using namespace std;
 
 const size_t size = 10000000;
 
-void
-touch(void *pp)
+void*
+chew(void* pp)
 {
-  char *p = (char*) pp;
+  char* p = (char*) pp;
   const int reps = 100;
-  time_t t0 = time(NULL);
+  pid_t pid = gettid();
+  timer t(": ");
+
+  // Pin this thread to the right processor.
+  cpu_set_t cs;
+  CPU_ZERO(&cs);
+  CPU_SET(1, &cs);
+  sched_setaffinity(pid, sizeof(cs), &cs);
+
+  // TODO: try shuffling indexes
   for (int c = 0; c < reps; c++) {
     for (size_t i = 0; i < size; i++) {
       p[i] = i;
     }
   }
-  time_t t1 = time(NULL);
-  cout << t1 - t0 << endl;
+
+  // Print the elapsed time;
+  cout << pid;
+  t.print();
+  return NULL;
 }
 
 int
-main()
+main(int argc, char** argv)
 {
+  if (argc < 2) {
+    cerr << "malloc <nthreads>" << endl;
+    return 1;
+  }
+
+  const int n = atoi(argv[1]);
   void *p = malloc(size);
-  touch(p);
+
+  // warmup
+  chew(p);
+  pthread_t ts[n];
+
+  // start thread on each core
+  for (int i = 0; i < n; i++) {
+    check(pthread_create(&ts[i], NULL, chew, p) == 0);
+  }
+  waitall(ts, n);
   return 0;
+
+  // THRASH
+
+  // spawn workers
+  for (int i = 0; i < n; i++) {
+    check(pthread_create(&ts[i], NULL, chew, p) == 0);
+  }
+  waitall(ts, n);
+  return 0;
 }

Added: numa-bench/trunk/src/openmp.cc
===================================================================
--- numa-bench/trunk/src/openmp.cc	                        (rev 0)
+++ numa-bench/trunk/src/openmp.cc	2008-02-10 21:48:17 UTC (rev 356)
@@ -0,0 +1,7 @@
+#pragma omp
+
+int
+main()
+{
+  return 0;
+}

Added: numa-bench/trunk/src/threads.cc
===================================================================
--- numa-bench/trunk/src/threads.cc	                        (rev 0)
+++ numa-bench/trunk/src/threads.cc	2008-02-10 21:48:17 UTC (rev 356)
@@ -0,0 +1,30 @@
+/**
+ * Demonstrates that each thread has its own unique TID.
+ */
+
+#include <iostream>
+
+#include <pthread.h>
+#include <sys/types.h>
+#include <commons/threads.h>
+
+using namespace std;
+using namespace commons;
+
+void*
+print_tid(void*)
+{
+  pid_t pid = gettid();
+  (pid_t)syscall(__NR_gettid);
+  cout << pid << endl;
+}
+
+int
+main()
+{
+  pthread_t t;
+  pthread_create(&t, NULL, &print_tid, NULL);
+  print_tid(NULL);
+  pthread_join(t, NULL);
+  return 0;
+}


This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.

[Assorted-commits] SF.net SVN: assorted: [355] cpp-commons/trunk/src/commons

[<yan...@us...>]

Revision: 355
          http://assorted.svn.sourceforge.net/assorted/?rev=355&view=rev
Author:   yangzhang
Date:     2008-02-10 10:52:12 -0800 (Sun, 10 Feb 2008)

Log Message:
-----------
forked x86asm off cpuid; fixed includes

Modified Paths:
--------------
    cpp-commons/trunk/src/commons/cppcommons.cpp
    cpp-commons/trunk/src/commons/cpuid.h
    cpp-commons/trunk/src/commons/files.h
    cpp-commons/trunk/src/commons/hash.h
    cpp-commons/trunk/src/commons/strings.h

Added Paths:
-----------
    cpp-commons/trunk/src/commons/x86asm.h

Modified: cpp-commons/trunk/src/commons/cppcommons.cpp
===================================================================
--- cpp-commons/trunk/src/commons/cppcommons.cpp	2008-02-10 18:45:24 UTC (rev 354)
+++ cpp-commons/trunk/src/commons/cppcommons.cpp	2008-02-10 18:52:12 UTC (rev 355)
@@ -23,11 +23,12 @@
 #include <config.h>
 #endif
 
-#include "commons/check.h"
-#include "commons/cpuid.h"
-#include "commons/files.h"
-#include "commons/hash.h"
-#include "commons/strings.h"
-#include "commons/time.h"
-#include "commons/threads.h"
+#include <commons/check.h>
+#include <commons/cpuid.h>
+#include <commons/files.h>
+#include <commons/hash.h>
+#include <commons/strings.h>
+#include <commons/time.h>
+#include <commons/threads.h>
+#include <commons/x86asm.h>
 

Modified: cpp-commons/trunk/src/commons/cpuid.h
===================================================================
--- cpp-commons/trunk/src/commons/cpuid.h	2008-02-10 18:45:24 UTC (rev 354)
+++ cpp-commons/trunk/src/commons/cpuid.h	2008-02-10 18:52:12 UTC (rev 355)
@@ -6,6 +6,8 @@
 #ifndef _COMMONS_CPUID_H
 #define _COMMONS_CPUID_H
 
+#include <commons/x86asm.h>
+
 namespace commons
 {
 
@@ -19,26 +21,6 @@
           "=a" (ax), "=b" (bx), "=c" (cx), "=d" (dx) : "a" (func));
 
   /**
-   * Given an extended general-purpose register (e.g. EAX), extract the high
-   * 8-bit register (AH).
-   */
-  inline unsigned char
-  high(unsigned int r)
-  {
-    return ((r >> 8) & 0xffU);
-  }
-
-  /**
-   * Given an extended general-purpose register (e.g. EAX), extract the low
-   * 8-bit register (AL).
-   */
-  inline unsigned char
-  low(unsigned int r)
-  {
-    return (r & 0xffU);
-  }
-
-  /**
    * Get cache line size in bytes on an Intel CPU.
    * References:
    * http://softpixel.com/~cwright/programming/simd/cpuid.php

Modified: cpp-commons/trunk/src/commons/files.h
===================================================================
--- cpp-commons/trunk/src/commons/files.h	2008-02-10 18:45:24 UTC (rev 354)
+++ cpp-commons/trunk/src/commons/files.h	2008-02-10 18:52:12 UTC (rev 355)
@@ -12,7 +12,7 @@
 #include <unistd.h>
 #include <fcntl.h>
 
-#include "commons/check.h"
+#include <commons/check.h>
 
 namespace commons
 {

Modified: cpp-commons/trunk/src/commons/hash.h
===================================================================
--- cpp-commons/trunk/src/commons/hash.h	2008-02-10 18:45:24 UTC (rev 354)
+++ cpp-commons/trunk/src/commons/hash.h	2008-02-10 18:52:12 UTC (rev 355)
@@ -36,7 +36,7 @@
   {
     unsigned long h = 0;
     for (; *s; ++s)
-      h = 31 * h + x;
+      h = 31 * h + *s;
     return size_t(h);
   }
 

Modified: cpp-commons/trunk/src/commons/strings.h
===================================================================
--- cpp-commons/trunk/src/commons/strings.h	2008-02-10 18:45:24 UTC (rev 354)
+++ cpp-commons/trunk/src/commons/strings.h	2008-02-10 18:52:12 UTC (rev 355)
@@ -5,7 +5,7 @@
 
 #include <strings.h>
 
-#include "commons/check.h"
+#include <commons/check.h>
 
 namespace commons
 {

Added: cpp-commons/trunk/src/commons/x86asm.h
===================================================================
--- cpp-commons/trunk/src/commons/x86asm.h	                        (rev 0)
+++ cpp-commons/trunk/src/commons/x86asm.h	2008-02-10 18:52:12 UTC (rev 355)
@@ -0,0 +1,29 @@
+#ifndef _COMMONS_X86ASM_H
+#define _COMMONS_X86ASM_H
+
+namespace commons
+{
+
+  /**
+   * Given an extended general-purpose register (e.g. EAX), extract the high
+   * 8-bit register (AH).
+   */
+  inline unsigned char
+  high(unsigned int r)
+  {
+    return ((r >> 8) & 0xffU);
+  }
+
+  /**
+   * Given an extended general-purpose register (e.g. EAX), extract the low
+   * 8-bit register (AL).
+   */
+  inline unsigned char
+  low(unsigned int r)
+  {
+    return (r & 0xffU);
+  }
+
+}
+
+#endif


This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.

[Assorted-commits] SF.net SVN: assorted: [354] scala-commons/trunk/src/commons/Hash.scala

[<yan...@us...>]

Revision: 354
          http://assorted.svn.sourceforge.net/assorted/?rev=354&view=rev
Author:   yangzhang
Date:     2008-02-10 10:45:24 -0800 (Sun, 10 Feb 2008)

Log Message:
-----------
added hash functions

Added Paths:
-----------
    scala-commons/trunk/src/commons/Hash.scala

Added: scala-commons/trunk/src/commons/Hash.scala
===================================================================
--- scala-commons/trunk/src/commons/Hash.scala	                        (rev 0)
+++ scala-commons/trunk/src/commons/Hash.scala	2008-02-10 18:45:24 UTC (rev 354)
@@ -0,0 +1,33 @@
+package commons
+
+object Hash {
+
+  /**
+   * From libstdc++ 4.1 __stl_hash_string.
+   */
+  def hashStl(xs: Seq[Int]) = {
+    var h = 0
+    for (x <- xs) h = 5 * h + x
+    h
+  }
+
+  /**
+   * From Sun JDK6 String.hashCode.
+   */
+  def hashJava(xs: Seq[Int]) = {
+    var h = 0
+    for (x <- xs) h = 31 * h + x
+    h
+  }
+
+  /**
+   * From http://www.cse.yorku.ca/~oz/hash.html. Not sure if this is correct,
+   * since Int is signed.
+   */
+  def hashDjb2(xs: Seq[Int]) = {
+    var h = 5381
+    for (x <- xs) h = ((h << 5) + h) + x
+    h
+  }
+
+}


This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.

[Assorted-commits] SF.net SVN: assorted: [353] cpp-commons/trunk/src/commons

[<yan...@us...>]

Revision: 353
          http://assorted.svn.sourceforge.net/assorted/?rev=353&view=rev
Author:   yangzhang
Date:     2008-02-10 10:44:43 -0800 (Sun, 10 Feb 2008)

Log Message:
-----------
added hash functions

Modified Paths:
--------------
    cpp-commons/trunk/src/commons/cppcommons.cpp

Added Paths:
-----------
    cpp-commons/trunk/src/commons/hash.h

Modified: cpp-commons/trunk/src/commons/cppcommons.cpp
===================================================================
--- cpp-commons/trunk/src/commons/cppcommons.cpp	2008-02-10 18:37:27 UTC (rev 352)
+++ cpp-commons/trunk/src/commons/cppcommons.cpp	2008-02-10 18:44:43 UTC (rev 353)
@@ -26,6 +26,7 @@
 #include "commons/check.h"
 #include "commons/cpuid.h"
 #include "commons/files.h"
+#include "commons/hash.h"
 #include "commons/strings.h"
 #include "commons/time.h"
 #include "commons/threads.h"

Added: cpp-commons/trunk/src/commons/hash.h
===================================================================
--- cpp-commons/trunk/src/commons/hash.h	                        (rev 0)
+++ cpp-commons/trunk/src/commons/hash.h	2008-02-10 18:44:43 UTC (rev 353)
@@ -0,0 +1,45 @@
+#ifndef _COMMONS_HASH_H
+#define _COMMONS_HASH_H
+
+namespace commons
+{
+
+  /**
+   * From libstdc++ 4.1 __stl_hash_string.
+   */
+  inline size_t
+  hash_stl(const char* s)
+  {
+    unsigned long h = 0;
+    for ( ; *s; ++s)
+      h = 5 * h + *s;
+    return size_t(h);
+  }
+
+  /**
+   * From http://www.cse.yorku.ca/~oz/hash.html.
+   */
+  inline size_t
+  hash_djb2(const char* s)
+  {
+    unsigned long h = 5381;
+    for (; *s; ++s)
+      h = ((h << 5) + h) + *s;
+    return size_t(h);
+  }
+
+  /**
+   * From Sun JDK6 String.hashCode.
+   */
+  inline size_t
+  hash_java(const char* s)
+  {
+    unsigned long h = 0;
+    for (; *s; ++s)
+      h = 31 * h + x;
+    return size_t(h);
+  }
+
+}
+
+#endif


This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.