assorted-commits Mailing List for Assorted projects (Page 24)
Brought to you by:
yangzhang
Menu
▾
▴
assorted-commits
You can subscribe to this list here.
| 2007 |
Jan
|
Feb
|
Mar
|
Apr
|
May
|
Jun
|
Jul
|
Aug
|
Sep
|
Oct
|
Nov
(9) |
Dec
(12) |
|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 2008 |
Jan
(86) |
Feb
(265) |
Mar
(96) |
Apr
(47) |
May
(136) |
Jun
(28) |
Jul
(57) |
Aug
(42) |
Sep
(20) |
Oct
(67) |
Nov
(37) |
Dec
(34) |
| 2009 |
Jan
(39) |
Feb
(85) |
Mar
(96) |
Apr
(24) |
May
(82) |
Jun
(13) |
Jul
(10) |
Aug
(8) |
Sep
(2) |
Oct
(20) |
Nov
(31) |
Dec
(17) |
| 2010 |
Jan
(16) |
Feb
(11) |
Mar
(17) |
Apr
(53) |
May
(31) |
Jun
(13) |
Jul
(3) |
Aug
(6) |
Sep
(11) |
Oct
(4) |
Nov
(17) |
Dec
(17) |
| 2011 |
Jan
(3) |
Feb
(19) |
Mar
(5) |
Apr
(17) |
May
(3) |
Jun
(4) |
Jul
(14) |
Aug
(3) |
Sep
(2) |
Oct
(1) |
Nov
(3) |
Dec
(2) |
| 2012 |
Jan
(3) |
Feb
(7) |
Mar
(1) |
Apr
|
May
(1) |
Jun
|
Jul
(4) |
Aug
(5) |
Sep
(2) |
Oct
(3) |
Nov
|
Dec
|
| 2013 |
Jan
|
Feb
|
Mar
(9) |
Apr
(5) |
May
|
Jun
(2) |
Jul
(1) |
Aug
(10) |
Sep
(1) |
Oct
(2) |
Nov
|
Dec
|
| 2014 |
Jan
(1) |
Feb
(3) |
Mar
(3) |
Apr
(1) |
May
(4) |
Jun
|
Jul
|
Aug
|
Sep
(2) |
Oct
|
Nov
|
Dec
|
| 2015 |
Jan
|
Feb
|
Mar
|
Apr
|
May
|
Jun
|
Jul
|
Aug
|
Sep
|
Oct
(1) |
Nov
|
Dec
|
| 2016 |
Jan
(1) |
Feb
|
Mar
(2) |
Apr
|
May
|
Jun
|
Jul
|
Aug
|
Sep
(1) |
Oct
|
Nov
|
Dec
|
| 2017 |
Jan
|
Feb
|
Mar
(1) |
Apr
|
May
(5) |
Jun
(1) |
Jul
|
Aug
|
Sep
|
Oct
|
Nov
|
Dec
(2) |
| 2018 |
Jan
|
Feb
|
Mar
|
Apr
|
May
|
Jun
(1) |
Jul
|
Aug
|
Sep
|
Oct
|
Nov
|
Dec
|
|
From: <yan...@us...> - 2009-03-24 00:10:25
|
Revision: 1327
http://assorted.svn.sourceforge.net/assorted/?rev=1327&view=rev
Author: yangzhang
Date: 2009-03-24 00:10:13 +0000 (Tue, 24 Mar 2009)
Log Message:
-----------
- Makefile updates:
- using bash as shell
- using TARGET_ARCH appropriately
- using mkdeps.py for automatic dependency generation
Modified Paths:
--------------
ydb/trunk/src/Makefile
Added Paths:
-----------
ydb/trunk/src/mkdeps.py
Modified: ydb/trunk/src/Makefile
===================================================================
--- ydb/trunk/src/Makefile 2009-03-23 04:50:10 UTC (rev 1326)
+++ ydb/trunk/src/Makefile 2009-03-24 00:10:13 UTC (rev 1327)
@@ -2,14 +2,16 @@
# Tool configurations
#
+SHELL := bash
WTF := wtf
-# CXX := $(WTF) ag++ -k --Xcompiler # $(CXX)
ORIGCXX := $(CXX)
CCACHE := ccache
export CCACHE_PREFIX := distcc
CXX := $(WTF) $(CCACHE) $(CXX) -pipe
-# CC := $(CXX) # for linking
+TARGET_ARCH := $(shell [[ "$$(uname -m)" == x86_64 ]] && echo -m64 || echo -m32 ) \
+ -march=$(shell gcc-config march)
+
WARNINGS = \
-Wall \
-Werror \
@@ -70,13 +72,8 @@
PPROF := -lprofiler
endif
-ARCH := $(shell gcc-config march)
-CXXFLAGS0 = $(OPT) -pthread $(GPROF) \
- $(WARNINGS) \
- -std=gnu++0x \
- -m64 \
- -march=$(ARCH) \
- $(ORIGCXXFLAGS)
+CXXFLAGS0 = $(OPT) -MD -pthread $(GPROF) $(WARNINGS) -std=gnu++0x \
+ $(ORIGCXXFLAGS)
ifneq ($(PCH),)
CXXFLAGS = $(CXXFLAGS0) -include pch.h
@@ -171,11 +168,8 @@
# Project-specific rules
#
-stxn.o: main.hh $(PBHDRS)
-main.o: util.hh msg.h $(PBHDRS)
-util.o: msg.h $(PBHDRS)
-ydb.o: main.hh stxn.hh tpcc.hh util.hh $(PBHDRS)
-tpcc.o: main.hh util.hh $(PBHDRS)
+include $(shell ./mkdeps.py > deps.mk; echo deps.mk)
+
ydb: $(OBJS)
tpcc/%.o: WARNINGS = \
@@ -201,4 +195,5 @@
serperf: ydb.pb.o
ser: ydb.pb.o
-ser.o: msg.h
+
+-include *.d
Added: ydb/trunk/src/mkdeps.py
===================================================================
--- ydb/trunk/src/mkdeps.py (rev 0)
+++ ydb/trunk/src/mkdeps.py 2009-03-24 00:10:13 UTC (rev 1327)
@@ -0,0 +1,50 @@
+#!/usr/bin/env python
+
+from __future__ import with_statement
+from subprocess import *
+from re import *
+from path import path
+
+pwd = path('.')
+
+def memoized(f):
+ cache = {}
+ return lambda *args: cache[args] if args in cache else cache.setdefault(args, f(*args))
+
+def settify(f): return lambda *args: set(f(*args))
+
+@memoized
+@settify
+def hdrs(i):
+ with file(i) as f:
+ for line in f:
+ if search(r'# *include +"', line):
+ yield i.dirname() / sub(r'.*"(.*)".*', r'\1', line.strip())
+
+@memoized
+def src(i):
+ if i.endswith('.hh'):
+ clamp = path(i[:-3] + '.lzz.clamp')
+ lzz = path(i[:-2] + '.lzz')
+ if clamp.isfile(): return clamp
+ if lzz.isfile(): return lzz
+ return i
+
+@memoized
+@settify
+def deps(i):
+ for hdr in hdrs(i): yield hdr
+ for hdr in hdrs(i):
+ if src(hdr).isfile():
+ for dep in deps(src(hdr)):
+ yield dep
+
+for i in pwd.glob('*.lzz') + pwd.glob('*.lzz.clamp'):
+ print sub(r'\.lzz(\.clamp)?', '.o', i), ':', ' '.join(deps(i))
+
+for i in pwd.glob('*.d'):
+ with file(i) as f:
+ for line in f:
+ for word in line.split():
+ if '.clamp' in word:
+ print sub(r'(\.clamp/(.+)_lambda_.+\.clamp_h)', r'\1: \2.lzz.clamp', word)
Property changes on: ydb/trunk/src/mkdeps.py
___________________________________________________________________
Added: svn:executable
+ *
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <yan...@us...> - 2009-03-23 04:50:15
|
Revision: 1326
http://assorted.svn.sourceforge.net/assorted/?rev=1326&view=rev
Author: yangzhang
Date: 2009-03-23 04:50:10 +0000 (Mon, 23 Mar 2009)
Log Message:
-----------
added demo of auto deps
Added Paths:
-----------
sandbox/trunk/src/make/deps/
sandbox/trunk/src/make/deps/Makefile
sandbox/trunk/src/make/deps/README
sandbox/trunk/src/make/deps/aux.h
sandbox/trunk/src/make/deps/main.c
sandbox/trunk/src/make/deps/main.h
Added: sandbox/trunk/src/make/deps/Makefile
===================================================================
--- sandbox/trunk/src/make/deps/Makefile (rev 0)
+++ sandbox/trunk/src/make/deps/Makefile 2009-03-23 04:50:10 UTC (rev 1326)
@@ -0,0 +1,6 @@
+-include *.d
+all: main
+main: main.o
+clean:
+ rm -f main *.d
+CFLAGS = -MD
Added: sandbox/trunk/src/make/deps/README
===================================================================
--- sandbox/trunk/src/make/deps/README (rev 0)
+++ sandbox/trunk/src/make/deps/README 2009-03-23 04:50:10 UTC (rev 1326)
@@ -0,0 +1 @@
+Simple demo of how to use auto deps with make/gcc
Added: sandbox/trunk/src/make/deps/main.c
===================================================================
--- sandbox/trunk/src/make/deps/main.c (rev 0)
+++ sandbox/trunk/src/make/deps/main.c 2009-03-23 04:50:10 UTC (rev 1326)
@@ -0,0 +1,3 @@
+#include "main.h"
+#include "aux.h"
+int main() { return 0; }
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <yan...@us...> - 2009-03-23 04:36:24
|
Revision: 1325
http://assorted.svn.sourceforge.net/assorted/?rev=1325&view=rev
Author: yangzhang
Date: 2009-03-23 04:36:19 +0000 (Mon, 23 Mar 2009)
Log Message:
-----------
arch is determined just once, on leader
Modified Paths:
--------------
ydb/trunk/src/Makefile
Modified: ydb/trunk/src/Makefile
===================================================================
--- ydb/trunk/src/Makefile 2009-03-23 04:25:32 UTC (rev 1324)
+++ ydb/trunk/src/Makefile 2009-03-23 04:36:19 UTC (rev 1325)
@@ -70,11 +70,12 @@
PPROF := -lprofiler
endif
+ARCH := $(shell gcc-config march)
CXXFLAGS0 = $(OPT) -pthread $(GPROF) \
$(WARNINGS) \
-std=gnu++0x \
-m64 \
- -march=$(shell ./gcc-config march) \
+ -march=$(ARCH) \
$(ORIGCXXFLAGS)
ifneq ($(PCH),)
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <yan...@us...> - 2009-03-23 04:25:43
|
Revision: 1324
http://assorted.svn.sourceforge.net/assorted/?rev=1324&view=rev
Author: yangzhang
Date: 2009-03-23 04:25:32 +0000 (Mon, 23 Mar 2009)
Log Message:
-----------
updated clamp req
Modified Paths:
--------------
ydb/trunk/README
Modified: ydb/trunk/README
===================================================================
--- ydb/trunk/README 2009-03-23 03:54:33 UTC (rev 1323)
+++ ydb/trunk/README 2009-03-23 04:25:32 UTC (rev 1324)
@@ -28,7 +28,7 @@
- [boost] 1.37
- [C++ Commons] svn r1082
-- [clamp] 153
+- [clamp] 2.0
- [GCC] 4.3.2
- [google-sparsehash] 1.4
- [googletest] 1.2.1
@@ -38,7 +38,7 @@
[boost]: http://www.boost.org/
[C++ Commons]: http://assorted.sourceforge.net/cpp-commons/
-[clamp]: http://home.clara.net/raoulgough/clamp/
+[clamp]: http://assorted.sf.net/clamp/
[GCC]: http://gcc.gnu.org/
[google-sparsehash]: http://code.google.com/p/google-sparsehash/
[googletest]: http://code.google.com/p/googletest/
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <yan...@us...> - 2009-03-23 03:54:39
|
Revision: 1323
http://assorted.svn.sourceforge.net/assorted/?rev=1323&view=rev
Author: yangzhang
Date: 2009-03-23 03:54:33 +0000 (Mon, 23 Mar 2009)
Log Message:
-----------
- distcc now works perfectly
- reorganized the Makefile and removed recursive make
- general cleanup and tweaks throughout the Makefile
Modified Paths:
--------------
ydb/trunk/src/Makefile
Modified: ydb/trunk/src/Makefile
===================================================================
--- ydb/trunk/src/Makefile 2009-03-22 09:08:37 UTC (rev 1322)
+++ ydb/trunk/src/Makefile 2009-03-23 03:54:33 UTC (rev 1323)
@@ -1,61 +1,16 @@
-TARGET := ydb
+#
+# Tool configurations
+#
+
WTF := wtf
+# CXX := $(WTF) ag++ -k --Xcompiler # $(CXX)
+ORIGCXX := $(CXX)
CCACHE := ccache
-DISTCC := distcc
-CCACHE_PREFIX := $(DISTCC)
-ifeq ($(CCACHE),)
-ACCEL := $(DISTCC)
-else
-ACCEL := $(CCACHE)
-endif
+export CCACHE_PREFIX := distcc
+CXX := $(WTF) $(CCACHE) $(CXX) -pipe
+# CC := $(CXX) # for linking
-CLAMPS := $(wildcard *.lzz.clamp)
-CLAMPLZZS:= $(patsubst %.clamp,%,$(CLAMPS))
-PURELZZS := $(foreach lzz,$(wildcard *.lzz),$(if $(wildcard $(lzz).clamp),,$(lzz)))
-LZZS := $(CLAMPLZZS) $(PURELZZS)
-LZZHDRS := $(foreach lzz,$(LZZS),$(patsubst %.lzz,%.hh,$(lzz)))
-LZZSRCS := $(foreach lzz,$(LZZS),$(patsubst %.lzz,%.cc,$(lzz)))
-LZZOBJS := $(foreach lzz,$(LZZS),$(patsubst %.lzz,%.o,$(lzz)))
-
-PBS := $(wildcard *.proto)
-PBHDRS := $(foreach pb,$(PBS),$(patsubst %.proto,%.pb.h,$(pb)))
-PBSRCS := $(foreach pb,$(PBS),$(patsubst %.proto,%.pb.cc,$(pb)))
-PBOBJS := $(foreach pb,$(PBS),$(patsubst %.proto,%.pb.o,$(pb)))
-
-GENHDRS := $(LZZHDRS) $(PBHDRS)
-GENSRCS := $(LZZSRCS) $(PBSRCS)
-GENOBJS := $(LZZOBJS) $(PBOBJS)
-
-TPCC_OBJS:= clock randomgenerator tpccclient tpccdb tpccgenerator tpcctables
-TPCC_OBJS:= $(foreach i,$(TPCC_OBJS),tpcc/$(i).o)
-
-HDRS := $(GENHDRS)
-SRCS := $(GENSRCS)
-OBJS := $(GENOBJS) $(TPCC_OBJS)
-
-ifneq ($(GPROF),)
- GPROF := -pg
-endif
-ifneq ($(GCOV),)
- GCOV := -fprofile-arcs -ftest-coverage
-endif
-ifneq ($(PPROF),)
- PPROF := -lprofiler
-endif
-ifneq ($(OPT),)
- OPT := -g3 -O3 -Wdisabled-optimization -DNDEBUG
-else
- OPT := -g3
-endif
-# CXX := $(WTF) ag++ -k --Xcompiler # $(CXX)
-CXX := $(WTF) $(ACCEL) $(CXX) -pipe
-CC := $(CXX) # for linking
-LDFLAGS := -pthread $(GPROF)
-LDLIBS := -lstx -lst -lresolv -lprotobuf -lgtest \
- -lboost_program_options-gcc43-mt -lboost_thread-gcc43-mt \
- -lboost_serialization-gcc43-mt $(PPROF)
-
-CXXFLAGS0 := $(OPT) -pthread $(GPROF) \
+WARNINGS = \
-Wall \
-Werror \
-Wextra \
@@ -84,11 +39,7 @@
-Wredundant-decls \
-Winvalid-pch \
-Wlong-long \
- -Wvolatile-register-var \
- -std=gnu++0x \
- -m64 \
- -march=$(shell gcc-config march) \
- $(CXXFLAGS)
+ -Wvolatile-register-var
# \
-Wmissing-noreturn \
@@ -102,54 +53,111 @@
-Wstrict-overflow \
-Winline \
+ORIGCXXFLAGS := $(CXXFLAGS)
+
+ifneq ($(GCOV),)
+ GCOV := -fprofile-arcs -ftest-coverage
+endif
+ifneq ($(OPT),)
+ OPT := -g3 -O3 -Wdisabled-optimization -DNDEBUG
+else
+ OPT := -g3
+endif
+ifneq ($(GPROF),)
+ GPROF := -pg
+endif
+ifneq ($(PPROF),)
+ PPROF := -lprofiler
+endif
+
+CXXFLAGS0 = $(OPT) -pthread $(GPROF) \
+ $(WARNINGS) \
+ -std=gnu++0x \
+ -m64 \
+ -march=$(shell ./gcc-config march) \
+ $(ORIGCXXFLAGS)
+
ifneq ($(PCH),)
- CXXFLAGS := $(CXXFLAGS0) -include pch.h
+ CXXFLAGS = $(CXXFLAGS0) -include pch.h
$(LZZOBJS): pch.h.gch
else
- CXXFLAGS := $(CXXFLAGS0)
+ CXXFLAGS = $(CXXFLAGS0)
endif
-PBCXXFLAGS := $(OPT) -Wall -Werror $(GPROF)
+LDFLAGS := -pthread $(GPROF) $(LDFLAGS)
-all: $(TARGET)
+LDLIBS := -lstx -lst -lresolv -lprotobuf -lgtest \
+ -lboost_program_options-gcc43-mt -lboost_thread-gcc43-mt \
+ -lboost_serialization-gcc43-mt $(PPROF)
-%.pb.o: %.pb.cc %.pb.h
- $(CXX) -c $(PBCXXFLAGS) $(OUTPUT_OPTION) $<
+.PHONY: clean
-stxn.o: main.hh $(PBHDRS)
-main.o: util.hh msg.h $(PBHDRS)
-util.o: msg.h $(PBHDRS)
-ydb.o: main.hh stxn.hh tpcc.hh util.hh $(PBHDRS)
-tpcc.o: main.hh util.hh $(PBHDRS)
-ydb: $(OBJS)
+#
+# Project sources
+#
-tpcc/%.o: tpcc/%.cc
- make -C tpcc/
+SVNURL := https://assorted.svn.sourceforge.net/svnroot/assorted/ydb/trunk/src
-tpcc/%.cc: tpcc/%.cc.cog
- make -C tpcc/
+TARGET := ydb
-.SECONDARY: tpcc/tpcctables.cc tpcc/tpcctables.o
+CLAMPS := $(wildcard *.lzz.clamp)
+CLAMPLZZS:= $(patsubst %.clamp,%,$(CLAMPS))
+PURELZZS := $(foreach lzz,$(wildcard *.lzz),$(if $(wildcard $(lzz).clamp),,$(lzz)))
+LZZS := $(CLAMPLZZS) $(PURELZZS)
+LZZHDRS := $(foreach lzz,$(LZZS),$(patsubst %.lzz,%.hh,$(lzz)))
+LZZSRCS := $(foreach lzz,$(LZZS),$(patsubst %.lzz,%.cc,$(lzz)))
+LZZOBJS := $(foreach lzz,$(LZZS),$(patsubst %.lzz,%.o,$(lzz)))
+PBS := $(wildcard *.proto)
+PBHDRS := $(foreach pb,$(PBS),$(patsubst %.proto,%.pb.h,$(pb)))
+PBSRCS := $(foreach pb,$(PBS),$(patsubst %.proto,%.pb.cc,$(pb)))
+PBOBJS := $(foreach pb,$(PBS),$(patsubst %.proto,%.pb.o,$(pb)))
+
+COGS := $(wildcard tpcc/*.cog)
+COGSRCS := $(foreach cog,$(COGS),$(patsubst %.cog,%,$(cog)))
+
+GENHDRS := $(LZZHDRS) $(PBHDRS) $(COGHDRS)
+GENSRCS := $(LZZSRCS) $(PBSRCS) $(COGSRCS)
+GENOBJS := $(LZZOBJS) $(PBOBJS) $(COGOBJS)
+
+TPCC_OBJS:= clock randomgenerator tpccclient tpccdb tpccgenerator tpcctables
+TPCC_OBJS:= $(foreach i,$(TPCC_OBJS),tpcc/$(i).o)
+
+HDRS := $(GENHDRS)
+SRCS := $(GENSRCS)
+OBJS := $(GENOBJS) $(TPCC_OBJS)
+
+#
+# Rules
+#
+
+all: $(TARGET)
+
+doc: $(SRCS) $(HDRS)
+ doxygen
+
+%.pb.o: WARNINGS = -Wall -Werror
+%.pb.o: %.pb.cc %.pb.h
+
+%.cc: %.cc.cog
+ cog.py $< > $@
+
%.cc %.hh: %.lzz
lzz -hx hh -sx cc -hl -sl -hd -sd $<
-%.pb.cc: %.proto
+%.pb.h %.pb.cc: %.proto
protoc --cpp_out=. $<
-%.pb.h: %.proto
- protoc --cpp_out=. $<
-
%.lzz: %.lzz.clamp
rm -f $@
- mkdir -p clamp/
- clamp --outdir clamp/ --prefix $(basename $@) < $< | \
+ mkdir -p .clamp/
+ clamp --outdir .clamp/ --prefix $(basename $@) < $< | \
sed "$$( echo -e '1i\\\n\#hdr\n1a\\\n\#end' )" | \
sed "$$( echo -e '$$i\\\n\#hdr\n$$a\\\n\#end' )" > $@
chmod -w $@
pch.h:
- svn ls -rHEAD -R https://assorted.svn.sourceforge.net/svnroot/assorted/ydb/trunk/src | \
+ svn ls -rHEAD -R $(SVNURL) | \
egrep -v '/$$|Makefile' | \
xargs sed 's/.*\binclude\b *<\(.*\)>.*/\#include <\1>/; t succ; d; :succ /commons/ d' | \
sort -u > $@
@@ -158,23 +166,38 @@
%.h.gch: %.h
$(LINK.cc) $(OUTPUT_OPTION) $<
+#
+# Project-specific rules
+#
+
+stxn.o: main.hh $(PBHDRS)
+main.o: util.hh msg.h $(PBHDRS)
+util.o: msg.h $(PBHDRS)
+ydb.o: main.hh stxn.hh tpcc.hh util.hh $(PBHDRS)
+tpcc.o: main.hh util.hh $(PBHDRS)
+ydb: $(OBJS)
+
+tpcc/%.o: WARNINGS = \
+ -Werror \
+ -Wall \
+ -Wextra \
+ -Wconversion \
+ -Wpointer-arith \
+ -Wcast-qual \
+ -Wcast-align \
+ -Wwrite-strings \
+ -Woverloaded-virtual \
+ -Wno-sign-compare \
+ -Wno-unused-parameter
+
clean:
- rm -rf clamp/ $(GENSRCS) $(GENHDRS) $(OBJS) $(TARGET) \
- $(CLAMPLZZS) $(LZZHDRS) $(LZZSRCS)
- make -C tpcc/ clean
+ rm -rf .clamp/ $(GENSRCS) $(GENHDRS) $(OBJS) $(TARGET) $(CLAMPLZZS)
distclean: clean
rm -f pch.h pch.h.gch
-doc: $(SRCS) $(HDRS)
- doxygen
+.SECONDARY: $(GENSRCS) $(GENHDRS) $(OBJS) main.lzz pch.h.gch
-.PHONY: clean
-
-.SECONDARY: $(SRCS) $(HDRS) $(OBJS) main.lzz pch.h.gch
-
-###
-
serperf: ydb.pb.o
ser: ydb.pb.o
ser.o: msg.h
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <yan...@us...> - 2009-03-22 09:08:46
|
Revision: 1322
http://assorted.svn.sourceforge.net/assorted/?rev=1322&view=rev
Author: yangzhang
Date: 2009-03-22 09:08:37 +0000 (Sun, 22 Mar 2009)
Log Message:
-----------
- added user setting
- made locking functions tolerate missing common.bash
- using new custom clamp
- general clean-up/tweaks
- added node-setup-shell-tools()
Modified Paths:
--------------
ydb/trunk/tools/test.bash
Modified: ydb/trunk/tools/test.bash
===================================================================
--- ydb/trunk/tools/test.bash 2009-03-22 09:06:24 UTC (rev 1321)
+++ ydb/trunk/tools/test.bash 2009-03-22 09:08:37 UTC (rev 1322)
@@ -1,7 +1,7 @@
#!/usr/bin/env bash
set -o errexit -o nounset
-if [[ "$1" != node-init-setup ]]
+if [[ "$1" != node-init-setup && "$1" != node-islocked && "$1" != node-lock && "$1" != node-unlock ]]
then . common.bash || exit 1
fi
@@ -21,6 +21,12 @@
then hosts="$( seq $range | sed 's/^/farm/' )"
fi
hosts="$( echo -n $hosts )"
+ if [[ "${user:-}" ]] ; then
+ for host in $hosts
+ do rhosts="${rhosts:-} $user@$host"
+ done
+ hosts="$rhosts"
+ fi
fi
}
@@ -66,7 +72,7 @@
remote() {
local host="$1"
shift
- scp -q "$(dirname "$0")/$script" "$host:"
+ scp -q "$0" "$host:"
tagssh "$host" "remote=1 ./$script" "$@"
}
@@ -84,7 +90,7 @@
parremote() {
export hosts range
- parhosts "./$script" remote ^ "$@"
+ parhosts "$0" remote ^ "$@"
}
hostargs() {
@@ -108,14 +114,14 @@
node-unlock() {
check-remote
sudo su - -c '
- sed -i "/^AllowUsers root yang$/ d" /etc/ssh/sshd_config &&
+ sed -i "/^AllowUsers/ d" /etc/ssh/sshd_config &&
/etc/init.d/ssh restart
'
}
node-islocked() {
check-remote
- sudo grep '^AllowUsers root yang$' /etc/ssh/sshd_config
+ sudo grep '^AllowUsers' /etc/ssh/sshd_config
}
lock() { parremote node-lock ; }
@@ -191,17 +197,21 @@
node-setup-clamp() {
check-remote
- cd /tmp/
- tar xzf clamp_053_src.tar.gz
- cd clamp_053/
- chmod u+w *
- patch -p1 < /tmp/clamp.patch
- make -s clamp
+ svn up -q ~/work/assorted/clamp/
+ make -sC ~/work/assorted/clamp/trunk/src/ clean all
mkdir -p ~/.local/pkg/clamp/bin/
- mv clamp ~/.local/pkg/clamp/bin/
+ ln -sf ~/work/assorted/clamp/trunk/src/clamp ~/.local/pkg/clamp/bin/
refresh-local
}
+node-setup-shell-tools() {
+ check-remote
+ svn up -q ~/work/assorted/
+ cd ~/work/assorted/shell-tools/trunk/
+ ./setup.bash -d -p ~/.local/pkg/shell-tools/
+ refresh-local
+}
+
node-setup-gtest() {
check-remote
toast --quiet arm googletest
@@ -219,7 +229,7 @@
refresh-local
cd ~/ydb/src
make clean
- PPROF=1 OPT=1 make WTF= ydb
+ PPROF=1 OPT=1 make WTF= ydb DISTCC=
}
node-setup-cog() {
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <yan...@us...> - 2009-03-22 09:06:41
|
Revision: 1321
http://assorted.svn.sourceforge.net/assorted/?rev=1321&view=rev
Author: yangzhang
Date: 2009-03-22 09:06:24 +0000 (Sun, 22 Mar 2009)
Log Message:
-----------
- cleaned up build, fixed distcc arch issues
- restored unsetprefs.h; somehow that got clobbered
Modified Paths:
--------------
ydb/trunk/src/Makefile
ydb/trunk/src/unsetprefs.h
Modified: ydb/trunk/src/Makefile
===================================================================
--- ydb/trunk/src/Makefile 2009-03-21 23:23:39 UTC (rev 1320)
+++ ydb/trunk/src/Makefile 2009-03-22 09:06:24 UTC (rev 1321)
@@ -1,10 +1,18 @@
TARGET := ydb
WTF := wtf
CCACHE := ccache
+DISTCC := distcc
+CCACHE_PREFIX := $(DISTCC)
+ifeq ($(CCACHE),)
+ACCEL := $(DISTCC)
+else
+ACCEL := $(CCACHE)
+endif
CLAMPS := $(wildcard *.lzz.clamp)
+CLAMPLZZS:= $(patsubst %.clamp,%,$(CLAMPS))
PURELZZS := $(foreach lzz,$(wildcard *.lzz),$(if $(wildcard $(lzz).clamp),,$(lzz)))
-LZZS := $(patsubst %.clamp,%,$(CLAMPS)) $(PURELZZS)
+LZZS := $(CLAMPLZZS) $(PURELZZS)
LZZHDRS := $(foreach lzz,$(LZZS),$(patsubst %.lzz,%.hh,$(lzz)))
LZZSRCS := $(foreach lzz,$(LZZS),$(patsubst %.lzz,%.cc,$(lzz)))
LZZOBJS := $(foreach lzz,$(LZZS),$(patsubst %.lzz,%.o,$(lzz)))
@@ -40,7 +48,7 @@
OPT := -g3
endif
# CXX := $(WTF) ag++ -k --Xcompiler # $(CXX)
-CXX := $(WTF) $(CCACHE) $(CXX) -pipe
+CXX := $(WTF) $(ACCEL) $(CXX) -pipe
CC := $(CXX) # for linking
LDFLAGS := -pthread $(GPROF)
LDLIBS := -lstx -lst -lresolv -lprotobuf -lgtest \
@@ -78,7 +86,8 @@
-Wlong-long \
-Wvolatile-register-var \
-std=gnu++0x \
- -march=native \
+ -m64 \
+ -march=$(shell gcc-config march) \
$(CXXFLAGS)
# \
@@ -93,9 +102,9 @@
-Wstrict-overflow \
-Winline \
-ifeq ($(NPCH),)
+ifneq ($(PCH),)
CXXFLAGS := $(CXXFLAGS0) -include pch.h
-%.o: pch.h.gch
+$(LZZOBJS): pch.h.gch
else
CXXFLAGS := $(CXXFLAGS0)
endif
@@ -140,7 +149,7 @@
chmod -w $@
pch.h:
- svn ls -rHEAD -R | \
+ svn ls -rHEAD -R https://assorted.svn.sourceforge.net/svnroot/assorted/ydb/trunk/src | \
egrep -v '/$$|Makefile' | \
xargs sed 's/.*\binclude\b *<\(.*\)>.*/\#include <\1>/; t succ; d; :succ /commons/ d' | \
sort -u > $@
@@ -151,8 +160,7 @@
clean:
rm -rf clamp/ $(GENSRCS) $(GENHDRS) $(OBJS) $(TARGET) \
- main.lzz ydb.lzz main.cc main.hh ydb.hh ydb.cc \
- util.cc util.hh tpcc.lzz tpcc.hh tpcc.cc
+ $(CLAMPLZZS) $(LZZHDRS) $(LZZSRCS)
make -C tpcc/ clean
distclean: clean
Modified: ydb/trunk/src/unsetprefs.h
===================================================================
--- ydb/trunk/src/unsetprefs.h 2009-03-21 23:23:39 UTC (rev 1320)
+++ ydb/trunk/src/unsetprefs.h 2009-03-22 09:06:24 UTC (rev 1321)
@@ -0,0 +1,5 @@
+#undef function
+#undef shared_ptr
+#undef ref
+#undef tuple
+#undef make_tuple
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <yan...@us...> - 2009-03-21 23:23:44
|
Revision: 1320
http://assorted.svn.sourceforge.net/assorted/?rev=1320&view=rev
Author: yangzhang
Date: 2009-03-21 23:23:39 +0000 (Sat, 21 Mar 2009)
Log Message:
-----------
added gcc-config
Modified Paths:
--------------
shell-tools/trunk/README
Added Paths:
-----------
shell-tools/trunk/src/gcc-config.bash
Modified: shell-tools/trunk/README
===================================================================
--- shell-tools/trunk/README 2009-03-20 22:30:59 UTC (rev 1319)
+++ shell-tools/trunk/README 2009-03-21 23:23:39 UTC (rev 1320)
@@ -57,6 +57,8 @@
`filter-urls` Given a list of URLs, filter out live URLs or Python
vice-versa.
+`gcc-config` Inspect gcc's default/automatic settings. bash, gcc
+
`gen-dates` Generate lists of dates. Python
`gen-list` Generate lists based on a template string. bash
Added: shell-tools/trunk/src/gcc-config.bash
===================================================================
--- shell-tools/trunk/src/gcc-config.bash (rev 0)
+++ shell-tools/trunk/src/gcc-config.bash 2009-03-21 23:23:39 UTC (rev 1320)
@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+# Specify an argument to just print the value of that argument, e.g. "march".
+# Specify no argument to print the whole cc1 line.
+# Specify CXXFLAGS (eg empty string) to control flags.
+
+echo |
+gcc -c ${CXXFLAGS--march=native} -v -o /dev/null -x c - 2>&1 |
+fgrep -- cc1 |
+if (( $# > 0 ))
+then sed 's/.* -'"$1"'=\([^ ]*\) .*/\1/'
+else cat
+fi
Property changes on: shell-tools/trunk/src/gcc-config.bash
___________________________________________________________________
Added: svn:executable
+ *
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <yan...@us...> - 2009-03-20 22:31:01
|
Revision: 1319
http://assorted.svn.sourceforge.net/assorted/?rev=1319&view=rev
Author: yangzhang
Date: 2009-03-20 22:30:59 +0000 (Fri, 20 Mar 2009)
Log Message:
-----------
- cleaned up syntax errors in the original html page
- transcribed the original page into new README file
- added publisher
Modified Paths:
--------------
clamp/trunk/doc/index.html
Added Paths:
-----------
clamp/trunk/README
clamp/trunk/publish.bash
Added: clamp/trunk/README
===================================================================
--- clamp/trunk/README (rev 0)
+++ clamp/trunk/README 2009-03-20 22:30:59 UTC (rev 1319)
@@ -0,0 +1,254 @@
+[original homepage] | [original author]
+
+[original homepage]: http://home.clara.net/raoulgough/clamp/
+[original author]: http://home.clara.net/raoulgough/
+
+Overview
+--------
+
+The C++ lambda preprocessor (clamp) converts C++ code containing lambda
+expressions into ordinary C++ code. It was originally written by [Raoul
+Gough], and is now maintained by [Yang Zhang].
+
+Changes that have taken place since the original clamp release:
+
+- updated the source to be more standards-conformant and build on modern
+ platforms out of the box
+- added `--prefix` option that allows you to customize the names of the
+ generated structs/files (to avoid name conflicts/clobbering)
+- added `--outdir` to allow you to specify an alternate output directory
+ instead of the current working directory (to minimize clutter)
+
+The rest of this document is mostly a direct copy of the content from the
+[original homepage].
+
+[Raoul Gough]: http://home.clara.net/raoulgough/
+[Yang Zhang]: http://www.mit.edu/~y_z/
+
+Introduction
+------------
+
+The C++ lambda preprocessor (clamp) converts C++ code containing lambda
+expressions into ordinary C++ code. Here's a simple example:
+
+~~~ {.cpp}
+vector<int> v;
+// ...
+std::for_each(v.begin(), v.end(), lambda (int &p) {
+ if (p == 5) p = 0;
+});
+~~~
+
+This example uses the standard algorithm `for_each` to apply an anonymous
+function to each element of a vector. The anonymous function accepts an integer
+parameter by reference, and resets the value to zero if it is currently five (a
+simple, but not very useful example). The preprocessor replaces the entire
+lambda expression in its output, so that the C++ compiler ends up seeing
+something like the following:
+
+~~~ {.cpp}
+std::for_each(v.begin(), v.end(),
+ lambda_generator_1<void, int &>::generate());
+~~~
+
+The exact nature of the template `lambda_generator_1` is beyond the scope of
+this introduction, except to say that its `generate()` member function returns
+a function object by value. The function object has, in this case, a member
+function `void operator()(int &)` which `for_each` applies to each element of
+the vector. Some people would probably prefer to use the standard `transform`
+algorithm for this example, as in:
+
+~~~ {.cpp}
+std::transform(v.begin(), v.end(), v.begin(),
+ lambda int (int p) { return p == 5 ? 0 : p; });
+~~~
+
+This example shows an anonymous function that returns a value, in this case
+an int. Rather than hard-wiring a value into the function body, it is also
+possible to include contextual information in the function object. For
+instance:
+
+~~~ {.cpp}
+void reset(std::vector<int> &v, int val) {
+ std::transform(v.begin(), v.end(), v.begin(),
+ lambda int (int p) { return p == __ctx(val) ? 0 : p; });
+}
+~~~
+
+The `__ctx` expression is an example of context information bound by value. The
+clamp preprocessor also supports reference semantics for contextual information
+via `__ref` expressions. For example:
+
+~~~ {.cpp}
+int sum = 0;
+std::for_each(v.begin(), v.end(),
+ lambda (int p) { __ref(sum) += p; });
+~~~
+
+This, of course, calculates the sum of elements in the vector.
+
+Getting into some more complicated examples, it is possible to name
+the type of the function object generated by a lambda expression by
+simply omitting the function body. You have to do this, for
+instance, if you want to use an anonymous function generated by a
+lambda expression as a function parameter or return value. For
+example, the type of the expression from the previous example:
+
+~~~ {.cpp}
+lambda (int p) { __ref(sum) += p; }
+~~~
+
+can be referred to in the code as `lambda (int &) (int)`. The first pair of
+brackets contains the context binding (or closure) parameters, and the second
+pair contains the function parameters. The closure parameter list is optional
+for context-less functions, as is the return type for functions returning
+`void`, such as this one. Putting all of that together, here's a templated
+function that returns a function object:
+
+~~~ {.cpp}
+template<typename T>
+lambda bool(T) (const T &)
+match(const T &target) {
+ return lambda bool(const T &candidate) {
+ return candidate == __ctx(target);
+ };
+}
+
+// Use a generated comparison object
+std::vector<int>::iterator i =
+ find_if(v.begin(), v.end(), match(7));
+~~~
+
+This `find_if` example returns an iterator to the first 7 in the vector (or
+`v.end()`, if none) using an instantiation of the match template with an int
+parameter. For a vector of strings, you could do the following:
+
+~~~
+std::vector<std::string>::iterator i =
+ find_if(v.begin(), v.end(), match(std::string("hello")));
+~~~
+
+Why a preprocessor?
+-------------------
+
+I wrote the preprocessor just for fun. There doesn't seem to be any way to
+achieve real lambda expressions in pure C++, since it won't let you insert a
+function definition in the middle of an expression. The limits of what pure C++
+allows are pretty well exhausted by the [boost lambda
+library](http://www.boost.org/libs/lambda/doc/).
+
+Lambda expressions simplify some coding tasks, so it would be nice to have them
+in C++. In the time it takes you to extract that one-liner into a named
+function, I bet you could write *two* lambda expressions for sure. Not to
+mention cases which require a named class that contains context information.
+
+What it does
+------------
+
+clamp scans its input for lambda expressions, passing any plain C++
+through unchanged. When it encounters a lambda expression, it
+extracts the function body into a separate file. It also generates
+a class template with a suitable `operator()` and (where necessary)
+member variables to store any context binding. This class template
+also goes into a separate file. The whole lambda expression is then
+replaced in the output by a single constructor call, which creates
+an object of the templated class.
+
+The first line of the output is always a `#include` directive, which
+drags in the generated templates and (indirectly) the function
+bodies. The generated templates do not refer explicitly to any
+types used in the original lambda expressions, which is how it can
+be included before any user code. The actual types are only bound
+at the point of use. Because of this, the clamp parser doesn't have
+to know what scope a lambda expression appears in, or where the
+required types are defined. This also makes including lambda
+expressions in templated code a breeze, since the type binding is
+done within the template scope where the expression was originally
+used.
+
+How it works
+------------
+
+The clamp preprocessor consists of a lexical analyser (lexer)
+written in flex, a parser written in bison and a code generator in
+plain C++.
+The clamp parser mostly tries to ignore everything in the input
+file, letting the lexer copy input to output. When the lexer
+encounters the `lambda` keyword, it enters a different mode ("start
+condition" in flex terminology) in which is behaves like a normal
+lexer and supplies tokens to the parser. The parser does some messy
+stuff redirecting output and resetting the lexer mode as necessary.
+Note: clamp is actually pretty dumb. It performs purely syntactic
+transformations on the input, without really understanding scope,
+types or variables. This will no doubt result in some
+incomprehensible errors from the C++ compiler if something goes
+wrong. This is also the reason that clamp requires the `__ctx` and
+`__ref` keywords, since it wouldn't otherwise be able to tell that
+an expression relies on surrounding context information.
+
+Grammar
+-------
+
+clamp introduces three keywords: `lambda`, `__ctx` and `__ref`. The parser
+recognises more or less the following grammar:
+
+<pre>
+ _lambda-expression_:
+ _lambda-decl_ _lambda-body~opt~_
+
+ _lambda-decl_:
+ `lambda` _return-type~opt~_ _param-list~opt~_ _param-list_
+
+ _return-type_:
+ _type-id_
+
+ _param-list_:
+ ( )
+ ( _parameter_ )
+ ( _parameter_ , ... )
+
+ parameter:
+ _type-id_ _identifier~opt~_ _initialiser~opt~_
+
+ initialiser:
+ = _expression_
+
+ lambda-body:
+ { _statement~opt~_ ... }
+</pre>
+
+where _statement_ represents any valid C++ statement, possibly making use of the
+following extended expressions:
+
+ _extended-expression_:
+ _lambda-expression_
+ `__ctx` ( _expression_ )
+ `__ref` ( _expression_ )
+
+Portability
+-----------
+
+I wrote clamp using the following tools, all under Cygwin on Windows 2000.
+
+- g++ 2.95.3-5
+- boost 1.25.1
+- flex 2.5.4
+- bision 1.28
+- gnu make 3.79.1
+
+The preprocessor builds successfully with g++ 3.1, but the code that it
+generates causes an internal compiler error when taking the address of a member
+function. This is probably fixed in later versions of g++.
+
+The preprocessor itself *might* build with yacc and/or traditional Unix make
+(maybe) with any reasonable C++ compiler. The lexer probably won't compile with
+plain lex, because (according to the flex manual), lex doesn't support
+exclusive start conditions.
+
+Contact information
+-------------------
+
+This page and clamp itself are Copyright (C) 2002, 2003 by Raoul Gough and may
+be used and distributed free of charge. Please send any clamp-related comments
+to <Rao...@ya...>. It might be a good idea to include the word
+"clamp" in the subject line, because that email address attracts a bit of spam.
Modified: clamp/trunk/doc/index.html
===================================================================
--- clamp/trunk/doc/index.html 2009-03-20 21:17:36 UTC (rev 1318)
+++ clamp/trunk/doc/index.html 2009-03-20 22:30:59 UTC (rev 1319)
@@ -78,7 +78,7 @@
std::for_each (v.begin(), v.end()<br>
- , <b>lambda</b> (int &p) {<br>
+ , <b>lambda</b> (int &p) {<br>
if (p == 5) p = 0;<br>
@@ -100,7 +100,7 @@
<font class="code">
std::for_each (v.begin(), v.end()<br>
- , lambda_generator_1<void, int &>::generate () );
+ , lambda_generator_1<void, int &>::generate () );
</font>
<p>
@@ -109,7 +109,7 @@
introduction, except to say that its <font
class="code">generate()</font> member function returns a function
object by value. The function object has, in this case, a member
-function <font class="code">void operator()(int &)</font>
+function <font class="code">void operator()(int &)</font>
which <font class="code">for_each</font> applies to each element of
the vector. Some people would probably prefer to use the standard
<font class="code">transform</font> algorithm for this example, as in:
@@ -136,7 +136,7 @@
<p>
<font class="code">
-void reset (std::vector<int> &v, int val) {<br>
+void reset (std::vector<int> &v, int val) {<br>
std::transform (v.begin(), v.end(), v.begin()<br>
@@ -183,7 +183,7 @@
</font>
<p>
-can be referred to in the code as "<font class="code">lambda (int &)
+can be referred to in the code as "<font class="code">lambda (int &)
(int)</font>". The first pair of brackets contains the context binding
(or closure) parameters, and the second pair contains the function
parameters. The closure parameter list is optional for context-less
@@ -194,9 +194,9 @@
<p>
<font class="code">
template<typename T><br>
-lambda bool (T) (const T &)<br>
-match (const T &target) {<br>
- return lambda bool (const T &candidate) {<br>
+lambda bool (T) (const T &)<br>
+match (const T &target) {<br>
+ return lambda bool (const T &candidate) {<br>
return candidate == __ctx(target);<br>
};<br>
}<br>
@@ -486,4 +486,4 @@
</table>
</body>
-</html>
\ No newline at end of file
+</html>
Copied: clamp/trunk/publish.bash (from rev 1272, ydb/trunk/publish.bash)
===================================================================
--- clamp/trunk/publish.bash (rev 0)
+++ clamp/trunk/publish.bash 2009-03-20 22:30:59 UTC (rev 1319)
@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+fullname='C++ lambda preprocessor'
+version=2.0
+license=gpl3
+websrcs=( README )
+rels=( src-tgz: )
+nodl=true
+. assorted.bash "$@"
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <yan...@us...> - 2009-03-20 21:51:13
|
Revision: 1318
http://assorted.svn.sourceforge.net/assorted/?rev=1318&view=rev
Author: yangzhang
Date: 2009-03-20 21:17:36 +0000 (Fri, 20 Mar 2009)
Log Message:
-----------
fixed the test case outdir
Modified Paths:
--------------
clamp/trunk/src/Makefile
Modified: clamp/trunk/src/Makefile
===================================================================
--- clamp/trunk/src/Makefile 2009-03-20 21:15:44 UTC (rev 1317)
+++ clamp/trunk/src/Makefile 2009-03-20 21:17:36 UTC (rev 1318)
@@ -68,4 +68,5 @@
.PRECIOUS: %.cc
%.cc: %.clamp clamp
- ./clamp --outdir asdf/ --prefix $(basename $<) <$< >$@
+ mkdir -p clampout
+ ./clamp --outdir clampout/ --prefix $(basename $<) <$< >$@
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <yan...@us...> - 2009-03-20 21:16:10
|
Revision: 1317
http://assorted.svn.sourceforge.net/assorted/?rev=1317&view=rev
Author: yangzhang
Date: 2009-03-20 21:15:44 +0000 (Fri, 20 Mar 2009)
Log Message:
-----------
added --outdir
Modified Paths:
--------------
clamp/trunk/src/CodeGen.cc
clamp/trunk/src/CodeGen.hh
clamp/trunk/src/Makefile
clamp/trunk/src/clamp.y
Modified: clamp/trunk/src/CodeGen.cc
===================================================================
--- clamp/trunk/src/CodeGen.cc 2009-03-20 20:40:27 UTC (rev 1316)
+++ clamp/trunk/src/CodeGen.cc 2009-03-20 21:15:44 UTC (rev 1317)
@@ -113,14 +113,17 @@
// Constructors
////////////////////////////////////////////////////////////////////////
- CodeGen::CodeGen (const string &prefix)
+ using namespace boost::filesystem;
+
+ CodeGen::CodeGen (const path &outdir, const string &prefix)
: mStack ()
, mLambdaCount (0)
- , mImplStream ((prefix + "_lambda_impl.clamp_h").c_str())
- , mGenrStream ((prefix + "_lambda_genr.clamp_h").c_str())
+ , mImplStream ((outdir / (prefix + "_lambda_impl.clamp_h")).string().c_str())
+ , mGenrStream ((outdir / (prefix + "_lambda_genr.clamp_h")).string().c_str())
, mLambdaMap ()
, mGeneratorStream ()
, mPrefix (prefix)
+ , mOutdir (outdir)
{
}
@@ -732,7 +735,7 @@
{
mStack.push (LambdaContext (f, openBracePos, lamp));
- string newName (bodyFileName (++mLambdaCount));
+ string newName ((mOutdir / bodyFileName (++mLambdaCount)).string().c_str());
f = fopen (newName.c_str(), "w+");
Modified: clamp/trunk/src/CodeGen.hh
===================================================================
--- clamp/trunk/src/CodeGen.hh 2009-03-20 20:40:27 UTC (rev 1316)
+++ clamp/trunk/src/CodeGen.hh 2009-03-20 21:15:44 UTC (rev 1317)
@@ -31,17 +31,19 @@
#include <fstream>
#include <map>
#include <sstream>
-#include "boost/smart_ptr.hpp"
+#include <boost/smart_ptr.hpp>
#include <string>
+#include <boost/filesystem.hpp>
namespace clamp
{
using namespace std;
+ using namespace boost::filesystem;
class CodeGen
{
public:
- CodeGen (const std::string &prefix);
+ CodeGen (const path &outdir, const std::string &prefix);
public:
void lambdaType (FILE *, std::auto_ptr<lambdaT>);
@@ -135,6 +137,7 @@
private:
std::string mPrefix;
+ path mOutdir;
};
Modified: clamp/trunk/src/Makefile
===================================================================
--- clamp/trunk/src/Makefile 2009-03-20 20:40:27 UTC (rev 1316)
+++ clamp/trunk/src/Makefile 2009-03-20 21:15:44 UTC (rev 1317)
@@ -35,7 +35,7 @@
LFLAGS =
YFLAGS = -v -d
-LDLIBS = -lboost_program_options-gcc43-mt
+LDLIBS = -lboost_program_options-gcc43-mt -lboost_filesystem-gcc43-mt
LD = $(CXX)
clamp.tab.h: clamp.cc
@@ -68,4 +68,4 @@
.PRECIOUS: %.cc
%.cc: %.clamp clamp
- ./clamp --prefix $(basename $<) <$< >$@
+ ./clamp --outdir asdf/ --prefix $(basename $<) <$< >$@
Modified: clamp/trunk/src/clamp.y
===================================================================
--- clamp/trunk/src/clamp.y 2009-03-20 20:40:27 UTC (rev 1316)
+++ clamp/trunk/src/clamp.y 2009-03-20 21:15:44 UTC (rev 1317)
@@ -33,6 +33,7 @@
#include <malloc.h>
#include <set>
#include <boost/program_options.hpp>
+#include <boost/filesystem.hpp>
using namespace clamp;
using namespace std;
@@ -349,22 +350,21 @@
int main (int argc, char *argv[])
{
+ using namespace boost::filesystem;
+
int result;
string prefix;
+ path outdir;
namespace po = boost::program_options;
po::options_description desc("Allowed options");
desc.add_options()
- ("help,h", "show this help message")
+ ("help,h", "show this help message")
("debug,d", "enable yydebug")
-#if 0
- ("exit-on-recovery,x", po::bool_switch(&stop_on_recovery),
- "exit after the joiner fully recovers (for leader)")
- ("batch-size,b", po::value<int>(&batch_size)->default_value(100),
- "number of txns to batch up in each msg (for leader)")
-#endif
- ("prefix,p", po::value<string>(&prefix)->default_value(""),
+ ("outdir,O", po::value<path>(&outdir),
+ "directory in which to place the output files")
+ ("prefix,p", po::value<string>(&prefix),
"hostname or address of the leader");
po::variables_map vm;
@@ -400,13 +400,13 @@
}
#endif
- cout << "#include \"" << prefix << "_lambda_impl.clamp_h\"\n";
+ cout << "#include \"" << outdir / prefix << "_lambda_impl.clamp_h\"\n";
- gGenPtr = new CodeGen(prefix);
+ gGenPtr = new CodeGen(outdir, prefix);
result = yyparse();
- cout << "#include \"" << prefix << "_lambda_genr.clamp_h\"\n";
+ cout << "#include \"" << outdir / prefix << "_lambda_genr.clamp_h\"\n";
if (result == 0)
{
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <yan...@us...> - 2009-03-20 20:40:36
|
Revision: 1316
http://assorted.svn.sourceforge.net/assorted/?rev=1316&view=rev
Author: yangzhang
Date: 2009-03-20 20:40:27 +0000 (Fri, 20 Mar 2009)
Log Message:
-----------
- simplified Makefile a bit for aux programs
- made ccache, pch usage optional
- updated, fixed warnings in aux programs
Modified Paths:
--------------
ydb/trunk/src/Makefile
ydb/trunk/src/main.lzz.clamp
ydb/trunk/src/serperf.cc
ydb/trunk/src/stxn.lzz.clamp
ydb/trunk/src/tpcc/Makefile
ydb/trunk/src/tpcc/tpcctables.h
ydb/trunk/src/unsetprefs.h
ydb/trunk/src/util.lzz
ydb/trunk/src/ydb.lzz.clamp
Modified: ydb/trunk/src/Makefile
===================================================================
--- ydb/trunk/src/Makefile 2009-03-20 20:20:25 UTC (rev 1315)
+++ ydb/trunk/src/Makefile 2009-03-20 20:40:27 UTC (rev 1316)
@@ -1,5 +1,6 @@
TARGET := ydb
WTF := wtf
+CCACHE := ccache
CLAMPS := $(wildcard *.lzz.clamp)
PURELZZS := $(foreach lzz,$(wildcard *.lzz),$(if $(wildcard $(lzz).clamp),,$(lzz)))
@@ -39,7 +40,7 @@
OPT := -g3
endif
# CXX := $(WTF) ag++ -k --Xcompiler # $(CXX)
-CXX := $(WTF) ccache $(CXX) -pipe
+CXX := $(WTF) $(CCACHE) $(CXX) -pipe
CC := $(CXX) # for linking
LDFLAGS := -pthread $(GPROF)
LDLIBS := -lstx -lst -lresolv -lprotobuf -lgtest \
@@ -79,7 +80,6 @@
-std=gnu++0x \
-march=native \
$(CXXFLAGS)
-CXXFLAGS := $(CXXFLAGS0) -include pch.h
# \
-Wmissing-noreturn \
@@ -93,6 +93,13 @@
-Wstrict-overflow \
-Winline \
+ifeq ($(NPCH),)
+ CXXFLAGS := $(CXXFLAGS0) -include pch.h
+%.o: pch.h.gch
+else
+ CXXFLAGS := $(CXXFLAGS0)
+endif
+
PBCXXFLAGS := $(OPT) -Wall -Werror $(GPROF)
all: $(TARGET)
@@ -100,13 +107,12 @@
%.pb.o: %.pb.cc %.pb.h
$(CXX) -c $(PBCXXFLAGS) $(OUTPUT_OPTION) $<
-%.o: pch.h.gch
stxn.o: main.hh $(PBHDRS)
main.o: util.hh msg.h $(PBHDRS)
util.o: msg.h $(PBHDRS)
ydb.o: main.hh stxn.hh tpcc.hh util.hh $(PBHDRS)
tpcc.o: main.hh util.hh $(PBHDRS)
-ydb: ydb.o tpcc.o main.o util.o stxn.o ydb.pb.o $(TPCC_OBJS) # $(OBJS)
+ydb: $(OBJS)
tpcc/%.o: tpcc/%.cc
make -C tpcc/
@@ -135,7 +141,7 @@
pch.h:
svn ls -rHEAD -R | \
- grep -v '/$$' | \
+ egrep -v '/$$|Makefile' | \
xargs sed 's/.*\binclude\b *<\(.*\)>.*/\#include <\1>/; t succ; d; :succ /commons/ d' | \
sort -u > $@
@@ -157,17 +163,10 @@
.PHONY: clean
-.SECONDARY: $(SRCS) $(HDRS) $(OBJS) main.lzz
+.SECONDARY: $(SRCS) $(HDRS) $(OBJS) main.lzz pch.h.gch
###
-serperf: serperf.o ydb.pb.o
- $(LINK.o) $^ $(LOADLIBES) $(LDLIBS) $(OUTPUT_OPTION)
-
-# serperf.cc ydb.pb.h
-
-p2: p2.cc
- $(LINK.cc) $^ $(LOADLIBES) $(LDLIBS) $(OUTPUT_OPTION)
-
-ser: ser.cc msg.h ydb.pb.o
- $(LINK.cc) $^ $(LOADLIBES) $(LDLIBS) $(OUTPUT_OPTION)
+serperf: ydb.pb.o
+ser: ydb.pb.o
+ser.o: msg.h
Modified: ydb/trunk/src/main.lzz.clamp
===================================================================
--- ydb/trunk/src/main.lzz.clamp 2009-03-20 20:20:25 UTC (rev 1315)
+++ ydb/trunk/src/main.lzz.clamp 2009-03-20 20:40:27 UTC (rev 1316)
@@ -2,17 +2,9 @@
#include "unsetprefs.h"
#include <boost/archive/binary_iarchive.hpp>
#include <boost/archive/binary_oarchive.hpp>
-#include <boost/bind.hpp>
-#include <boost/foreach.hpp>
-//#include <boost/range/iterator_range.hpp>
-//#include <boost/shared_ptr.hpp>
#include <boost/tuple/tuple.hpp>
-#include <commons/assert.h>
-#include <commons/nullptr.h>
#include <commons/st/st.h>
-#include <commons/time.h>
#include <fstream> // ofstream
-#include <iostream>
#include <vector>
#include "msg.h"
#include "util.hh"
@@ -27,7 +19,14 @@
#end
#src
+#include "unsetprefs.h"
+#include <boost/foreach.hpp>
+#include <commons/assert.h>
+#include <commons/nullptr.h>
+#include <commons/time.h>
+#include <iostream>
#include <unistd.h> // pipe, write, sync
+#include "setprefs.h"
#end
typedef tuple<sized_array<char>, char*, char*> chunk;
Modified: ydb/trunk/src/serperf.cc
===================================================================
--- ydb/trunk/src/serperf.cc 2009-03-20 20:20:25 UTC (rev 1315)
+++ ydb/trunk/src/serperf.cc 2009-03-20 20:40:27 UTC (rev 1316)
@@ -1,16 +1,18 @@
#include <iostream>
#include <sstream>
#include <commons/time.h>
+#include <commons/utility.h>
#include "ydb.pb.h"
#include <boost/archive/binary_oarchive.hpp>
using namespace boost::archive;
using namespace std;
using namespace commons;
+using namespace ydb::pb;
-int main(int argc, char **argv) {
+int main(UNUSED int argc, char **argv) {
const int count = atoi(argv[1]), batchsize = atoi(argv[2]);
-
+
TxnBatch batch;
for (int i = 0; i < batchsize; ++i) {
Txn &txn = *batch.add_txn();
@@ -30,7 +32,7 @@
batch.SerializeToOstream(&ss);
}
long long time = current_time_millis() - start;
- double tps = 1000 * static_cast<double>(count * batchsize) / time;
+ double tps = 1000 * count * batchsize / double(time);
cout << "protobuf: " << time << " ms, " << tps << " tps" << endl;
}
@@ -51,7 +53,7 @@
}
}
long long time = current_time_millis() - start;
- double tps = 1000 * static_cast<double>(count * batchsize) / time;
+ double tps = 1000 * count * batchsize / double(time);
cout << "boost: " << time << " ms, " << tps << " tps" << endl;
}
@@ -61,7 +63,7 @@
stringbuf sb;
for (int j = 0; j < batchsize; ++j) {
const Txn &txn = batch.txn(j);
-#define write(x) { typeof(x) __x = x; sb.sputn((char*)(&__x), sizeof __x); }
+#define write(x) { typeof(x) __x = x; sb.sputn(reinterpret_cast<char*>(&__x), sizeof __x); }
write(txn.seqno());
for (int k = 0; k < 5; ++k) {
const Op &op = txn.op(k);
@@ -72,7 +74,7 @@
}
}
long long time = current_time_millis() - start;
- double tps = 1000 * static_cast<double>(count * batchsize) / time;
+ double tps = 1000 * count * batchsize / double(time);
cout << "streambuf.sputn: " << time << " ms, " << tps << " tps" << endl;
}
Modified: ydb/trunk/src/stxn.lzz.clamp
===================================================================
--- ydb/trunk/src/stxn.lzz.clamp 2009-03-20 20:20:25 UTC (rev 1315)
+++ ydb/trunk/src/stxn.lzz.clamp 2009-03-20 20:40:27 UTC (rev 1316)
@@ -1,5 +1,6 @@
#hdr
#include "unsetprefs.h"
+#include <boost/bind.hpp>
#include <commons/memory.h>
#include <boost/foreach.hpp>
#include <commons/snap_map.h>
Modified: ydb/trunk/src/tpcc/Makefile
===================================================================
--- ydb/trunk/src/tpcc/Makefile 2009-03-20 20:20:25 UTC (rev 1315)
+++ ydb/trunk/src/tpcc/Makefile 2009-03-20 20:40:27 UTC (rev 1316)
@@ -1,6 +1,7 @@
WARNINGS = -Werror -Wall -Wextra -Wconversion -Wpointer-arith -Wcast-qual -Wcast-align -Wwrite-strings -Woverloaded-virtual -Wno-sign-compare -Wno-unused-parameter
-CXX := ccache $(CXX)
+CCACHE := ccache
+CXX := $(CCACHE) $(CXX)
# Debug flags
ifeq ($(OPT),)
Modified: ydb/trunk/src/tpcc/tpcctables.h
===================================================================
--- ydb/trunk/src/tpcc/tpcctables.h 2009-03-20 20:20:25 UTC (rev 1315)
+++ ydb/trunk/src/tpcc/tpcctables.h 2009-03-20 20:40:27 UTC (rev 1316)
@@ -1,6 +1,7 @@
#ifndef TPCCTABLES_H__
#define TPCCTABLES_H__
+#include <limits>
#include <map>
#include <set>
#include <vector>
Modified: ydb/trunk/src/unsetprefs.h
===================================================================
--- ydb/trunk/src/unsetprefs.h 2009-03-20 20:20:25 UTC (rev 1315)
+++ ydb/trunk/src/unsetprefs.h 2009-03-20 20:40:27 UTC (rev 1316)
@@ -1,5 +0,0 @@
-#undef function
-#undef shared_ptr
-#undef ref
-#undef tuple
-#undef make_tuple
Modified: ydb/trunk/src/util.lzz
===================================================================
--- ydb/trunk/src/util.lzz 2009-03-20 20:20:25 UTC (rev 1315)
+++ ydb/trunk/src/util.lzz 2009-03-20 20:40:27 UTC (rev 1316)
@@ -7,12 +7,10 @@
#include <map>
#include <set>
#include <utility>
-#include <boost/function.hpp>
#include <boost/scoped_array.hpp>
#include <commons/array.h>
#include <commons/nullptr.h>
#include <commons/time.h>
-#include <sys/socket.h> // getpeername
#include <google/protobuf/io/zero_copy_stream_impl.h>
#include "msg.h"
using namespace std;
@@ -21,6 +19,7 @@
using namespace google::protobuf::io;
#end
#src
+#include <sys/socket.h> // getpeername
#include <gtest/gtest.h>
#include <netinet/in.h> // in_addr etc.
#end
Modified: ydb/trunk/src/ydb.lzz.clamp
===================================================================
--- ydb/trunk/src/ydb.lzz.clamp 2009-03-20 20:20:25 UTC (rev 1315)
+++ ydb/trunk/src/ydb.lzz.clamp 2009-03-20 20:40:27 UTC (rev 1316)
@@ -1,5 +1,6 @@
#hdr
#include "unsetprefs.h"
+#include <boost/bind.hpp>
#include <boost/function.hpp>
#include <boost/scoped_ptr.hpp>
#include <string>
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <yan...@us...> - 2009-03-20 20:20:38
|
Revision: 1315
http://assorted.svn.sourceforge.net/assorted/?rev=1315&view=rev
Author: yangzhang
Date: 2009-03-20 20:20:25 +0000 (Fri, 20 Mar 2009)
Log Message:
-----------
- fixed all global functions to be either inline or static
- added break_exception
- added UNUSED
- tweaked timer ctor
- fixed missing include for snap_map
- added operator[]() and at() to array classes
Modified Paths:
--------------
cpp-commons/trunk/src/commons/algo.h
cpp-commons/trunk/src/commons/array.h
cpp-commons/trunk/src/commons/delegates.h
cpp-commons/trunk/src/commons/exceptions.h
cpp-commons/trunk/src/commons/memory.h
cpp-commons/trunk/src/commons/snap_map.h
cpp-commons/trunk/src/commons/sockets.h
cpp-commons/trunk/src/commons/st/st.h
cpp-commons/trunk/src/commons/time.h
cpp-commons/trunk/src/commons/utility.h
Modified: cpp-commons/trunk/src/commons/algo.h
===================================================================
--- cpp-commons/trunk/src/commons/algo.h 2009-03-20 17:45:38 UTC (rev 1314)
+++ cpp-commons/trunk/src/commons/algo.h 2009-03-20 20:20:25 UTC (rev 1315)
@@ -5,7 +5,7 @@
namespace commons {
- __attribute__((unused)) inline void
+ inline void
swap(size_t &x, size_t &y) {
x = x ^ y;
y = x ^ y;
Modified: cpp-commons/trunk/src/commons/array.h
===================================================================
--- cpp-commons/trunk/src/commons/array.h 2009-03-20 17:45:38 UTC (rev 1314)
+++ cpp-commons/trunk/src/commons/array.h 2009-03-20 20:20:25 UTC (rev 1315)
@@ -3,7 +3,7 @@
#include <algorithm>
#include <commons/algo.h>
-#include <commons/check.h>
+#include <commons/assert.h>
#include <commons/nullptr.h>
#include <commons/unique_ptr.h>
#include <commons/utility.h>
@@ -34,6 +34,8 @@
T *end() const { return p_ + n_; }
const T &operator[](size_t i) const { return p_[i]; }
T &operator[](size_t i) { return p_[i]; }
+ const T &at(size_t i) const { ASSERT(i < n_); return p_[i]; }
+ T &at(size_t i) { ASSERT(i < n_); return p_[i]; }
void reset(T *p, size_t n) { p_ = p; n_ = n; }
private:
T *p_;
@@ -74,6 +76,8 @@
T *release() { return p_.release(); }
T *begin() const { return p_.get(); }
T *end() const { return this->get() + n_; }
+ const T &at(size_t i) const { ASSERT(i < n_); return p_[i]; }
+ T &at(size_t i) { ASSERT(i < n_); return p_[i]; }
const T &operator[](size_t i) const { return p_[i]; }
T &operator[](size_t i) { return p_[i]; }
void reset(T *p, size_t n) { p_.reset(p); n_ = n; }
@@ -86,7 +90,7 @@
* Swap two arrays.
*/
template<typename T>
- void
+ inline void
swap(array<T> &a, array<T> &b)
{
std::swap(a.p_, b.p_);
@@ -99,7 +103,7 @@
*/
template<typename T>
void
- swap(sized_array<T> &a, sized_array<T> &b)
+ inline swap(sized_array<T> &a, sized_array<T> &b)
{
std::swap(a.p_, b.p_);
swap(a.n_, b.n_);
@@ -118,6 +122,8 @@
T *release() { T *p = p_; p_ = nullptr; scoped_ = false; return p; }
T *get() { return p_; }
const T *get() const { return p_; }
+ const T &operator[](size_t i) const { return p_[i]; }
+ T &operator[](size_t i) { return p_[i]; }
operator T*() { return p_; }
operator const T*() const { return p_; }
bool scoped() const { return scoped_; }
Modified: cpp-commons/trunk/src/commons/delegates.h
===================================================================
--- cpp-commons/trunk/src/commons/delegates.h 2009-03-20 17:45:38 UTC (rev 1314)
+++ cpp-commons/trunk/src/commons/delegates.h 2009-03-20 20:20:25 UTC (rev 1315)
@@ -4,6 +4,7 @@
#include <boost/function.hpp>
#include <boost/scoped_ptr.hpp>
#include <iostream> // for cerr
+#include <commons/utility.h>
namespace commons
{
@@ -13,7 +14,7 @@
typedef std::function<void()> fn;
- void
+ UNUSED static void
swallow(const fn f) {
try { f(); }
catch (std::exception &ex) { cerr << ex.what() << endl; }
@@ -22,7 +23,7 @@
/**
* Delegate for running a 0-ary void function.
*/
- void
+ UNUSED static void
run_function0(const void *p)
{
scoped_ptr<const fn> pf(reinterpret_cast<const fn*>(p));
@@ -32,7 +33,7 @@
/**
* NULL void*-returning delegate for running a 0-ary void function.
*/
- void*
+ UNUSED static void*
run_function0_null(void* p)
{
run_function0(p);
Modified: cpp-commons/trunk/src/commons/exceptions.h
===================================================================
--- cpp-commons/trunk/src/commons/exceptions.h 2009-03-20 17:45:38 UTC (rev 1314)
+++ cpp-commons/trunk/src/commons/exceptions.h 2009-03-20 20:20:25 UTC (rev 1315)
@@ -39,6 +39,11 @@
const string msg_;
};
+ /**
+ * Convenience class for performing long-jumping break.
+ */
+ class break_exception : public std::exception {};
+
#define throw_not_supported() throw not_supported_exception(__PRETTY_FUNCTION__)
#define throw_not_implemented() throw not_implemented_exception(__PRETTY_FUNCTION__)
Modified: cpp-commons/trunk/src/commons/memory.h
===================================================================
--- cpp-commons/trunk/src/commons/memory.h 2009-03-20 17:45:38 UTC (rev 1314)
+++ cpp-commons/trunk/src/commons/memory.h 2009-03-20 20:20:25 UTC (rev 1315)
@@ -1,6 +1,8 @@
#ifndef COMMONS_MEMORY_H
#define COMMONS_MEMORY_H
+#include <cstring> // for size_t
+
namespace commons
{
Modified: cpp-commons/trunk/src/commons/snap_map.h
===================================================================
--- cpp-commons/trunk/src/commons/snap_map.h 2009-03-20 17:45:38 UTC (rev 1314)
+++ cpp-commons/trunk/src/commons/snap_map.h 2009-03-20 20:20:25 UTC (rev 1315)
@@ -6,6 +6,7 @@
#include <commons/assert.h>
#include <commons/exceptions.h>
#include <utility>
+#include <map>
namespace commons {
@@ -143,7 +144,7 @@
typedef typename traits::const_reference const_reference;
typedef typename traits::pointer pointer;
typedef typename traits::const_pointer const_pointer;
- typedef map<size_t, unique_ptr<array<value_type> > > shadowmap;
+ typedef map<size_t, unique_ptr<commons::array<value_type> > > shadowmap;
private:
static const size_t initsize = 1 << 20, pagesize = 1 << 15, pagemask = pagesize - 1;
Modified: cpp-commons/trunk/src/commons/sockets.h
===================================================================
--- cpp-commons/trunk/src/commons/sockets.h 2009-03-20 17:45:38 UTC (rev 1314)
+++ cpp-commons/trunk/src/commons/sockets.h 2009-03-20 20:20:25 UTC (rev 1315)
@@ -31,7 +31,7 @@
/**
* Make a TCP socket non-blocking.
*/
- int
+ inline int
set_non_blocking(int fd)
{
checknnegerr(fcntl(fd, F_SETFL,
@@ -42,7 +42,7 @@
/**
* Create a TCP socket.
*/
- int
+ inline int
tcp_socket(bool nb)
{
int fd = checknnegerr(socket(PF_INET, SOCK_STREAM, 0));
@@ -63,7 +63,7 @@
/**
* Initialize an inet address with just the port.
*/
- void
+ inline void
sockaddr_init(sockaddr_in &a, uint16_t port)
{
bzero(&a, sizeof a);
@@ -74,7 +74,7 @@
/**
* Initialize an inet address.
*/
- void
+ static void
sockaddr_init(sockaddr_in &a, const char *host, uint16_t port)
{
sockaddr_init(a, port);
@@ -93,7 +93,7 @@
/**
* Initialize an inet address.
*/
- void
+ inline void
sockaddr_init(sockaddr_in &a, in_addr host, uint16_t port)
{
sockaddr_init(a, port);
@@ -104,7 +104,7 @@
* Construct an inet address.
*/
template <typename T>
- sockaddr_in
+ inline sockaddr_in
make_sockaddr(T host, uint16_t port)
{
sockaddr_in a;
@@ -118,7 +118,7 @@
* \param[in] nb Whether the socket should be non-blocking.
* \return The server socket.
*/
- int
+ UNUSED static int
server_socket(uint16_t port, bool nb = false)
{
// Create the socket.
@@ -145,7 +145,7 @@
* \param[in] nb Whether the socket should be non-blocking.
* \return The listener socket.
*/
- int
+ UNUSED static int
tcp_listen(uint16_t port, bool nb = false)
{
int fd = server_socket(port, nb);
@@ -163,7 +163,7 @@
/**
* Connect to a TCP socket.
*/
- int
+ UNUSED static int
tcp_connect(const char *host, uint16_t port)
{
closingfd c(tcp_socket(false));
Modified: cpp-commons/trunk/src/commons/st/st.h
===================================================================
--- cpp-commons/trunk/src/commons/st/st.h 2009-03-20 17:45:38 UTC (rev 1314)
+++ cpp-commons/trunk/src/commons/st/st.h 2009-03-20 20:20:25 UTC (rev 1315)
@@ -50,7 +50,7 @@
st_mutex_t mx_;
};
- void
+ UNUSED static void
st_read(st_netfd_t fd, void *buf, size_t len)
{
checkeqnneg(st_read_fully(fd, buf, len, ST_UTIME_NO_TIMEOUT), ssize_t(len));
@@ -63,7 +63,7 @@
st_read(fd, &x, sizeof x);
}
- void
+ UNUSED static void
st_write(st_netfd_t fd, const void *buf, size_t len)
{
checkeqnneg(st_write(fd, buf, len, ST_UTIME_NO_TIMEOUT), ssize_t(len));
@@ -82,7 +82,7 @@
* \return The new pthread_t on success, 0 on failure.
* \todo Is it safe to treat the pthread_t as a pointer?
*/
- st_thread_t
+ UNUSED static st_thread_t
st_spawn(const fn& f)
{
return st_thread_create(&run_function0_null,
@@ -91,7 +91,7 @@
default_stack_size);
}
- void
+ UNUSED static void
st_join(st_thread_t t)
{
check0x(st_thread_join(t, nullptr));
@@ -103,7 +103,7 @@
* \param[in] port The destination port.
* \param[in] timeout The timeout for the connect operation.
*/
- st_netfd_t
+ UNUSED static st_netfd_t
st_tcp_connect(in_addr host, uint16_t port, st_utime_t timeout)
{
// Create remote socket address.
@@ -125,7 +125,7 @@
* operation.
* \todo Create variants that take and/or return sockaddr_in's.
*/
- st_netfd_t
+ UNUSED static st_netfd_t
st_tcp_connect(const char *host, uint16_t port, st_utime_t timeout)
{
in_addr ipaddr;
@@ -145,7 +145,7 @@
* \param[in] port The port to listen on.
* \return The st_netfd_t.
*/
- st_netfd_t
+ UNUSED static st_netfd_t
st_tcp_listen(uint16_t port)
{
int sfd = tcp_listen(port);
@@ -193,7 +193,7 @@
bool b;
};
- void toggle(st_bool& b) { if (b) b.reset(); else b.set(); }
+ UNUSED static void toggle(st_bool& b) { if (b) b.reset(); else b.set(); }
/**
* Wraps st_mutex_* errno-functions with exceptions and cleans up on
Modified: cpp-commons/trunk/src/commons/time.h
===================================================================
--- cpp-commons/trunk/src/commons/time.h 2009-03-20 17:45:38 UTC (rev 1314)
+++ cpp-commons/trunk/src/commons/time.h 2009-03-20 20:20:25 UTC (rev 1315)
@@ -32,7 +32,7 @@
class timer
{
public:
- timer(const string label) :
+ timer(const string &label) :
label(label), start(current_time_millis()), last(start) {}
void print()
{
Modified: cpp-commons/trunk/src/commons/utility.h
===================================================================
--- cpp-commons/trunk/src/commons/utility.h 2009-03-20 17:45:38 UTC (rev 1314)
+++ cpp-commons/trunk/src/commons/utility.h 2009-03-20 20:20:25 UTC (rev 1315)
@@ -15,4 +15,6 @@
#define BEGIN_NAMESPACE(ns) namespace ns {
#define END_NAMESPACE }
+#define UNUSED __attribute__((unused))
+
#endif
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <yan...@us...> - 2009-03-20 17:45:49
|
Revision: 1314
http://assorted.svn.sourceforge.net/assorted/?rev=1314&view=rev
Author: yangzhang
Date: 2009-03-20 17:45:38 +0000 (Fri, 20 Mar 2009)
Log Message:
-----------
- added precompiled headers
- renamed main2 to ydb
- moved some more code around
Modified Paths:
--------------
ydb/trunk/src/Makefile
ydb/trunk/src/main.lzz.clamp
ydb/trunk/src/msg.h
ydb/trunk/src/stxn.lzz.clamp
Added Paths:
-----------
ydb/trunk/src/ydb.lzz.clamp
Removed Paths:
-------------
ydb/trunk/src/main2.lzz.clamp
Modified: ydb/trunk/src/Makefile
===================================================================
--- ydb/trunk/src/Makefile 2009-03-20 07:58:28 UTC (rev 1313)
+++ ydb/trunk/src/Makefile 2009-03-20 17:45:38 UTC (rev 1314)
@@ -1,7 +1,9 @@
TARGET := ydb
WTF := wtf
-LZZS := $(patsubst %.clamp,%,$(wildcard *.lzz.clamp))
+CLAMPS := $(wildcard *.lzz.clamp)
+PURELZZS := $(foreach lzz,$(wildcard *.lzz),$(if $(wildcard $(lzz).clamp),,$(lzz)))
+LZZS := $(patsubst %.clamp,%,$(CLAMPS)) $(PURELZZS)
LZZHDRS := $(foreach lzz,$(LZZS),$(patsubst %.lzz,%.hh,$(lzz)))
LZZSRCS := $(foreach lzz,$(LZZS),$(patsubst %.lzz,%.cc,$(lzz)))
LZZOBJS := $(foreach lzz,$(LZZS),$(patsubst %.lzz,%.o,$(lzz)))
@@ -38,13 +40,13 @@
endif
# CXX := $(WTF) ag++ -k --Xcompiler # $(CXX)
CXX := $(WTF) ccache $(CXX) -pipe
-LD := $(CXX)
+CC := $(CXX) # for linking
LDFLAGS := -pthread $(GPROF)
LDLIBS := -lstx -lst -lresolv -lprotobuf -lgtest \
-lboost_program_options-gcc43-mt -lboost_thread-gcc43-mt \
-lboost_serialization-gcc43-mt $(PPROF)
-CXXFLAGS := $(OPT) -pthread $(GPROF) \
+CXXFLAGS0 := $(OPT) -pthread $(GPROF) \
-Wall \
-Werror \
-Wextra \
@@ -77,6 +79,7 @@
-std=gnu++0x \
-march=native \
$(CXXFLAGS)
+CXXFLAGS := $(CXXFLAGS0) -include pch.h
# \
-Wmissing-noreturn \
@@ -97,12 +100,13 @@
%.pb.o: %.pb.cc %.pb.h
$(CXX) -c $(PBCXXFLAGS) $(OUTPUT_OPTION) $<
+%.o: pch.h.gch
stxn.o: main.hh $(PBHDRS)
main.o: util.hh msg.h $(PBHDRS)
util.o: msg.h $(PBHDRS)
-main2.o: main.hh stxn.hh tpcc.hh $(PBHDRS)
+ydb.o: main.hh stxn.hh tpcc.hh util.hh $(PBHDRS)
tpcc.o: main.hh util.hh $(PBHDRS)
-ydb: main.o main2.o util.o tpcc.o stxn.o
+ydb: ydb.o tpcc.o main.o util.o stxn.o ydb.pb.o $(TPCC_OBJS) # $(OBJS)
tpcc/%.o: tpcc/%.cc
make -C tpcc/
@@ -126,23 +130,27 @@
mkdir -p clamp/
clamp --outdir clamp/ --prefix $(basename $@) < $< | \
sed "$$( echo -e '1i\\\n\#hdr\n1a\\\n\#end' )" | \
- sed "$$( echo -e '$$i\\\n\#src\n$$a\\\n\#end' )" > $@
+ sed "$$( echo -e '$$i\\\n\#hdr\n$$a\\\n\#end' )" > $@
chmod -w $@
-all.h:
- fgrep '#include' main.lzz.clamp > all.h
+pch.h:
+ svn ls -rHEAD -R | \
+ grep -v '/$$' | \
+ xargs sed 's/.*\binclude\b *<\(.*\)>.*/\#include <\1>/; t succ; d; :succ /commons/ d' | \
+ sort -u > $@
-all.h.gch: all.h
- $(COMPILE.cc) $(PBHDRS) $(OUTPUT_OPTION) $<
+%.h.gch: CXXFLAGS = $(CXXFLAGS0)
+%.h.gch: %.h
+ $(LINK.cc) $(OUTPUT_OPTION) $<
clean:
rm -rf clamp/ $(GENSRCS) $(GENHDRS) $(OBJS) $(TARGET) \
- main.lzz main2.lzz main.cc main.hh main2.hh main2.cc \
+ main.lzz ydb.lzz main.cc main.hh ydb.hh ydb.cc \
util.cc util.hh tpcc.lzz tpcc.hh tpcc.cc
make -C tpcc/ clean
distclean: clean
- rm -f all.h all.h.gch
+ rm -f pch.h pch.h.gch
doc: $(SRCS) $(HDRS)
doxygen
Modified: ydb/trunk/src/main.lzz.clamp
===================================================================
--- ydb/trunk/src/main.lzz.clamp 2009-03-20 07:58:28 UTC (rev 1313)
+++ ydb/trunk/src/main.lzz.clamp 2009-03-20 17:45:38 UTC (rev 1314)
@@ -205,180 +205,6 @@
txn_wal *g_twal;
//tpcc_wal *g_tpcc_wal;
-class response_handler
-{
-public:
- response_handler(st_netfd_t replica, const int &seqno, int rid,
- st_multichannel<long long> &recover_signals, bool caught_up)
- :
- replica(replica),
- seqno(seqno),
- rid(rid),
- recover_signals(recover_signals),
- caught_up(caught_up),
- sub(recover_signals.subscribe()),
- start_time(current_time_millis()),
- recovery_start_time(caught_up ? -1 : start_time),
- recovery_end_time(-1),
- start_seqno(seqno),
- recovery_start_seqno(caught_up ? -1 : seqno),
- recovery_end_seqno(-1),
- last_seqno(-1)
- {}
-
- template<typename Types>
- void run() {
- typedef typename Types::Response Response;
- typedef typename Types::ResponseBatch ResponseBatch;
-
- finally f(bind(&response_handler::cleanup, this));
-
- commons::array<char> rbuf(read_buf_size), wbuf(buf_size);
- st_reader reader(replica, rbuf.get(), rbuf.size());
- writer w(lambda(const void*, size_t) {
- throw not_supported_exception("response handler should not be writing");
- }, wbuf.get(), wbuf.size());
- stream s(reader,w);
-
- scoped_ptr<ResponseBatch> pbatch(new_ResponseBatch<ResponseBatch>(s));
- ResponseBatch &batch = *pbatch;
-
- long long last_display_time = current_time_millis();
-
- function<void()> loop_cleanup =
- bind(&response_handler::loop_cleanup, this);
-
- while (true) {
- finally f(loop_cleanup);
- uint32_t prefix = 0;
-
- // Read the message, but correctly respond to interrupts so that we can
- // cleanly exit (slightly tricky).
- if (last_seqno + 1 == seqno) {
- // Stop-interruptible in case we're already caught up.
- try {
- st_intr intr(stop_hub);
- if (Types::is_pb()) readmsg(reader, batch);
- else { prefix = ntohl(reader.read<uint32_t>()); batch.Clear(); }
- } catch (...) { // TODO: only catch interruptions
- // This check on seqnos is OK for termination since the seqno will
- // never grow again if stop_hub is set.
- if (last_seqno + 1 == seqno) {
- cout << rid << ": ";
- cout << "clean stop; next expected seqno is " << seqno
- << " (last seqno was " << last_seqno << ")" << endl;
- break;
- } else {
- continue;
- }
- }
- } else {
- // Only kill-interruptible because we want a clean termination (want
- // to get all the acks back).
- st_intr intr(kill_hub);
- if (Types::is_pb()) readmsg(reader, batch);
- else { prefix = ntohl(reader.read<uint32_t>()); batch.Clear(); }
- }
-
- for (int i = 0; i < batch.res_size(); ++i) {
- const Response &res = batch.res(i);
- // Determine if this response handler's host (the only joiner) has finished
- // catching up. If it has, then broadcast a signal so that all response
- // handlers will know about this event.
- int rseqno = res.seqno();
- if (rseqno <= last_seqno)
- throw msg_exception(string("response seqno decreased from ") +
- lexical_cast<string>(last_seqno) + " to " +
- lexical_cast<string>(rseqno));
- bool rcaught_up = res.caught_up();
- for (int r = 0; r < res.result_size(); ++r) {
- cout << rseqno << last_seqno << res.result_size() << " " << r << " " << res.result(r) << endl;
- }
- if (!caught_up && rcaught_up) {
- long long now = current_time_millis(), time_diff = now - start_time;
- caught_up = true;
- recover_signals.push(now);
- cout << rid << ": ";
- cout << "recovering node caught up; took "
- << time_diff << " ms" << endl;
- // This will cause the program to exit eventually, but cleanly, such that
- // the recovery time will be set first, before the eventual exit (which
- // may not even happen in the current iteration).
- if (stop_on_recovery) {
- cout << "stopping on recovery" << endl;
- stop_hub.set();
- }
- }
- if (check_interval(rseqno, handle_responses_display)) {
- cout << rid << ": " << "got response " << rseqno << " from "
- << replica << "; ";
- long long display_time = current_time_millis();
- showtput("handling", display_time, last_display_time, rseqno,
- rseqno - handle_responses_display);
- last_display_time = display_time;
- }
- if (check_interval(rseqno, yield_interval)) {
- st_sleep(0);
- }
- last_seqno = rseqno;
- }
- }
- }
-
-private:
- void loop_cleanup() {
- // The first timestamp that comes down the subscription pipeline is the
- // recovery start time, issued by the main thread. The second one is the
- // recovery end time, issued by the response handler associated with the
- // joiner.
- if (recovery_start_time == -1 && !sub.empty()) {
- recovery_start_time = sub.take();
- recovery_start_seqno = last_seqno;
- cout << rid << ": ";
- showtput("before recovery, finished", recovery_start_time, start_time,
- recovery_start_seqno, 0);
- } else if (recovery_end_time == -1 && !sub.empty()) {
- recovery_end_time = sub.take();
- recovery_end_seqno = last_seqno;
- cout << rid << ": ";
- showtput("during recovery, finished roughly", recovery_end_time,
- recovery_start_time, recovery_end_seqno, recovery_start_seqno);
- }
- }
-
- void cleanup() {
- long long end_time = current_time_millis();
- cout << rid << ": ";
- showtput("handled roughly", end_time, start_time, seqno, start_seqno);
- if (recovery_end_time > -1) {
- cout << rid << ": ";
- showtput("after recovery, finished", end_time, recovery_end_time,
- seqno, recovery_end_seqno);
- }
- }
-
- st_netfd_t replica;
- const int &seqno;
- int rid;
- st_multichannel<long long> &recover_signals;
- bool caught_up;
- st_channel<long long> ⊂
- long long start_time, recovery_start_time, recovery_end_time;
- int start_seqno, recovery_start_seqno, recovery_end_seqno, last_seqno;
-};
-
-/**
- * Swallow replica responses.
- */
-template<typename Types>
-void
-handle_responses(st_netfd_t replica, const int &seqno, int rid,
- st_multichannel<long long> &recover_signals, bool caught_up)
-{
- response_handler h(replica, seqno, rid, recover_signals, caught_up);
- h.run<Types>();
-}
-
struct recreq {
int start_seqno, end_seqno;
};
Deleted: ydb/trunk/src/main2.lzz.clamp
===================================================================
--- ydb/trunk/src/main2.lzz.clamp 2009-03-20 07:58:28 UTC (rev 1313)
+++ ydb/trunk/src/main2.lzz.clamp 2009-03-20 17:45:38 UTC (rev 1314)
@@ -1,1073 +0,0 @@
-#hdr
-#include "unsetprefs.h"
-#include <boost/function.hpp>
-#include <boost/scoped_ptr.hpp>
-#include <string>
-#include <iostream>
-#include <st.h>
-#include <commons/st/st.h>
-#include "tpcc/clock.h"
-#include "tpcc/randomgenerator.h"
-#include "tpcc/tpccclient.h"
-#include "tpcc/tpccgenerator.h"
-#include "tpcc/tpcctables.h"
-#include "util.hh"
-#include "tpcc.hh"
-#include "stxn.hh"
-#include "main.hh"
-
-using namespace boost;
-using namespace std;
-using namespace commons;
-#end
-
-#src
-#include "unsetprefs.h"
-#include <csignal> // sigaction, etc.
-#include <cstring> // strsignal, size_t
-#include <boost/program_options.hpp>
-#include <gtest/gtest.h>
-#include <malloc.h>
-#include <string>
-#include "setprefs.h"
-#end
-
-using namespace google;
-using namespace testing;
-
-//
-// Utilities/system
-//
-
-/**
- * Delegate for running thread targets.
- * \param[in] f The function to execute.
- * \param[in] intr Whether to signal stop_hub on an exception.
- */
-void
-my_spawn_helper(const function0<void> f, bool intr)
-{
- thread_eraser eraser;
- try {
- f();
- } catch (std::exception &ex) {
- cerr_thread_ex(ex) << (intr ? "; interrupting!" : "") << endl;
- if (intr) stop_hub.set();
- }
-}
-
-/**
- * Spawn a thread using ST but wrap it in an exception handler that interrupts
- * all other threads (hopefully causing them to unwind).
- * \param[in] f The function to execute.
- * \param[in] intr Whether to signal stop_hub on an exception. Not actually
- * used anywhere.
- */
-st_thread_t
-my_spawn(const function0<void> &f, string name, bool intr = false)
-{
- st_thread_t t = st_spawn(bind(my_spawn_helper, f, intr));
- threads.insert(t);
- threadnames[t] = name;
- return t;
-}
-
-/**
- * Memory monitor.
- */
-void
-memmon()
-{
- while (!stop_hub) {
- {
- st_intr intr(stop_hub);
- st_sleep(1);
- }
- malloc_stats();
- }
-}
-
-int sig_pipe[2];
-
-//
-// Signals
-//
-
-/**
- * Raw signal handler that triggers the (synchronous) handler.
- */
-void handle_sig(int sig) {
- int err = errno;
- cerr << "got signal: " << strsignal(sig) << " (" << sig << ")" << endl;
- checkeqnneg(write(sig_pipe[1], &sig, sizeof sig),
- static_cast<ssize_t>(sizeof sig));
- errno = err;
-}
-
-/**
- * Synchronous part of the signal handler; cleanly interrrupts any threads that
- * have marked themselves as interruptible.
- */
-void handle_sig_sync() {
- st_closing fd(checkerr(st_netfd_open(sig_pipe[0])));
- while (true) {
- int sig;
- checkeqnneg(st_read(fd, &sig, sizeof sig, ST_UTIME_NO_TIMEOUT),
- static_cast<ssize_t>(sizeof sig));
- if (sig == SIGINT) {
- if (!stop_hub) stop_hub.set();
- else kill_hub.set();
- } else if (sig == SIGTERM) {
- foreach (st_thread_t t, threads) {
- st_thread_interrupt(t);
- }
- } else if (sig == SIGUSR1) {
- toggle(do_pause);
- }
- }
-}
-
-//
-// Main
-//
-
-/**
- * Initialization and command-line parsing.
- */
-int
-main(int argc, char **argv)
-{
- namespace po = boost::program_options;
- try {
- GOOGLE_PROTOBUF_VERIFY_VERSION;
-
- bool is_leader, use_epoll;
- int minreps;
- uint16_t leader_port, listen_port;
- string leader_host;
-
- // Parse options.
- po::options_description desc("Allowed options");
- desc.add_options()
- ("help,h", "show this help message")
- ("debug-memory,M", po::bool_switch(&debug_memory),
- "enable memory monitoring/debug outputs")
- ("debug-threads,d",po::bool_switch(&debug_threads),
- "enable context switch debug outputs")
- ("profile-threads,q",po::bool_switch(&profile_threads),
- "enable profiling of threads")
- ("epoll,e", po::bool_switch(&use_epoll),
- "use epoll (select is used by default)")
- ("yield-build-up", po::bool_switch(&yield_during_build_up),
- "yield periodically during build-up phase of recovery (for recoverer)")
- ("yield-catch-up", po::bool_switch(&yield_during_catch_up),
- "yield periodically during catch-up phase of recovery (for recoverer)")
- ("multirecover,m", po::bool_switch(&multirecover),
- "recover from multiple hosts, instead of just one (specified via leader)")
- ("rec-twal", po::bool_switch(&rec_twal),
- "recover from twal")
- ("rec-pwal", po::bool_switch(&rec_pwal),
- "recover from pwal")
- ("disk,k", po::bool_switch(&disk),
- "use disk-based recovery")
- ("ship-log", po::bool_switch(&ship_log),
- "ship the log instead of the complete database state")
- ("dump,D", po::bool_switch(&dump),
- "replicas should finally dump their state to a tmp file for "
- "inspection/diffing")
- ("suppress-txn-msgs", po::bool_switch(&suppress_txn_msgs),
- "suppress txn msgs")
- ("fake-exec", po::bool_switch(&fake_exec),
- "don't actually execute txns")
- ("fake-bcast", po::bool_switch(&fake_bcast),
- "when using --bcast-async, don't actually perform the socket write")
- ("show-updates,U", po::bool_switch(&show_updates),
- "log operations that touch (update/read/delete) an existing key")
- ("count-updates,u",po::bool_switch(&count_updates),
- "count operations that touch (update/read/delete) an existing key")
- ("general-txns,g", po::bool_switch(&general_txns),
- "issue read and delete transactions as well as the default of (only) insertion/update transactions (for leader)")
- ("use-pb", po::bool_switch(&use_pb),
- "use protocol buffers instead of raw buffers for txns")
- ("use-pb-res", po::bool_switch(&use_pb_res),
- "use protocol buffers instead of raw buffers for responses")
- ("twal", po::bool_switch(&use_twal),
- "enable transactional write-ahead logging")
- ("pwal", po::bool_switch(&use_pwal),
- "enable physical write-ahead logging")
- ("force-ser", po::bool_switch(&force_ser),
- "force issue_txns to serialize its Txns")
- ("leader,l", po::bool_switch(&is_leader),
- "run the leader (run replica by default)")
- ("exit-on-recovery,x", po::bool_switch(&stop_on_recovery),
- "exit after the joiner fully recovers (for leader)")
- ("batch-size,b", po::value<int>(&batch_size)->default_value(100),
- "number of txns to batch up in each msg (for leader)")
- ("tpcc", po::bool_switch(&do_tpcc),
- "run the TPCC workload")
- ("exit-on-seqno,X",po::value<int>(&stop_on_seqno)->default_value(-1),
- "exit after txn seqno is issued (for leader)")
- ("accept-joiner-size,s",
- po::value<size_t>(&accept_joiner_size)->default_value(0),
- "accept recovering joiner (start recovery) after DB grows to this size "
- "(for leader)")
- ("handle-responses-display",
- po::value<int>(&handle_responses_display)->default_value(0),
- "number of responses before printing current handling rate (for leader)")
- ("catch-up-display",
- po::value<int>(&catch_up_display)->default_value(0),
- "number of catch-up txns before printing current recovery rate and queue length (for recoverer)")
- ("issue-display",
- po::value<int>(&issue_display)->default_value(0),
- "number of txns before showing the current issue rate (for leader)")
- ("process-display",
- po::value<int>(&process_display)->default_value(0),
- "number of txns before showing the current issue rate (for worker)")
- ("issuing-interval",
- po::value<int>(&issuing_interval)->default_value(0),
- "seconds to sleep between issuing txns (for leader)")
- ("min-ops,o",
- po::value<int>(&min_ops)->default_value(5),
- "lower bound on randomly generated number of operations per txn (for leader)")
- ("max-ops,O",
- po::value<int>(&max_ops)->default_value(5),
- "upper bound on randomly generated number of operations per txn (for leader)")
- ("warehouses,w",
- po::value<int>(&nwarehouses)->default_value(1),
- "number of warehouses to model")
- ("fail-seqno",
- po::value<int>(&fail_seqno)->default_value(0),
- "fail after processing this seqno (for replica only)")
- ("accept-joiner-seqno,j",
- po::value<int>(&accept_joiner_seqno)->default_value(0),
- "accept recovering joiner (start recovery) after this seqno (for leader "
- "only)")
- ("leader-host,H",
- po::value<string>(&leader_host)->default_value(string("localhost")),
- "hostname or address of the leader")
- ("leader-port,P",
- po::value<uint16_t>(&leader_port)->default_value(7654),
- "port the leader listens on")
- ("read-buf", po::value<size_t>(&read_buf_size)->default_value(1e7),
- "size of the incoming (read) buffer in bytes")
- ("write-buf", po::value<size_t>(&buf_size)->default_value(1e5),
- "size of the outgoing (write) buffer in bytes")
- ("yield-interval,y", po::value<int>(&yield_interval)->default_value(1000),
- "number of txns before yielding")
- ("timelim,T", po::value<long long>(&timelim)->default_value(0),
- "general network IO time limit in milliseconds, or 0 for none")
- ("write-thresh,w", po::value<long long>(&write_thresh)->default_value(200),
- "if positive and any txn write exceeds this, then print a message")
- ("read-thresh,r", po::value<long long>(&read_thresh)->default_value(0),
- "if positive and any txn read exceeds this, then print a message")
- ("listen-port,p", po::value<uint16_t>(&listen_port)->default_value(7654),
- "port to listen on (for worker)")
- ("timeout,t", po::value<st_utime_t>(&timeout)->default_value(200000),
- "timeout for some IO operations that should actually time out (in microseconds)")
- ("test", "execute unit tests instead of running the normal system")
- ("minreps,n", po::value<int>(&minreps)->default_value(2),
- "minimum number of replicas the system is willing to process txns on");
-
- po::variables_map vm;
- try {
- po::store(po::parse_command_line(argc, argv, desc), vm);
- po::notify(vm);
-
- if (vm.count("help")) {
- cout << desc << endl;
- return 0;
- }
-
- // Validate arguments.
- check(min_ops > 0);
- check(max_ops > 0);
- check(max_ops >= min_ops);
- } catch (std::exception &ex) {
- cerr << ex.what() << endl << endl << desc << endl;
- return 1;
- }
-
- // Run unit-tests.
- if (vm.count("test")) {
- InitGoogleTest(&argc, argv);
- return RUN_ALL_TESTS();
- }
-
- // Initialize support for ST working with asynchronous signals.
- check0x(pipe(sig_pipe));
- struct sigaction sa;
- sa.sa_handler = handle_sig;
- check0x(sigemptyset(&sa.sa_mask));
- sa.sa_flags = 0;
- check0x(sigaction(SIGINT, &sa, nullptr));
- check0x(sigaction(SIGTERM, &sa, nullptr));
- check0x(sigaction(SIGUSR1, &sa, nullptr));
-
- // Initialize ST.
- if (use_epoll) check0x(st_set_eventsys(ST_EVENTSYS_ALT));
- check0x(st_init());
- my_spawn(bind(handle_sig_sync), "handle_sig_sync");
- if (debug_threads || profile_threads) {
- st_set_switch_out_cb(switch_out_cb);
- st_set_switch_in_cb(switch_in_cb);
- }
-
- // Initialize thread manager for clean shutdown of all threads.
- thread_eraser eraser;
- threads.insert(st_thread_self());
- threadnames[st_thread_self()] = "main";
-
- // Print memory debugging information.
- if (debug_memory) {
- my_spawn(memmon, "memmon");
- }
-
- long long start = thread_start_time = current_time_millis();
-
- // At the end, cleanly stop the bcaster thread and print thread profiling
- // information.
- finally f(lambda() {
- if (profile_threads) {
- long long end = current_time_millis();
- long long all = end - __ref(start);
- cout << "thread profiling results:" << endl;
- long long total = 0;
- typedef pair<st_thread_t, long long> entry;
- foreach (entry p, threadtimes) {
- total += p.second;
- }
- foreach (entry p, threadtimes) {
- cout << "- " << threadname(p.first) << ": " << p.second << " ms ("
- << pct(p.second, total) << "% of total, "
- << pct(p.second, all) << "% of all)" << endl;
- }
- cout << "- total: " << total << " ms (" << pct(total, all)
- << "% of all)" << endl;
- cout << "- unaccounted: " << all - total << " ms ("
- << pct(all - total, all) << "% of all)" << endl;
- cout << "- all: " << all << " ms" << endl;
- }
- });
-
- // Initialize the map.
- init_map(g_map);
-
- cout << "pid " << getpid() << endl;
-
- // Which role are we?
- if (is_leader) {
- if (use_pb) {
- if (use_pb_res) {
- run_leader<pb_traits, pb_traits>(minreps, leader_port);
- } else {
- run_leader<pb_traits, rb_traits>(minreps, leader_port);
- }
- } else {
- if (use_pb_res) {
- run_leader<rb_traits, pb_traits>(minreps, leader_port);
- } else {
- run_leader<rb_traits, rb_traits>(minreps, leader_port);
- }
- }
- } else {
- if (use_pb) {
- if (use_pb_res) {
- run_replica<pb_traits, pb_traits>(leader_host, leader_port, listen_port);
- } else {
- run_replica<pb_traits, rb_traits>(leader_host, leader_port, listen_port);
- }
- } else {
- if (use_pb_res) {
- run_replica<rb_traits, pb_traits>(leader_host, leader_port, listen_port);
- } else {
- run_replica<rb_traits, rb_traits>(leader_host, leader_port, listen_port);
- }
- }
- }
-
- return 0;
- } catch (std::exception &ex) {
- // Must catch all exceptions at the top to make the stack unwind.
- cerr_thread_ex(ex) << endl;
- return 1;
- }
-}
-
-/**
- * Run the leader.
- */
-template<typename Types, typename RTypes>
-void
-run_leader(int minreps, uint16_t leader_port)
-{
- cout << "starting as leader" << endl;
- st_multichannel<long long> recover_signals;
-
- scoped_ptr<txn_wal> twal(new txn_wal(use_twal ? "twal" : "/dev/null"));
- g_twal = twal.get();
- scoped_ptr<wal> pwal(new wal(use_pwal ? "pwal" : "/dev/null"));
- g_wal = pwal.get();
-
- // Wait until all replicas have joined.
- st_netfd_t listener = st_tcp_listen(leader_port);
- st_closing close_listener(listener);
- vector<replica_info> replicas;
- st_closing_all_infos close_replicas(replicas);
- cout << "waiting for at least " << minreps << " replicas to join" << endl;
- for (int i = 0; i < minreps; ++i) {
- st_netfd_t fd;
- {
- st_intr intr(stop_hub);
- fd = checkerr(st_accept(listener, nullptr, nullptr,
- ST_UTIME_NO_TIMEOUT));
- }
- Join join = readmsg<Join>(fd);
- replicas.push_back(replica_info(fd, static_cast<uint16_t>(join.port())));
- }
- cout << "got all " << minreps << " replicas" << endl;
-
- // Construct the initialization message.
- Init init;
- init.set_txnseqno(0);
- init.set_multirecover(multirecover);
- foreach (replica_info r, replicas) {
- SockAddr *psa = init.add_node();
- psa->set_host(r.host());
- psa->set_port(r.port());
- }
-
- // Send init to each initial replica.
- foreach (replica_info r, replicas) {
- init.set_yourhost(r.host());
- sendmsg(r.fd(), init);
- }
-
- // Start dispatching queries.
- st_bool accept_joiner;
- int seqno = 0;
- st_channel<replica_info> newreps;
- st_channel<st_netfd_t> delreps;
- foreach (const replica_info &r, replicas) newreps.push(r);
- function<void()> f;
- if (do_tpcc)
- f = bind(issue_tpcc, ref(newreps), ref(delreps), ref(seqno), ref(accept_joiner));
- else
- f = bind(issue_txns<Types>, ref(newreps), ref(seqno), ref(accept_joiner));
- st_joining join_issue_txns(my_spawn(f, "issue_txns"));
-
- finally fin(bind(summarize, "LEADER", ref(seqno)));
-
- try {
- // Start handling responses.
- st_thread_group handlers;
- int rid = 0;
- foreach (replica_info r, replicas) {
- function<void()> fn;
- if (do_tpcc)
- fn = bind(handle_tpcc_responses, r.fd(), ref(seqno), rid++,
- ref(recover_signals), ref(delreps), true);
- else
- fn = bind(handle_responses<RTypes>, r.fd(), ref(seqno), rid++,
- ref(recover_signals), true);
- handlers.insert(my_spawn(fn, "handle_responses"));
- }
-
- // Accept the recovering node, and tell it about the online replicas.
- st_netfd_t joiner;
- try {
- st_intr intr(stop_hub);
- joiner = checkerr(st_accept(listener, nullptr, nullptr,
- ST_UTIME_NO_TIMEOUT));
- accept_joiner.waitset();
- } catch (std::exception &ex) {
- string s(ex.what());
- if (s.find("Interrupted system call") == s.npos)
- throw;
- else
- throw break_exception();
- }
- Join join = readmsg<Join>(joiner);
- replicas.push_back(replica_info(joiner, static_cast<uint16_t>(join.port())));
- cout << "setting seqno to " << seqno << endl;
- init.set_txnseqno(seqno);
- init.set_yourhost(replicas.back().host());
- sendmsg(joiner, init);
- recover_signals.push(current_time_millis());
-
- // Start streaming txns to joiner.
- cout << "start streaming txns to joiner" << endl;
- function<void()> handle_responses_joiner_fn;
- if (do_tpcc)
- handle_responses_joiner_fn =
- bind(handle_tpcc_responses, joiner, ref(seqno), rid++,
- ref(recover_signals), ref(delreps), false);
- else
- handle_responses_joiner_fn =
- bind(handle_responses<RTypes>, joiner, ref(seqno), rid++,
- ref(recover_signals), false);
- newreps.push(replicas.back());
- handlers.insert(my_spawn(handle_responses_joiner_fn,
- "handle_responses_joiner"));
- } catch (break_exception &ex) {
- } catch (std::exception &ex) {
- // TODO: maybe there's a cleaner way to do this final step before waiting with the join
- cerr_thread_ex(ex) << endl;
- throw;
- }
-}
-
-void
-summarize(const char *role, int seqno)
-{
- cout << role << " SUMMARY\n";
- if (do_tpcc) {
- cout << "seqno: " << seqno << endl;
- if (g_tables != nullptr) {
- cout << "state:\n";
- g_tables->show();
- string fname = string("/tmp/ydb") + lexical_cast<string>(getpid());
- if (dump) {
- // XXX iterate & dump
- }
- }
- } else {
- cout << "- total updates = " << updates << "\n"
- << "- final DB state: seqno = " << seqno << ", size = "
- << g_map.size() << endl;
- string fname = string("/tmp/ydb") + lexical_cast<string>(getpid());
- if (dump) {
- cout << "- dumping to " << fname << endl;
- ofstream of(fname.c_str());
- of << "seqno: " << seqno << endl;
- foreach (const entry &p, g_map) {
- of << p.first << ": " << p.second << endl;
- }
- }
- }
-}
-
-/**
- * Run a replica.
- */
-template<typename Types, typename RTypes>
-void
-run_replica(string leader_host, uint16_t leader_port, uint16_t listen_port)
-{
- if (disk) {
- // Disk IO threads.
- for (int i = 0; i < 5; ++i) {
- //thread somethread(threadfunc);
- }
- }
-
- // Initialize database state.
- int seqno = -1;
- mii &map = g_map;
- commons::array<char> recarr(0);
- if (do_tpcc) {
- TPCCTables *tables = new TPCCTables();
- g_tables.reset(tables);
- SystemClock* clock = new SystemClock();
-
- // Create a generator for filling the database.
- RealRandomGenerator* random = new RealRandomGenerator();
- NURandC cLoad = NURandC::makeRandom(random);
- random->setC(cLoad);
-
- // Generate the data
- cout << "loading " << nwarehouses << " warehouses" << endl;
- char now[Clock::DATETIME_SIZE+1];
- clock->getDateTimestamp(now);
- TPCCGenerator generator(random, now, Item::NUM_ITEMS,
- District::NUM_PER_WAREHOUSE,
- Customer::NUM_PER_DISTRICT,
- NewOrder::INITIAL_NUM_PER_DISTRICT);
- long long start_time = current_time_millis();
- generator.makeItemsTable(tables);
- for (int i = 0; i < nwarehouses; ++i) {
- generator.makeWarehouse(tables, i+1);
- }
- cout << "loaded " << nwarehouses << " warehouses in "
- << current_time_millis() - start_time << " ms" << endl;
- tables->show();
- }
- recovery_t orig = rec_twal ? g_tables->ser(0, 0, seqno) : recovery_t();
-
- finally f(bind(summarize, "REPLICA", ref(seqno)));
- st_channel<recovery_t> send_states;
-
- cout << "starting as replica on port " << listen_port << endl;
-
- // Listen for connections from other replicas.
- st_netfd_t listener = st_tcp_listen(listen_port);
-
- // Connect to the leader and join the system.
- st_netfd_t leader = st_tcp_connect(leader_host.c_str(), leader_port,
- timeout);
- st_closing closing(leader);
- Join join;
- join.set_port(listen_port);
- sendmsg(leader, join);
- Init init;
- {
- st_intr intr(stop_hub);
- readmsg(leader, init);
- }
- uint32_t listen_host = init.yourhost();
- multirecover = init.multirecover();
-
- // Display the info.
- cout << "got init msg with txn seqno " << init.txnseqno()
- << " and hosts:" << endl;
- vector<st_netfd_t> replicas;
- st_closing_all close_replicas(replicas);
- int mypos = -1;
- for (int i = 0; i < init.node_size(); ++i) {
- const SockAddr &sa = init.node(i);
- char buf[INET_ADDRSTRLEN];
- in_addr host = { sa.host() };
- bool is_self = sa.host() == listen_host && sa.port() == listen_port;
- cout << "- " << checkerr(inet_ntop(AF_INET, &host, buf,
- INET_ADDRSTRLEN))
- << ':' << sa.port() << (is_self ? " (self)" : "") << endl;
- if (is_self) mypos = i;
- if (!is_self && (init.txnseqno() > 0 || rec_twal)) {
- replicas.push_back(st_tcp_connect(host,
- static_cast<uint16_t>(sa.port()),
- timeout));
- }
- }
-
- // Initialize physical or txn log.
- scoped_ptr<txn_wal> twal(new txn_wal(use_twal ? "twal" : "/dev/null"));
- g_twal = twal.get();
- scoped_ptr<wal> pwal(new wal(use_pwal ? "pwal" : "/dev/null"));
- g_wal = pwal.get();
-
- // Process txns.
- st_channel<chunk> backlog;
- function<void()> process_fn;
- if (do_tpcc)
- process_fn = bind(process_tpccs, leader, ref(seqno), ref(send_states),
- ref(backlog), init.txnseqno(), mypos, init.node_size());
- else
- process_fn = bind(process_txns<Types, RTypes>, leader, ref(map), ref(seqno),
- ref(send_states), ref(backlog), init.txnseqno(), mypos,
- init.node_size());
- st_joining join_proc(my_spawn(process_fn, "process_txns"));
- st_joining join_rec(init.txnseqno() == 0 && (multirecover || mypos == 0) ?
- my_spawn(bind(recover_joiner, listener, ref(send_states)),
- "recover_joiner") :
- nullptr);
-
- try {
- // If there's anything to recover.
- if (init.txnseqno() > 0 || fail_seqno > 0) {
- if (do_tpcc) {
-
- //
- // TPCC txns
- //
-
- function<void()> rec_twal_fn = lambda() {
- int &seqno = __ref(seqno);
- cout << "recovering from twal" << endl;
- long long start_time = current_time_millis();
- g_twal->flush();
- sync();
- ifstream inf("twal");
- TpccReq req;
- while (inf.peek() != ifstream::traits_type::eof()) {
- ASSERT(inf.good());
- readmsg(inf, req);
- process_tpcc(req, seqno, nullptr);
- if (check_interval(seqno, yield_interval)) st_sleep(0);
- }
- showdatarate("recovered from twal", inf.tellg(),
- current_time_millis() - start_time);
- cout << "now at seqno " << seqno << endl;
- };
-
- function<void()> recv_log_fn = lambda() {
- st_netfd_t src = __ref(replicas[0]);
- int &seqno = __ref(seqno);
- ASSERT(fail_seqno == seqno);
- recreq r = { fail_seqno + 1, resume.take() };
- st_write(src, r);
- sized_array<char> rbuf(new char[read_buf_size], read_buf_size);
- function<void(anchored_stream_reader &reader)> overflow_fn =
- lambda(anchored_stream_reader &reader) {
- shift_reader(reader);
- };
- anchored_stream_reader reader(st_read_fn(src),
- st_read_fully_fn(src),
- overflow_fn, rbuf.get(), rbuf.size());
- TpccReq req;
- while (seqno < r.end_seqno) {
- { st_intr intr(stop_hub); readmsg(reader, req); }
- process_tpcc(req, seqno, nullptr);
- reader.set_anchor();
- if (check_interval(seqno, yield_interval)) st_sleep(0);
- }
- };
-
- if (rec_twal) {
- failed.waitset();
- g_tables.reset(new TPCCTables);
- tpcc_recovery_header &hdr = *reinterpret_cast<tpcc_recovery_header*>(orig.begin());
- commons::array<char> body(orig.begin() + sizeof(tpcc_recovery_header),
- orig.size() - sizeof(tpcc_recovery_header));
- g_tables->deser(mypos, init.node_size(), hdr, body);
- body.release();
- rec_twal_fn();
- failed.reset();
- recv_log_fn();
- }
-
-#if 0
- st_thread_t rec_twal_thread = my_spawn(rec_twal_fn, "rec_twal");
- st_thread_t recv_log_thread = my_spawn(recv_log_fn, "recv_log");
-
- st_join(rec_twal_thread);
- st_join(recv_log_thread);
-#endif
-
- if (rec_pwal) {
- // Recover from phy log.
- } else if (rec_twal) {
- // Recover from txn log.
- } else {
-
- g_tables.reset(new TPCCTables);
-
- //
- // Build-up
- //
-
- if (ship_log) {
- } else {
- // XXX indent
-
- cout << "waiting for recovery message" << (multirecover ? "s" : "")
- << endl;
- long long before_recv = current_time_millis();
-
- vector<st_thread_t> recovery_builders;
- ASSERT(seqno == -1);
- bool first = true;
- for (int i = 0; i < (multirecover ? init.node_size() : 1); ++i) {
- recovery_builders.push_back(my_spawn(lambda() {
- // Read the recovery message length and header.
- tpcc_recovery_header hdr;
- checkeqnneg(st_read_fully(__ref(replicas[i]),
- &hdr, sizeof hdr,
- ST_UTIME_NO_TIMEOUT),
- ssize_t(sizeof hdr));
- check(hdr.seqno >= 0);
-
- cout << "receiving recovery of " << hdr.len << " bytes" << endl;
-
- long long start_time = current_time_millis();
- __ref(recarr).reset(new char[hdr.len], hdr.len);
- checkeqnneg(st_read_fully(__ref(replicas[i]),
- __ref(recarr).get(), hdr.len,
- ST_UTIME_NO_TIMEOUT),
- ssize_t(hdr.len));
-
- long long before_deser = current_time_millis();
- showdatarate("received recovery message", size_t(hdr.len), before_deser - start_time);
-
- if (__ref(seqno) == -1)
- __ref(seqno) = hdr.seqno;
- else
- checkeq(__ref(seqno), hdr.seqno);
-
- g_tables->deser(__ctx(i), __ref(init).node_size(), hdr, __ref(recarr));
-
- long long end_time = current_time_millis();
- showdatarate("deserialized recovery message", size_t(hdr.len), end_time - before_deser);
- cout << "receive & deserialize took " << end_time - __ref(before_recv)
- << " ms total; now at seqno " << hdr.seqno << endl;
- cout << "after deserialize, db state is now at seqno "
- << hdr.seqno << ":" << endl;
- g_tables->show();
-
-#if 0
- // Resize the table if necessary.
-
- commons::array<entry> &table = __ref(map).get_table();
- if (!__ref(first)) {
- checkeq(table.size(), hdr.total);
- checkeq(__ref(map).size(), hdr.size);
- } else {
- __ref(first) = false;
- if (table.size() != hdr.total) {
- table.reset(new entry[hdr.total], hdr.total);
- }
- }
-
- // Receive straight into the table.
- pair<size_t, size_t> range =
- recovery_range(table.size(), __ctx(i), __ref(init).node_size());
- // Check that we agree on the number of entries.
- checkeq(range.second - range.first, hdr.count);
- // Check that the count is a power of two.
- checkeq(hdr.count & (hdr.count - 1), size_t(0));
- size_t rangelen = sizeof(entry) * hdr.count;
- // Read an extra char to ensure that we're at the EOF.
- checkeqnneg(st_read_fully(__ref(replicas[i]),
- table.begin() + range.first, rangelen + 1,
- ST_UTIME_NO_TIMEOUT),
- ssize_t(rangelen));
-#endif
- }, "recovery_builder" + lexical_cast<string>(i)));
- }
- foreach (st_thread_t t, recovery_builders) {
- st_join(t);
- }
-
- }
- }
-
- //
- // Catch-up
- //
-
- long long mid_time = current_time_millis();
- int mid_seqno = seqno;
- TpccReq req;
- while (!backlog.empty()) {
- chunk chunk = backlog.take();
- cout << "took from backlog, now has " << backlog.queue().size()
- << " chunks" << endl;
- sized_array<char> &buf = chunk.get<0>();
- char *begin = chunk.get<1>(), *end = chunk.get<2>();
- ASSERT(buf.get() <= begin && begin < buf.end());
- ASSERT(buf.get() < end && end < buf.end());
- process_buf(begin, end, req, seqno);
- }
- showtput("replayer caught up; from backlog replayed",
- current_time_millis(), mid_time, seqno, mid_seqno);
-
- } else {
-
- //
- // Simple txns
- //
-
- if (rec_pwal) {
- // Recover from physical log.
- cout << "recovering from pwal" << endl;
- long long start_time = current_time_millis();
- ifstream inf("pwal");
- binary_iarchive in(inf);
- int rseqno = -1;
- while (inf.peek() != ifstream::traits_type::eof()) {
- int op;
- in & op;
- switch (op) {
- case op_del:
- {
- int key;
- in & key;
- mii::iterator it = map.find(key);
- map.erase(it);
- break;
- }
- case op_write:
- {
- int key, val;
- in & key & val;
- map[key] = val;
- break;
- }
- case op_commit:
- ++rseqno;
- break;
- }
- if (check_interval(rseqno, yield_interval)) st_sleep(0);
- }
- seqno = init.txnseqno() - 1;
- showdatarate("recovered from pwal", inf.tellg(), current_time_millis() - start_time);
- cout << "now at seqno " << rseqno << " (really: " << seqno << ")" << endl;
- } else {
-
- //
- // Build-up
- //
-
- cout << "waiting for recovery message" << (multirecover ? "s" : "")
- << endl;
- long long before_recv = current_time_millis();
-
- vector<st_thread_t> recovery_builders;
- ASSERT(seqno == -1);
- bool first = true;
- for (int i = 0; i < (multirecover ? init.node_size() : 1); ++i) {
- recovery_builders.push_back(my_spawn(lambda() {
- // Read the recovery message length and header.
- size_t len;
- recovery_header hdr;
- char buf[sizeof len + sizeof hdr];
- //try {
- checkeqnneg(st_read_fully(__ref(replicas[i]),
- buf, sizeof len + sizeof hdr,
- ST_UTIME_NO_TIMEOUT),
- ssize_t(sizeof len + sizeof hdr));
- //} catch (...) { // TODO just catch "Connection reset by peer"
- //return;
- //}
- raw_reader rdr(buf);
- rdr.read(len);
- rdr.read(hdr);
- check(hdr.seqno >= 0);
-
- // Resize the table if necessary.
- commons::array<entry> &table = __ref(map).get_table();
- if (!__ref(first)) {
- checkeq(table.size(), hdr.total);
- checkeq(__ref(map).size(), hdr.size);
- } else {
- __ref(first) = false;
- __ref(map).set_size(hdr.size);
- if (table.size() != hdr.total) {
- table.reset(new entry[hdr.total], hdr.total);
- }
- }
-
- // Receive straight into the table.
- pair<size_t, size_t> range =
- recovery_range(table.size(), __ctx(i), __ref(init).node_size());
- // Check that we agree on the number of entries.
- checkeq(range.second - range.first, hdr.count);
- // Check that the count is a power of two.
- checkeq(hdr.count & (hdr.count - 1), size_t(0));
- size_t rangelen = sizeof(entry) * hdr.count;
- // Read an extra char to ensure that we're at the EOF.
- long long start_time = current_time_millis();
- checkeqnneg(st_read_fully(__ref(replicas[i]),
- table.begin() + range.first, rangelen + 1,
- ST_UTIME_NO_TIMEOUT),
- ssize_t(rangelen));
- long long end_time = current_time_millis();
-
- if (__ref(seqno) != -1)
- checkeq(__ref(seqno), hdr.seqno);
- __ref(seqno) = hdr.seqno;
- showdatarate("got recovery message", len, end_time - start_time);
- cout << "receive took " << end_time - __ref(before_recv)
- << " ms total; now at seqno " << hdr.seqno << endl;
-#if 0
- Recovery recovery;
- long long receive_start = 0, receive_end = 0;
- size_t len = 0;
- {
- st_intr intr(stop_hub);
- len = readmsg(__ref(replicas)[__ctx(i)], recovery, &receive_start,
- &receive_end);
- }
- long long build_start = current_time_millis();
- cout << "got recovery message of " << len << " bytes in "
- << build_start - __ref(before_recv) << " ms: xfer took "
- << receive_end - receive_start << " ms, deserialization took "
- << build_start - receive_end << " ms" << endl;
- for (int i = 0; i < recovery.pair_size(); ++i) {
- const Recovery_Pair &p = recovery.pair(i);
- __ref(map)[p.key()] = p.value();
- if (i % yield_interval == 0) {
- if (yield_during_build_up) st_sleep(0);
- }
- }
- check(recovery.seqno() >= 0);
- int seqno = __ref(seqno) = recovery.seqno();
- long long build_end = current_time_millis();
- cout << "receive and build-up took "
- << build_end - __ref(before_recv)
- << " ms; built up map of " << recovery.pair_size()
- << " records in " << build_end - build_start
- << " ms; now at seqno " << seqno << endl;
-#endif
- }, "recovery_builder" + lexical_cast<string>(i)));
- }
- foreach (st_thread_t t, recovery_builders) {
- st_join(t);
- }
- }
-
- //
- // Catch-up
- //
-
- long long mid_time = current_time_millis();
- int mid_seqno = seqno;
- // XXX
- using msg::TxnBatch;
- using msg::Txn;
- commons::array<char> rbuf(0), wbuf(buf_size);
- reader reader(nullptr, rbuf.get(), rbuf.size());
- writer writer(lambda(const void*, size_t) {
- throw not_supported_exception("should not be writing responses during catch-up phase");
- }, wbuf.get(), wbuf.size());
- stream s(reader, writer);
- TxnBatch batch(s);
- while (!backlog.empty()) {
- chunk chunk = backlog.take();
- sized_array<char> &buf = chunk.get<0>();
- ASSERT(buf.get() <= chunk.get<1>() && chunk.get<1>() < buf.end());
- ASSERT(buf.get() < chunk.get<2>() && chunk.get<2>() < buf.end());
- ASSERT(chunk.get<1>() < chunk.get<2>());
- swap(buf, reader.buf());
- reader.reset_range(chunk.get<1>(), chunk.get<2>());
- while (reader.start() < reader.end()) {
- char *start = reader.start();
- uint32_t prefix = ntohl(reader.read<uint32_t>());
- ASSERT(prefix < 10000);
- ASSERT(start + sizeof(uint32_t) + prefix <= reader.end());
- batch.Clear();
- for (int t = 0; t < batch.txn_size(); ++t) {
- const Txn &txn = batch.txn(t);
- if (rec_pwal) seqno = txn.seqno() - 1;
- process_txn<rb_traits, rb_traits>(map, txn, seqno, nullptr);
- if (fake_exec && !Types::is_pb()) {
- reader.skip(txn.op_size() * Op_Size);
- }
-
- if (check_interval(txn.seqno(), yield_interval)) st_sleep(0);
- if (check_interval(txn.seqno(), process_display)) {
- cout << "caught up txn " << txn.seqno()
- << "; db size = " << map.size()
- << "; seqno = " << seqno
- << "; backlog.size = " << backlog.queue().size() << endl;
- }
- }
- ASSERT(start + sizeof(uint32_t) + prefix == reader.start());
- }
- }
- g_caught_up = true;
-#if 0
- while (!backlog.empty()) {
- using pb::Txn;
- shared_ptr<Txn> p = backlog.take();
- process_txn<pb_traits, pb_traits>(map, *p, seqno, nullptr);
- if (check_interval(p->seqno(), catch_up_display)) {
- cout << "processed txn " << p->seqno() << " off the backlog; "
- << "backlog.size = " << backlog.queue().size() << endl;
- }
- if (check_interval(p->seqno(), yield_interval)) {
- // Explicitly yield. (Note that yielding does still effectively
- // happen anyway because process_txn is a yield point.)
- st_sleep(0);
- }
- }
-#endif
- showtput("replayer caught up; from backlog replayed",
- current_time_millis(), mid_time, seqno, mid_seqno);
- }
- }
- } catch (std::exception &ex) {
- cerr_thread_ex(ex) << endl;
- throw;
- }
-
- stop_hub.insert(st_thread_self());
-}
Modified: ydb/trunk/src/msg.h
===================================================================
--- ydb/trunk/src/msg.h 2009-03-20 07:58:28 UTC (rev 1313)
+++ ydb/trunk/src/msg.h 2009-03-20 17:45:38 UTC (rev 1314)
@@ -55,7 +55,7 @@
using namespace commons;
using namespace std;
-short unset = -7654;
+static const short unset = -7654;
using ydb::pb::Op_OpType;
Modified: ydb/trunk/src/stxn.lzz.clamp
===================================================================
--- ydb/trunk/src/stxn.lzz.clamp 2009-03-20 07:58:28 UTC (rev 1313)
+++ ydb/trunk/src/stxn.lzz.clamp 2009-03-20 17:45:38 UTC (rev 1314)
@@ -556,3 +556,178 @@
{
return recovery_t();
}
+
+class response_handler
+{
+public:
+ response_handler(st_netfd_t replica, const int &seqno, int rid,
+ st_multichannel<long long> &recover_signals, bool caught_up)
+ :
+ replica(replica),
+ seqno(seqno),
+ rid(rid),
+ recover_signals(recover_signals),
+ caught_up(caught_up),
+ sub(recover_signals.subscribe()),
+ start_time(current_time_millis()),
+ recovery_start_time(caught_up ? -1 : start_time),
+ recovery_end_time(-1),
+ start_seqno(seqno),
+ recovery_start_seqno(caught_up ? -1 : seqno),
+ recovery_end_seqno(-1),
+ last_seqno(-1)
+ {}
+
+ template<typename Types>
+ void run() {
+ typedef typename Types::Response Response;
+ typedef typename Types::ResponseBatch ResponseBatch;
+
+ finally f(bind(&response_handler::cleanup, this));
+
+ commons::array<char> rbuf(read_buf_size), wbuf(buf_size);
+ st_reader reader(replica, rbuf.get(), rbuf.size());
+ writer w(lambda(const void*, size_t) {
+ throw not_supported_exception("response handler should not be writing");
+ }, wbuf.get(), wbuf.size());
+ stream s(reader,w);
+
+ scoped_ptr<ResponseBatch> pbatch(new_ResponseBatch<ResponseBatch>(s));
+ ResponseBatch &batch = *pbatch;
+
+ long long last_display_time = current_time_millis();
+
+ function<void()> loop_cleanup =
+ bind(&response_handler::loop_cleanup, this);
+
+ while (true) {
+ finally f(loop_cleanup);
+ uint32_t prefix = 0;
+
+ // Read the message, but correctly respond to interrupts so that we can
+ // cleanly exit (slightly tricky).
+ if (last_seqno + 1 == seqno) {
+ // Stop-interruptible in case we're already caught up.
+ try {
+ st_intr intr(stop_hub);
+ if (Types::is_pb()) readmsg(reader, batch);
+ else { prefix = ntohl(reader.read<uint32_t>()); batch.Clear(); }
+ } catch (...) { // TODO: only catch interruptions
+ // This check on seqnos is OK for termination since the seqno will
+ // never grow again if stop_hub is set.
+ if (last_seqno + 1 == seqno) {
+ cout << rid << ": ";
+ cout << "clean stop; next expected seqno is " << seqno
+ << " (last seqno was " << last_seqno << ")" << endl;
+ break;
+ } else {
+ continue;
+ }
+ }
+ } else {
+ // Only kill-interruptible because we want a clean termination (want
+ // to get all the acks back).
+ st_intr intr(kill_hub);
+ if (Types::is_pb()) readmsg(reader, batch);
+ else { prefix = ntohl(reader.read<uint32_t>()); batch.Clear(); }
+ }
+
+ for (int i = 0; i < batch.res_size(); ++i) {
+ const Response &res = batch.res(i);
+ // Determine if this response handler's host (the only joiner) has finished
+ // catching up. If it has, then broadcast a signal so that all response
+ // handlers will know about this event.
+ int rseqno = res.seqno();
+ if (rseqno <= last_seqno)
+ throw msg_exception(string("response seqno decreased from ") +
+ lexical_cast<string>(last_seqno) + " to " +
+ lexical_cast<string>(rseqno));
+ bool rcaught_up = res.caught_up();
+ for (int r = 0; r < res.result_size(); ++r) {
+ cout << rseqno << last_seqno << res.result_size() << " " << r << " " << res.result(r) << endl;
+ }
+ if (!caught_up && rcaught_up) {
+ long long now = current_time_millis(), time_diff = now - start_time;
+ caught_up = true;
+ recover_signals.push(now);
+ cout << rid << ": ";
+ cout << "recovering node caught up; took "
+ << time_diff << " ms" << endl;
+ // This will cause the program to exit eventually, but cleanly, such that
+ // the recovery time will be set first, before the eventual exit (which
+ // may not even happen in the current iteration).
+ if (stop_on_recovery) {
+ cout << "stopping on recovery" << endl;
+ stop_hub.set();
+ }
+ }
+ if (check_interval(rseqno, handle_responses_display)) {
+ cout << rid << ": " << "got response " << rseqno << " from "
+ << replica << "; ";
+ long long display_time = current_time_millis();
+ showtput("handling", display_time, last_display_time, rseqno,
+ rseqno - handle_responses_display);
+ last_display_time = display_time;
+ }
+ if (check_interval(rseqno, yield_interval)) {
+ st_sleep(0);
+ }
+ last_seqno = rseqno;
+ }
+ }
+ }
+
+private:
+ void loop_cleanup() {
+ // The first timestamp that comes down the subscription pipeline is the
+ // recovery start time, issued by the main thread. The second one is the
+ // recovery end time, issued by the response handler associated with the
+ // joiner.
+ if (recovery_start_time == -1 && !sub.empty()) {
+ recovery_start_time = sub.take();
+ recovery_start_seqno = last_seqno;
+ cout << rid << ": ";
+ showtput("before recovery, finished", recovery_start_time, start_time,
+ recovery_start_seqno, 0);
+ } else if (recovery_end_time == -1 && !sub.empty()) {
+ recovery_end_time = sub.take();
+ recovery_end_seqno = last_seqno;
+ cout << rid << ": ";
+ showtput("during recovery, finished roughly", recovery_end_time,
+ recovery_start_time, recovery_end_seqno, recovery_start_seqno);
+ }
+ }
+
+ void cleanup() {
+ long long end_time = current_time_millis();
+ cout << rid << ": ";
+ showtput("handled roughly", end_time, start_time, seqno, start_seqno);
+ if (recovery_end_time > -1) {
+ cout << rid << ": ";
+ showtput("after recovery, finished", end_time, recovery_end_time,
+ seqno, recovery_end_seqno);
+ }
+ }
+
+ st_netfd_t replica;
+ const int &seqno;
+ int rid;
+ st_multichannel<long long> &recover_signals;
+ bool caught_up;
+ st_channel<long long> ⊂
+ long long start_time, recovery_start_time, recovery_end_time;
+ int start_seqno, recovery_start_seqno, recovery_end_seqno, last_seqno;
+};
+
+/**
+ * Swallow replica responses.
+ */
+template<typename Types>
+void
+handle_responses(st_netfd_t replica, const int &seqno, int rid,
+ st_multichannel<long long> &recover_signals, bool caught_up)
+{
+ response_handler h(replica, seqno, rid, recover_signals, caught_up);
+ h.run<Types>();
+}
+
Copied: ydb/trunk/src/ydb.lzz.clamp (from rev 1313, ydb/trunk/src/main2.lzz.clamp)
===================================================================
--- ydb/trunk/src/ydb.lzz.clamp (rev 0)
+++ ydb/trunk/src/ydb.lzz.clamp 2009-03-20 17:45:38 UTC (rev 1314)
@@ -0,0 +1,1073 @@
+#hdr
+#include "unsetprefs.h"
+#include <boost/function.hpp>
+#include <boost/scoped_ptr.hpp>
+#include <string>
+#include <iostream>
+#include <st.h>
+#include <commons/st/st.h>
+#include "tpcc/clock.h"
+#include "tpcc/randomgenerator.h"
+#include "tpcc/tpccclient.h"
+#include "tpcc/tpccgenerator.h"
+#include "tpcc/tpcctables.h"
+#include "util.hh"
+#include "tpcc.hh"
+#include "stxn.hh"
+#include "main.hh"
+
+using namespace boost;
+using namespace std;
+using namespace commons;
+#end
+
+#src
+#include "unsetprefs.h"
+#include <csignal> // sigaction, etc.
+#include <cstring> // strsignal, size_t
+#include <boost/program_options.hpp>
+#include <gtest/gtest.h>
+#include <malloc.h>
+#include <string>
+#include "setprefs.h"
+#end
+
+using namespace google;
+using namespace testing;
+
+//
+// Utilities/system
+//
+
+/**
+ * Delegate for running thread targets.
+ * \param[in] f The function to execute.
+ * \param[in] intr Whether to signal stop_hub on an exception.
+ */
+void
+my_spawn_helper(const function0<void> f, bool intr)
+{
+ thread_eraser eraser;
+ try {
+ f();
+ } catch (std::exception &ex) {
+ cerr_thread_ex(ex) << (intr ? "; interrupting!" : "") << endl;
+ if (intr) stop_hub.set();
+ }
+}
+
+/**
+ * Spawn a thread using ST but wrap it in an exception handler that interrupts
+ * all other threads (hopefully causing them to unwind).
+ * \param[in] f The function to execute.
+ * \param[in] intr Whether to signal stop_hub on an exception. Not actually
+ * used anywhere.
+ */
+st_thread_t
+my_spawn(const function0<void> &f, string name, bool intr = false)
+{
+ st_thread_t t = st_spawn(bind(my_spawn_helper, f, intr));
+ threads.insert(t);
+ threadnames[t] = name;
+ return t;
+}
+
+/**
+ * Memory monitor.
+ */
+void
+memmon()
+{
+ while (!stop_hub) {
+ {
+ st_intr intr(stop_hub);
+ st_sleep(1);
+ }
+ malloc_stats();
+ }
+}
+
+int sig_pipe[2];
+
+//
+// Signals
+//
+
+/**
+ * Raw signal handler that triggers the (synchronous) handler.
+ */
+void handle_sig(int sig) {
+ int err = errno;
+ cerr << "got signal: " << strsignal(sig) << " (" << sig << ")" << endl;
+ checkeqnneg(write(sig_pipe[1], &sig, sizeof sig),
+ static_cast<ssize_t>(sizeof sig));
+ errno = err;
+}
+
+/**
+ * Synchronous part of the signal handler; cleanly interrrupts any threads that
+ * have marked themselves as interruptible.
+ */
+void handle_sig_sync() {
+ st_closing fd(checkerr(st_netfd_open(sig_pipe[0])));
+ while (true) {
+ int sig;
+ checkeqnneg(st_read(fd, &sig, sizeof sig, ST_UTIME_NO_TIMEOUT),
+ static_cast<ssize_t>(sizeof sig));
+ if (sig == SIGINT) {
+ ...
[truncated message content] |
|
From: <yan...@us...> - 2009-03-20 07:58:33
|
Revision: 1313
http://assorted.svn.sourceforge.net/assorted/?rev=1313&view=rev
Author: yangzhang
Date: 2009-03-20 07:58:28 +0000 (Fri, 20 Mar 2009)
Log Message:
-----------
- broke up the program into multiple modules
- using new custom clamp
- ser.h -> msg.h
- still some renaming to do
Modified Paths:
--------------
ydb/trunk/src/Makefile
ydb/trunk/src/main.lzz.clamp
ydb/trunk/src/ser.cc
Added Paths:
-----------
ydb/trunk/src/main2.lzz.clamp
ydb/trunk/src/msg.h
ydb/trunk/src/setprefs.h
ydb/trunk/src/stxn.lzz.clamp
ydb/trunk/src/tpcc.lzz.clamp
ydb/trunk/src/unsetprefs.h
ydb/trunk/src/util.lzz
Removed Paths:
-------------
ydb/trunk/src/ser.h
Modified: ydb/trunk/src/Makefile
===================================================================
--- ydb/trunk/src/Makefile 2009-03-20 05:35:47 UTC (rev 1312)
+++ ydb/trunk/src/Makefile 2009-03-20 07:58:28 UTC (rev 1313)
@@ -38,6 +38,7 @@
endif
# CXX := $(WTF) ag++ -k --Xcompiler # $(CXX)
CXX := $(WTF) ccache $(CXX) -pipe
+LD := $(CXX)
LDFLAGS := -pthread $(GPROF)
LDLIBS := -lstx -lst -lresolv -lprotobuf -lgtest \
-lboost_program_options-gcc43-mt -lboost_thread-gcc43-mt \
@@ -93,13 +94,15 @@
all: $(TARGET)
-$(TARGET): $(OBJS)
- $(LINK.cc) $^ $(LOADLIBES) $(LDLIBS) -o $@
-
%.pb.o: %.pb.cc %.pb.h
$(CXX) -c $(PBCXXFLAGS) $(OUTPUT_OPTION) $<
-main.o: main.cc $(PBHDRS)
+stxn.o: main.hh $(PBHDRS)
+main.o: util.hh msg.h $(PBHDRS)
+util.o: msg.h $(PBHDRS)
+main2.o: main.hh stxn.hh tpcc.hh $(PBHDRS)
+tpcc.o: main.hh util.hh $(PBHDRS)
+ydb: main.o main2.o util.o tpcc.o stxn.o
tpcc/%.o: tpcc/%.cc
make -C tpcc/
@@ -109,12 +112,8 @@
.SECONDARY: tpcc/tpcctables.cc tpcc/tpcctables.o
-%.o: %.cc
- $(COMPILE.cc) $(OUTPUT_OPTION) $<
-
%.cc %.hh: %.lzz
lzz -hx hh -sx cc -hl -sl -hd -sd $<
- python -c 'pars = file("lambda_impl.clamp_h").read().split("\n\n"); hh = file("main.hh").read(); print >> file("main.cc", "a"), pars[-1]; print >> file("main.hh", "w"), "\n\n".join(pars[:-1] + [hh])'
%.pb.cc: %.proto
protoc --cpp_out=. $<
@@ -124,11 +123,12 @@
%.lzz: %.lzz.clamp
rm -f $@
- clamp < $< | sed '1d' > $@
+ mkdir -p clamp/
+ clamp --outdir clamp/ --prefix $(basename $@) < $< | \
+ sed "$$( echo -e '1i\\\n\#hdr\n1a\\\n\#end' )" | \
+ sed "$$( echo -e '$$i\\\n\#src\n$$a\\\n\#end' )" > $@
chmod -w $@
-main.o: ser.h
-
all.h:
fgrep '#include' main.lzz.clamp > all.h
@@ -136,7 +136,9 @@
$(COMPILE.cc) $(PBHDRS) $(OUTPUT_OPTION) $<
clean:
- rm -f $(GENSRCS) $(GENHDRS) $(OBJS) $(TARGET) main.lzz *.clamp_h
+ rm -rf clamp/ $(GENSRCS) $(GENHDRS) $(OBJS) $(TARGET) \
+ main.lzz main2.lzz main.cc main.hh main2.hh main2.cc \
+ util.cc util.hh tpcc.lzz tpcc.hh tpcc.cc
make -C tpcc/ clean
distclean: clean
@@ -159,5 +161,5 @@
p2: p2.cc
$(LINK.cc) $^ $(LOADLIBES) $(LDLIBS) $(OUTPUT_OPTION)
-ser: ser.cc ser.h ydb.pb.o
+ser: ser.cc msg.h ydb.pb.o
$(LINK.cc) $^ $(LOADLIBES) $(LDLIBS) $(OUTPUT_OPTION)
Modified: ydb/trunk/src/main.lzz.clamp
===================================================================
--- ydb/trunk/src/main.lzz.clamp 2009-03-20 05:35:47 UTC (rev 1312)
+++ ydb/trunk/src/main.lzz.clamp 2009-03-20 07:58:28 UTC (rev 1313)
@@ -1,103 +1,52 @@
#hdr
-#define __STDC_FORMAT_MACROS
+#include "unsetprefs.h"
#include <boost/archive/binary_iarchive.hpp>
#include <boost/archive/binary_oarchive.hpp>
#include <boost/bind.hpp>
#include <boost/foreach.hpp>
-#include <boost/program_options.hpp>
-#include <boost/range/iterator_range.hpp>
-#include <boost/scoped_array.hpp>
-#include <boost/shared_ptr.hpp>
+//#include <boost/range/iterator_range.hpp>
+//#include <boost/shared_ptr.hpp>
#include <boost/tuple/tuple.hpp>
#include <commons/assert.h>
-#include <commons/memory.h>
#include <commons/nullptr.h>
-#include <commons/rand.h>
-#include <commons/snap_map.h>
#include <commons/st/st.h>
#include <commons/time.h>
-#include <commons/unique_ptr.h>
-#include <csignal> // sigaction etc.
-#include <cstdio>
-#include <cstring> // strsignal
#include <fstream> // ofstream
-#include <google/dense_hash_map>
-#include <google/protobuf/io/zero_copy_stream_impl.h>
-#include <gtest/gtest.h>
-#include <inttypes.h> // PRId64
#include <iostream>
-#include <malloc.h>
-#include <map>
-#include <netinet/in.h> // in_addr etc.
-#include <set>
-#include <sys/socket.h> // getpeername
-#include <sys/types.h> // ssize_t
-#include <tr1/unordered_map>
-#include <unistd.h> // pipe, write, sync
#include <vector>
-#include "ser.h"
-#include "tpcc/clock.h"
-#include "tpcc/randomgenerator.h"
-#include "tpcc/tpccclient.h"
-#include "tpcc/tpccgenerator.h"
-#include "tpcc/tpcctables.h"
-#include "ydb.pb.h"
+#include "msg.h"
+#include "util.hh"
+#include "setprefs.h"
-#define function boost::function
-#define foreach BOOST_FOREACH
-#define shared_ptr boost::shared_ptr
-#define ref boost::ref
-#define tuple boost::tuple
-#define make_tuple boost::make_tuple
-
using namespace boost;
using namespace boost::archive;
using namespace commons;
-using namespace google;
-using namespace google::protobuf::io;
using namespace std;
-using namespace std::tr1;
-using namespace testing;
using namespace ydb;
-using namespace ydb::pb;
using namespace ydb::msg;
#end
-//#define map_t unordered_map
-//#define map_t map
-//#define map_t dense_hash_map
-#define map_t snap_map
-typedef map_t<int, int> mii;
-typedef mii::value_type entry;
+#src
+#include <unistd.h> // pipe, write, sync
+#end
typedef tuple<sized_array<char>, char*, char*> chunk;
-//typedef unique_ptr<Recovery> recovery_t;
typedef commons::array<char> recovery_t;
-template<typename T> void init_map(T &map) {}
-template<> void init_map(dense_hash_map<int, int> &map) {
- map.set_empty_key(-1);
- map.set_deleted_key(-2);
-}
-template<> void init_map(snap_map<int, int> &map) {
- map.set_empty_key(-1);
- map.set_deleted_key(-2);
-}
-
// Configuration.
st_utime_t timeout;
int yield_interval, accept_joiner_seqno, issuing_interval, min_ops, max_ops,
stop_on_seqno, batch_size, handle_responses_display, fail_seqno,
catch_up_display, issue_display, nwarehouses,
process_display;
-size_t accept_joiner_size, buf_size, read_buf_size;
+size_t accept_joiner_size, read_buf_size;
bool yield_during_build_up, yield_during_catch_up, dump, show_updates,
- count_updates, stop_on_recovery, general_txns, profile_threads,
- debug_threads, multirecover, disk, debug_memory, use_pwal, use_twal,
+ count_updates, stop_on_recovery, general_txns,
+ disk, debug_memory, use_pwal, use_twal,
use_pb, use_pb_res, g_caught_up, rec_pwal, rec_twal, do_tpcc,
- suppress_txn_msgs, fake_bcast, force_ser, fake_exec, ship_log;
-long long timelim, read_thresh, write_thresh;
+ suppress_txn_msgs, force_ser, fake_exec, ship_log;
+long long timelim, read_thresh;
// Control.
st_intr_bool stop_hub, kill_hub;
@@ -112,147 +61,6 @@
int updates;
/**
- * Convenience function for calculating percentages.
- */
-template<typename T>
-inline double pct(T sub, T tot)
-{
- return 100 * double(sub) / double(tot);
-}
-
-/**
- * Convenience class for performing long-jumping break.
- */
-class break_exception : public std::exception {};
-
-/**
- * The list of all threads. Keep track of these so that we may cleanly shut
- * down all threads.
- */
-set<st_thread_t> threads;
-
-/**
- * RAII for adding/removing the current thread from the global threads set.
- */
-class thread_eraser
-{
- public:
- thread_eraser() { threads.insert(st_thread_self()); }
- ~thread_eraser() { threads.erase(st_thread_self()); }
-};
-
-/**
- * For debug/error-printing purposes.
- */
-map<st_thread_t, string> threadnames;
-st_thread_t last_thread;
-
-/**
- * For profiling.
- */
-map<st_thread_t, long long> threadtimes;
-long long thread_start_time;
-
-/**
- * Look up thread name, or just show thread ID.
- */
-inline string
-threadname(st_thread_t t = st_thread_self()) {
- if (threadnames.find(t) != threadnames.end()) {
- return threadnames[t];
- } else {
- return lexical_cast<string>(t);
- }
-}
-
-/**
- * Debug function for thread names. Remember what we're switching from.
- */
-inline void
-switch_out_cb()
-{
- if (debug_threads) last_thread = st_thread_self();
- if (profile_threads)
- threadtimes[st_thread_self()] += current_time_millis() - thread_start_time;
-}
-
-/**
- * Debug function for thread names. Show what we're switching from/to.
- */
-inline void switch_in_cb()
-{
- if (debug_threads && last_thread != st_thread_self()) {
- cout << "switching";
- if (last_thread != 0) cout << " from " << threadname(last_thread);
- cout << " to " << threadname() << endl;
- }
- if (profile_threads)
- thread_start_time = current_time_millis();
-}
-
-/**
- * Print to cerr a thread exception.
- */
-ostream&
-cerr_thread_ex(const std::exception &ex)
-{
- return cerr << "exception in thread " << threadname()
- << ": " << ex.what();
-}
-
-/**
- * Delegate for running thread targets.
- * \param[in] f The function to execute.
- * \param[in] intr Whether to signal stop_hub on an exception.
- */
-void
-my_spawn_helper(const function0<void> f, bool intr)
-{
- thread_eraser eraser;
- try {
- f();
- } catch (std::exception &ex) {
- cerr_thread_ex(ex) << (intr ? "; interrupting!" : "") << endl;
- if (intr) stop_hub.set();
- }
-}
-
-/**
- * Spawn a thread using ST but wrap it in an exception handler that interrupts
- * all other threads (hopefully causing them to unwind).
- * \param[in] f The function to execute.
- * \param[in] intr Whether to signal stop_hub on an exception. Not actually
- * used anywhere.
- */
-st_thread_t
-my_spawn(const function0<void> &f, string name, bool intr = false)
-{
- st_thread_t t = st_spawn(bind(my_spawn_helper, f, intr));
- threads.insert(t);
- threadnames[t] = name;
- return t;
-}
-
-char *
-show_sockaddr(st_netfd_t fd)
-{
- sockaddr_in sa;
- socklen_t salen = sizeof sa;
- check0x(getpeername(st_netfd_fileno(fd),
- reinterpret_cast<sockaddr*>(&sa),
- &salen));
- return inet_ntoa(sa.sin_addr);
-}
-
-map<st_netfd_t, string> nfdnames;
-
-inline const string&
-nfd2name(st_netfd_t fd)
-{
- return nfdnames[fd];
-}
-
-/**
* Used by the leader to bookkeep information about replicas.
*/
class replica_info
@@ -309,95 +117,10 @@
const vector<st_netfd_t> &rs_;
};
+#if 0
st_channel<pair<st_netfd_t, shared_ptr<string> > > msgs;
/**
- * Adapter for arrays to look like strings (for PB serialization).
- */
-class ser_array
-{
- commons::array<char> a_;
- size_t size_;
-public:
- ser_array(size_t size = buf_size) : a_(size), size_(0) {}
- char *data() const { return a_.get(); }
- size_t size() const { return size_; }
- void clear() { size_ = 0; }
- void stretch(size_t size) {
- if (size > a_.size())
- a_.reset(new char[size], size);
- size_ = size;
- }
-};
-
-//typedef string ser_t;
-typedef ser_array ser_t;
-
-template<typename T>
-void
-ser(writer &w, const T &msg)
-{
- uint32_t len = msg.ByteSize();
- w.mark();
- w.reserve(len);
- check(msg.SerializeToArray(w.cur(), len));
- w.skip(len);
-}
-
-/**
- * Serialization.
- *
- * TODO: experiment with which method is the fastest: using a string as shown
- * here or computing the bytesize then allocating (or grabbing/reserving) the
- * array.
- */
-template<typename T>
-void
-ser(string &s, const T &msg)
-{
- // Serialize message to a buffer.
- uint32_t len;
- s.append(sizeof len, '\0');
- check(msg.AppendToString(&s));
-
- // Warn if the message is large.
- if (s.size() > 1000000)
- cout << "serializing large message of " << s.size() << " bytes" << endl;
-
- // Prefix the message with a four-byte length.
- len = htonl(static_cast<uint32_t>(s.size() - sizeof len));
- char *plen = reinterpret_cast<char*>(&len);
- copy(plen, plen + sizeof len, s.begin());
-}
-
-template<typename T>
-inline void
-ser(ser_array &s, const T &msg)
-{
- int len = msg.ByteSize();
-
- // Grow the array as needed.
- s.stretch(len + sizeof(uint32_t));
-
- // Serialize message to a buffer with four-byte length prefix.
- check(msg.SerializeToArray(s.data() + sizeof(uint32_t), len));
- *reinterpret_cast<uint32_t*>(s.data()) = htonl(uint32_t(len));
-}
-
-/**
- * Serialization.
- */
-template<typename T>
-inline void
-ser(ostream &s, const T &msg)
-{
- uint32_t len = htonl(uint32_t(msg.ByteSize()));
- s.write(reinterpret_cast<const char*>(&len), sizeof len);
- check(msg.SerializeToOstream(&s));
-}
-
-#if 0
-/**
* The worker that performs the actual broadcasting.
*/
void
@@ -431,193 +154,6 @@
}
#endif
-/**
- * Perform an st_write but warn if it took over write_thresh ms.
- */
-void
-st_timed_write(st_netfd_t dst, const void *buf, size_t len)
-{
- long long before_write = -1;
- if (write_thresh > 0) {
- before_write = current_time_millis();
- }
-
- checkeqnneg(st_write(dst, buf, len, ST_UTIME_NO_TIMEOUT),
- static_cast<ssize_t>(len));
-
- if (write_thresh > 0) {
- long long write_time = current_time_millis() - before_write;
- if (write_time > write_thresh) {
- cout << "thread " << threadname() << " write of " << len
- << " bytes to dst " << show_sockaddr(dst) << " blocked for "
- << write_time << " ms" << endl;
- }
- }
-}
-
-/**
- * Send a message to some destinations.
- */
-inline void
-bcastbuf(const vector<st_netfd_t> &dsts, const ser_t &msg)
-{
- if (!fake_bcast) {
- foreach (st_netfd_t dst, dsts) {
- st_timed_write(dst, msg.data(), msg.size());
- }
- }
-}
-
-/**
- * Send a message to some destinations, using whichever method of network IO
- * was chosen (sync or async).
- */
-template<typename T>
-inline void
-bcastmsg(const vector<st_netfd_t> &dsts, const T &msg)
-{
- ser_t s;
- ser(s, msg);
- bcastbuf(dsts, s);
-}
-
-/**
- * Send a message to a single recipient.
- */
-inline void
-sendbuf(st_netfd_t dst, const ser_t &msg)
-{
- if (!fake_bcast)
- st_timed_write(dst, msg.data(), msg.size());
-}
-
-/**
- * Send a message to a single recipient.
- */
-template<typename T>
-inline void
-sendmsg(st_netfd_t dst, const T &msg)
-{
- ser_t s;
- ser(s, msg);
- sendbuf(dst, s);
-}
-
-/**
- * Read a message. This is done in two steps: first by reading the length
- * prefix, then by reading the actual body. This function also provides a way
- * to measure how much time is spent actually reading the message from the
- * network. Such measurement only makes sense for large messages which take a
- * long time to receive.
- *
- * \param[in] src The socket from which to read.
- *
- * \param[in] msg The protobuf to read into.
- *
- * \param[out] start_time If not null, record the time at which we start to
- * receive the message (after the length is received).
- *
- * \param[out] stop_time If not null, record the time at which we finish
- * receiving the message (before we deserialize the protobuf).
- *
- * \param[out] len If not null, record the size of the serialized message
- * in bytes.
- *
- * \param[in] timeout on each of the two read operations (first one is on
- * length, second one is on the rest).
- *
- * \return The length of the serialized message.
- */
-template <typename T>
-size_t
-readmsg(st_netfd_t src, T & msg, long long *start_time = nullptr, long long
- *stop_time = nullptr, st_utime_t timeout = ST_UTIME_NO_TIMEOUT)
-{
- // Read the message length.
- uint32_t len;
- checkeqnneg(st_read_fully(src, static_cast<void*>(&len), sizeof len,
- timeout),
- static_cast<ssize_t>(sizeof len));
- if (start_time != nullptr)
- *start_time = current_time_millis();
- len = ntohl(len);
-
- // Parse the message body. Try stack-allocation if possible.
- scoped_array<char> sbuf;
- char *buf;
- if (len <= 4096) buf = reinterpret_cast<char*>(alloca(len));
- else sbuf.reset(buf = new char[len]);
- checkeqnneg(st_read_fully(src, buf, len, timeout), int(len));
- if (stop_time != nullptr)
- *stop_time = current_time_millis();
- check(msg.ParseFromArray(buf, len));
-
- return len;
-}
-
-/**
- * Same as the above readmsg(), but returns an internally constructed message.
- * This is a "higher-level" readmsg() that relies on return-value optimization
- * for avoiding unnecessary copies.
- */
-template <typename T>
-inline T
-readmsg(st_netfd_t src, st_utime_t timeout = ST_UTIME_NO_TIMEOUT)
-{
- T msg;
- readmsg(src, msg, nullptr, nullptr, timeout);
- return msg;
-}
-
-/**
- * Same as the above readmsg() but uses an st_reader instead of a raw
- * st_netfd_t.
- */
-template <typename T>
-inline void
-readmsg(st_reader &src, T & msg)
-{
- managed_array<char> a = src.read(sizeof(uint32_t));
- uint32_t len = ntohl(*reinterpret_cast<const uint32_t*>(a.get()));
- check(msg.ParseFromArray(src.read(len), len));
-}
-
-template<typename T>
-inline void
-readmsg(anchored_stream_reader &src, T &msg)
-{
- uint32_t len = ntohl(src.read<uint32_t>());
- check(msg.ParseFromArray(checkpass(src.read(len)), len));
-}
-
-template<typename T>
-inline void
-readmsg(istream &src, T &msg)
-{
- uint32_t len;
- src.read(reinterpret_cast<char*>(&len), sizeof len);
- len = ntohl(len);
-#if 0
- IstreamInputStream iis(&src);
- LimitingInputStream lis(&iis, len);
- check(msg.ParseFromZeroCopyStream(&lis));
-#else
- char buf[len];
- src.read(buf, len);
- check(msg.ParseFromArray(buf, len));
-#endif
-}
-
-inline uint32_t
-readlen(istream &src)
-{
- uint32_t len;
- src.read(reinterpret_cast<char*>(&len), sizeof len);
- len = ntohl(len);
- ASSERT(len < 10000);
- return len;
-}
-
enum { op_del, op_write, op_commit };
/**
@@ -665,971 +201,10 @@
};
// Globals
-mii g_map;
wal *g_wal;
txn_wal *g_twal;
//tpcc_wal *g_tpcc_wal;
-/**
- * Keep issuing transactions to the replicas.
- */
-template<typename Types>
-void
-issue_txns(st_channel<replica_info> &newreps, int &seqno,
- st_bool &accept_joiner)
-{
- typedef typename Types::TxnBatch TxnBatch;
- typedef typename Types::Txn Txn;
- typedef typename Types::Op Op;
-
- Op_OpType types[] = {Op::read, Op::write, Op::del};
- vector<st_netfd_t> fds;
- long long start_time = current_time_millis();
-
- finally f(lambda () {
- showtput("issued", current_time_millis(), __ref(start_time), __ref(seqno),
- 0);
- });
-
- commons::array<char> rbuf(read_buf_size), wbuf(buf_size);
- reader r(nullptr, rbuf.get(), rbuf.size());
- function<void(const void*, size_t)> fn;
- if (use_twal)
- fn = bind(&txn_wal::logbuf, g_twal, _1, _2);
- else
- fn = lambda(const void *buf, size_t len) {
- foreach (st_netfd_t dst, __ref(fds))
- st_timed_write(dst, buf, len);
- };
-
- char *real_wbuf = newreps.empty() ? rbuf.get() : wbuf.get();
- size_t real_wbuf_size = newreps.empty() ? rbuf.size() : wbuf.size();
- writer w(fn, real_wbuf, real_wbuf_size);
- stream s(r,w);
- scoped_ptr<TxnBatch> pbatch(new_TxnBatch<TxnBatch>(s));
- TxnBatch batch = *pbatch;
- if (Types::is_pb())
- for (int t = 0; t < batch_size; ++t)
- batch.add_txn();
-
- ser_t serbuf;
- while (!stop_hub) {
- w.mark();
- batch.Clear();
-
- // Did we get a new member? If so, notify an arbitrary member (the first
- // one) to prepare to send recovery information (by sending an
- // empty/default Txn).
- // XXX rec_pwal
- if (!newreps.empty() && seqno > 0 && !rec_pwal) {
- start_txn(batch);
- fin_txn(batch);
- // TODO: verify that this made the catch-up stream more efficient,
- // starting it only at the point necessary
- w.mark_and_flush();
- if (Types::is_pb()) {
- if (multirecover) bcastmsg(fds, batch);
- else sendmsg(fds[0], batch);
- }
- batch.Clear();
- }
- // Bring in any new members.
- // TODO more efficient: copy/extend/append
- while (!newreps.empty()) {
- fds.push_back(newreps.take().fd());
- }
-
- // Generate some random transactions.
- start_txn(batch);
- for (int t = 0; t < batch_size && !stop_hub; ++t) {
- char *txn_start = w.cur();
- Txn &txn = *batch.add_txn();
- txn.set_seqno(seqno);
- int count = randint(min_ops, max_ops + 1);
- start_op(txn);
- for (int o = 0; o < count; ++o) {
- Op *op = txn.add_op();
- int rtype = general_txns ? randint(3) : 1,
- rkey = randint(),
- rvalue = randint();
- op->set_type(types[rtype]);
- op->set_key(rkey);
- op->set_value(rvalue);
- }
- fin_op(txn);
-
- // Process immediately if not bcasting.
- if (fds.empty()) {
- --seqno;
- r.reset_range(txn_start, w.cur());
- if (!Types::is_pb()) txn.Clear();
- process_txn<Types, pb_traits>(g_map, txn, seqno, nullptr);
- }
-
- // Checkpoint.
- if (check_interval(seqno, yield_interval)) st_sleep(0);
- if (check_interval(seqno, issue_display)) {
- cout << "issued txn " << seqno << endl;
- if (timelim > 0 && current_time_millis() - start_time > timelim) {
- cout << "time's up; issued " << seqno << " txns in " << timelim
- << " ms" << endl;
- stop_hub.set();
- }
- }
-
- // For debugging purposes.
- if (issuing_interval > 0) {
- st_sleep(issuing_interval);
- }
-
- // Are we to accept a new joiner?
- if (seqno == accept_joiner_seqno) {
- accept_joiner.set();
- }
-
- // Set the stopping seqno.
- if (seqno == stop_on_seqno) {
- cout << "stopping on issue of seqno " << seqno << endl;
- stop_hub.set();
- }
-
- ++seqno;
- }
- fin_txn(batch);
-
- bool do_bcast = !fds.empty() && !suppress_txn_msgs;
- if (Types::is_pb()) {
- // Broadcast/log/serialize.
- if (force_ser || do_bcast || use_twal) {
- serbuf.clear();
- ser(serbuf, batch);
- if (do_bcast) bcastbuf(fds, serbuf);
- if (use_twal) g_twal->logbuf(serbuf);
- }
- } else {
- // Reset if we have nobody to send to (incl. disk) or if we actually have
- // no txns (possible due to loop structure; want to avoid to avoid
- // confusing with the 0-txn message signifying "prepare a recovery msg").
- if (!do_bcast && !use_twal) {
- w.reset();
- }
- }
-
- // Pause?
- if (do_pause)
- do_pause.waitreset();
- }
-
- // This means "The End."
- if (!fds.empty()) {
- w.mark();
- batch.Clear();
- start_txn(batch);
- Txn &txn = *batch.add_txn();
- txn.set_seqno(-1);
- start_op(txn);
- fin_op(txn);
- fin_txn(batch);
- if (Types::is_pb()) bcastmsg(fds, batch);
- w.mark_and_flush();
- }
-}
-
-#if 0
-template<typename Types, typename RTypes>
-void
-process_txn_ext(mii &map, const typename Types::Txn &txn, int &seqno,
- typename RTypes::Response *res, ext_map ext)
-{
- response
-}
-#endif
-
-/**
- * Process a transaction: update DB state (incl. seqno) and send response to
- * leader.
- */
-template<typename Types, typename RTypes>
-void
-process_txn(mii &map, const typename Types::Txn &txn, int &seqno,
- typename RTypes::Response *res)
-{
- typedef typename Types::Txn Txn;
- typedef typename Types::Op Op;
- checkeq(txn.seqno(), seqno + 1);
- seqno = txn.seqno();
- if (res != nullptr) {
- res->set_seqno(seqno);
- res->set_caught_up(true);
- start_result(*res);
- }
- if (!fake_exec) {
- for (int o = 0; o < txn.op_size(); ++o) {
- const Op &op = txn.op(o);
- const char type = op.type();
- const int key = op.key();
- mii::iterator it = map.find(key);
- if (show_updates || count_updates) {
- if (it != map.end()) {
- if (show_updates) cout << "existing key: " << key << endl;
- if (count_updates) ++updates;
- }
- }
- switch (type) {
- case Op::read:
- if (res != nullptr) {
- if (it == map.end()) res->add_result(0);
- else res->add_result(it->second);
- }
- break;
- case Op::write:
- {
- int value = op.value();
- if (use_pwal) g_wal->logwrite(key, value);
- if (it == map.end()) map[key] = value;
- else it->second = value;
- break;
- }
- case Op::del:
- if (it != map.end()) {
- if (use_pwal) g_wal->logdel(key);
- map.erase(it);
- }
- break;
- }
- }
- }
- if (res != nullptr)
- fin_result(*res);
- if (use_pwal) g_wal->logcommit();
-}
-
-void
-showdatarate(const char *action, streamoff len, long long time)
-{
- cout << action << " of " << len << " bytes in " << time << " ms ("
- << double(len) / double(time) / 1000 << " MB/s)" << endl;
-}
-
-void
-showdatarate(const char *action, size_t len, long long time)
-{
- cout << action << " of " << len << " bytes in " << time << " ms ("
- << double(len) / double(time) / 1000 << " MB/s)" << endl;
-}
-
-void
-showtput(const char *action, long long stop_time, long long start_time,
- int stop_count, int start_count)
-{
- long long time_diff = stop_time - start_time;
- int count_diff = stop_count - start_count;
- double rate = double(count_diff) * 1000. / double(time_diff);
- cout << action << " " << count_diff << " txns ["
- << start_count << ".." << stop_count
- << "] in " << time_diff << " ms ["
- << start_time << ".." << stop_time
- << "] ("
- << rate << " tps)" << endl;
-}
-
-/**
- * Return range * part / nparts, but with proper casting. Assumes that part <
- * nparts.
- */
-inline int
-interp(int range, int part, int nparts) {
- return static_cast<int>(static_cast<long long>(range) * part / nparts);
-}
-
-#src
-TEST(interp_test, basics) {
- EXPECT_EQ(0, interp(3, 0, 3));
- EXPECT_EQ(1, interp(3, 1, 3));
- EXPECT_EQ(2, interp(3, 2, 3));
- EXPECT_EQ(3, interp(3, 3, 3));
-
- EXPECT_EQ(0, interp(RAND_MAX, 0, 2));
- EXPECT_EQ(RAND_MAX / 2, interp(RAND_MAX, 1, 2));
- EXPECT_EQ(RAND_MAX, interp(RAND_MAX, 2, 2));
-}
-#end
-
-unique_ptr<TPCCTables> g_tables;
-
-void
-mkres(TpccRes *res, const OrderStatusOutput &output)
-{
- OrderStatusOutputMsg &msg = *res->mutable_order_status();
- msg.set_c_id(output.c_id);
- msg.set_c_balance(output.c_balance);
- msg.set_o_id(output.o_id);
- msg.set_o_carrier_id(output.o_carrier_id);
- foreach (const OrderStatusOutput::OrderLineSubset &src, output.lines) {
- OrderLineSubsetMsg &dst = *msg.add_line();
- dst.set_ol_i_id(src.ol_i_id);
- dst.set_ol_supply_w_id(src.ol_supply_w_id);
- dst.set_ol_quantity(src.ol_quantity);
- dst.set_ol_amount(src.ol_amount);
- dst.set_ol_delivery_d(src.ol_delivery_d);
- }
- msg.set_c_first(output.c_first);
- msg.set_c_middle(output.c_middle);
- msg.set_c_last(output.c_last);
- msg.set_o_entry_d(output.o_entry_d);
-}
-
-void
-mkres(TpccRes *res, const PaymentOutput &output)
-{
- PaymentOutputMsg &msg = *res->mutable_payment();
-
- WarehouseMsg &w = *msg.mutable_warehouse();
- w.set_w_id(output.warehouse.w_id);
- w.set_w_tax(output.warehouse.w_tax);
- w.set_w_ytd(output.warehouse.w_ytd);
- w.set_w_name(output.warehouse.w_name);
- w.set_w_street_1(output.warehouse.w_street_1);
- w.set_w_street_2(output.warehouse.w_street_2);
- w.set_w_city(output.warehouse.w_city);
- w.set_w_state(output.warehouse.w_state);
- w.set_w_zip(output.warehouse.w_zip);
-
- DistrictMsg &d = *msg.mutable_district();
- d.set_d_id(output.district.d_id);
- d.set_d_w_id(output.district.d_w_id);
- d.set_d_tax(output.district.d_tax);
- d.set_d_ytd(output.district.d_ytd);
- d.set_d_next_o_id(output.district.d_next_o_id);
- d.set_d_name(output.district.d_name);
- d.set_d_street_1(output.district.d_street_1);
- d.set_d_street_2(output.district.d_street_2);
- d.set_d_city(output.district.d_city);
- d.set_d_state(output.district.d_state);
- d.set_d_zip(output.district.d_zip);
-
- CustomerMsg &c = *msg.mutable_customer();
- c.set_c_id(output.customer.c_id);
- c.set_c_d_id(output.customer.c_d_id);
- c.set_c_w_id(output.customer.c_w_id);
- c.set_c_credit_lim(output.customer.c_credit_lim);
- c.set_c_discount(output.customer.c_discount);
- c.set_c_balance(output.customer.c_balance);
- c.set_c_ytd_payment(output.customer.c_ytd_payment);
- c.set_c_payment_cnt(output.customer.c_payment_cnt);
- c.set_c_delivery_cnt(output.customer.c_delivery_cnt);
- c.set_c_first(output.customer.c_first);
- c.set_c_middle(output.customer.c_middle);
- c.set_c_last(output.customer.c_last);
- c.set_c_street_1(output.customer.c_street_1);
- c.set_c_street_2(output.customer.c_street_2);
- c.set_c_city(output.customer.c_city);
- c.set_c_state(output.customer.c_state);
- c.set_c_zip(output.customer.c_zip);
- c.set_c_phone(output.customer.c_phone);
- c.set_c_since(output.customer.c_since);
- c.set_c_credit(output.customer.c_credit);
- c.set_c_data(output.customer.c_data);
-}
-
-void
-process_tpcc(const TpccReq &req, int &seqno, TpccRes *res)
-{
- checkeq(req.seqno(), seqno + 1);
- ++seqno;
- if (res != nullptr) {
- res->Clear();
- res->set_seqno(seqno);
- }
- // First three are read-only txns, so doesn't make sense to exec if no res to
- // put results. They constitute only 8% of the workload.
- if (req.has_stock_level()) {
- if (res != nullptr) {
- const StockLevelMsg &sl = req.stock_level();
- int result = g_tables->stockLevel(sl.warehouse_id(), sl.district_id(), sl.threshold());
- StockLevelOutputMsg &msg = *res->mutable_stock_level();
- msg.set_result(result);
- }
- } else if (req.has_order_status_1()) {
- if (res != nullptr) {
- const OrderStatusMsg1 &os = req.order_status_1();
- OrderStatusOutput output;
- g_tables->orderStatus(os.warehouse_id(), os.district_id(), os.customer_id(), &output);
- mkres(res, output);
- }
- } else if (req.has_order_status_2()) {
- if (res != nullptr) {
- const OrderStatusMsg2 &os = req.order_status_2();
- OrderStatusOutput output;
- g_tables->orderStatus(os.warehouse_id(), os.district_id(), os.c_last().c_str(), &output);
- mkres(res, output);
- }
- } else if (req.has_new_order()) {
- const NewOrderMsg &no = req.new_order();
- vector<NewOrderItem> items(no.item_size());
- for (int i = 0; i < no.item_size(); ++i) {
- NewOrderItem &dst = items[i];
- const NewOrderItemMsg &src = no.item(i);
- dst.i_id = src.i_id();
- dst.ol_supply_w_id = src.ol_supply_w_id();
- dst.ol_quantity = src.ol_quantity();
- }
- NewOrderOutput output;
- g_tables->newOrder(no.warehouse_id(), no.district_id(),
- no.customer_id(), items, no.now().c_str(),
- &output);
- if (res != nullptr) {
- NewOrderOutputMsg &msg = *res->mutable_new_order();
- msg.set_w_tax(output.w_tax);
- msg.set_d_tax(output.d_tax);
- msg.set_o_id(output.o_id);
- msg.set_c_discount(output.c_discount);
- msg.set_total(output.total);
- foreach (const NewOrderOutput::ItemInfo &src, output.items) {
- ItemInfoMsg &dst = *msg.add_item();
- dst.set_s_quantity(src.s_quantity);
- dst.set_i_price(src.i_price);
- dst.set_ol_amount(src.ol_amount);
- dst.set_brand_generic(src.brand_generic);
- dst.set_i_name(src.i_name);
- }
- msg.set_c_last(output.c_last);
- msg.set_c_credit(output.c_credit);
- msg.set_status(output.status);
- }
- } else if (req.has_payment_1()) {
- const PaymentMsg1 &p = req.payment_1();
- PaymentOutput output;
- g_tables->payment(p.warehouse_id(), p.district_id(), p.c_warehouse_id(),
- p.c_district_id(), p.customer_id(), p.h_amount(),
- p.now().c_str(), &output);
- if (res != nullptr) mkres(res, output);
- } else if (req.has_payment_2()) {
- const PaymentMsg2 &p = req.payment_2();
- PaymentOutput output;
- g_tables->payment(p.warehouse_id(), p.district_id(), p.c_warehouse_id(),
- p.c_district_id(), p.c_last().c_str(), p.h_amount(),
- p.now().c_str(), &output);
- if (res != nullptr) mkres(res, output);
- } else if (req.has_delivery()) {
- const DeliveryMsg &d = req.delivery();
- vector<DeliveryOrderInfo> orders;
- g_tables->delivery(d.warehouse_id(), d.carrier_id(), d.now().c_str(), &orders);
- if (res != nullptr) {
- DeliveryOutputMsg &msg = *res->mutable_delivery();
- foreach (const DeliveryOrderInfo &src, orders) {
- DeliveryOrderInfoMsg &dst = *msg.add_order();
- dst.set_d_id(src.d_id);
- dst.set_o_id(src.o_id);
- }
- }
- } else {
- ASSERT(false);
- }
-}
-
-void
-process_tpccs(st_netfd_t leader, int &seqno,
- st_channel<recovery_t> &send_states,
- st_channel<chunk> &backlog, int init_seqno,
- int mypos, int nnodes)
-{
- bool caught_up = init_seqno == 0;
- // Means we're currently ignoring the incoming txns until we see a fail-ack
- // from the leader.
- bool depleting = false;
- long long start_time = current_time_millis(),
- time_failed = -1,
- time_caught_up = caught_up ? start_time : -1;
- int seqno_caught_up = caught_up ? seqno : -1;
- // Used by joiner only to tell where we actually started (init_seqno is just
- // the seqno reported by the leader in the Init message, but it may have
- // issued more since the Init message).
- int first_seqno = -1;
- char *marker = nullptr;
- int first_seqno_in_chunk = -1;
- TpccReq req;
- TpccRes res;
- txn_wal &wal = *g_twal;
-
- function<void(anchored_stream_reader& reader)> overflow_fn =
- lambda(anchored_stream_reader &reader) {
- if (__ref(caught_up)) {
- // Anchor should already be correctly set, so just shift down.
- shift_reader(reader);
- } else if (__ref(first_seqno_in_chunk) == __ref(seqno) + 1) {
- // Has the replayer just caught up to the start of the chunk?
- ASSERT(reader.buf().get() == reader.anchor());
- // Replay all messages up to but not included the current unprocessed
- // message (which we may be in the middle of receiving, triggering this
- // overflow).
- process_buf(reader.anchor(), __ref(marker), __ref(req), __ref(seqno));
- // Update the anchor to point to the unprocessed message, so that we
- // shift the unprocessed message down.
- reader.anchor() = __ref(marker);
- shift_reader(reader);
- } else {
- // Push onto backlog and put in new buffer.
- ASSERT(reader.buf().get() == reader.anchor());
- __ref(backlog).push(make_tuple(reader.buf(), reader.anchor(), __ref(marker)));
- reader.anchor() = __ref(marker);
- replace_reader(reader);
- cout << "added to backlog, now has " << __ref(backlog).queue().size()
- << " chunks" << endl;
- }
- __ref(marker) = reader.buf().get();
- };
-
- sized_array<char> rbuf(new char[read_buf_size], read_buf_size);
- commons::array<char> wbuf(buf_size);
- anchored_stream_reader reader(st_read_fn(leader), st_read_fully_fn(leader),
- overflow_fn, rbuf.get(), rbuf.size());
- writer w(lambda(const void *buf, size_t len) {
- st_write(__ref(leader), buf, len);
- }, wbuf.get(), wbuf.size());
-
- finally f(lambda () {
- long long now = current_time_millis();
- showtput("processed", now, __ref(start_time), __ref(seqno),
- __ref(init_seqno));
- if (!__ref(caught_up)) {
- cout << "live-processing: never entered this phase (never caught up)" << endl;
- } else {
- showtput("live-processed", now, __ref(time_caught_up), __ref(seqno),
- __ref(seqno_caught_up));
- }
- __ref(send_states).push(recovery_t());
- __ref(w).mark_and_flush();
- });
-
- function<void()> send_failure_msg = lambda() {
- TpccRes &res = __ref(res);
- writer &w = __ref(w);
- res.Clear();
- res.set_seqno(-1);
- ser(w, res);
- w.mark_and_flush();
- };
-
- while (true)
- {
- marker = reader.start();
-
- {
- st_intr intr(stop_hub);
- readmsg(reader, req);
- }
-
- if (req.seqno() == -1) {
- // End of stream.
- break;
- } else if (req.seqno() == -2) {
- // Prepare recovery msg.
- send_states.push(make_tpcc_recovery(mypos, nnodes, seqno));
- } else {
-
- if (depleting) {
- if (req.seqno() == -3) {
- // Fail-ack. Should not be receiving anything until we resume.
- failed.waitreset();
- send_failure_msg();
- // Note that we don't reset depleting; we want the next iteration to
- // fall through to the next case in this if-else chain....
-
- // Adjust reader so that the next xact (the first one after failure)
- // will go to the start of the buffer; this is necessary for
- // backlogging.
- reader.set_anchor();
- shift_reader(reader);
- } else if (!failed) {
- // This is the first txn after resuming. Tell the recoverer task
- // that this is the seqno to build up to (from another replica's
- // log).
- resume.push(req.seqno());
- depleting = false;
- }
- // Ignore all other messages.
- }
-
- if (!depleting) {
- if (req.seqno() == -3) {
- // Ignore the fail-ack.
- } else {
- if (use_twal) wal.logbuf(marker, reader.start() - marker);
-
- // Backlog (auto/implicit) or process.
- if (!caught_up) {
- // If we were at the start of a new buffer (our chunk was recently reset).
- if (reader.buf().get() == marker)
- first_seqno_in_chunk = req.seqno();
- // If we fully caught up.
- if (req.seqno() == seqno + 1) {
- time_caught_up = current_time_millis();
- seqno_caught_up = seqno;
- showtput("process_tpccs caught up; backlogged",
- time_caught_up, start_time, seqno_caught_up,
- first_seqno == -1 ? init_seqno - 1 : first_seqno);
- caught_up = true;
- }
- }
- if (caught_up) {
- // Process.
- process_tpcc(req, seqno, &res);
- ser(w, res);
- reader.set_anchor();
- }
-
- // Display/yield.
- if (check_interval(req.seqno(), process_display))
- cout << (caught_up ? "processed req " : "backlogged req ")
- << req.seqno() << endl;
- if (check_interval(req.seqno(), yield_interval)) st_sleep(0);
-
- // Die.
- if (fail_seqno > 0 && req.seqno() == fail_seqno) {
- cout << "process_tpccs failing on seqno " << fail_seqno;
- time_failed = current_time_millis();
- showtput("; live-processed ", time_failed, start_time, seqno, 0);
- ASSERT(init_seqno == 0);
- caught_up = false;
- depleting = true;
- seqno = -1;
-
- failed.set();
- send_failure_msg();
- }
- }
- }
-
- }
- }
-
-}
-
-void
-process_buf(char *begin, char *end, TpccReq &req, int &seqno)
-{
- ASSERT(begin < end);
- raw_reader reader(begin);
- while (reader.ptr() < end) {
- uint32_t len = ntohl(reader.read<uint32_t>());
- ASSERT(len < 10000);
- ASSERT(reinterpret_cast<char*>(reader.ptr()) + len <= end);
- check(req.ParseFromArray(reader.readptr(len), len));
- process_tpcc(req, seqno, nullptr);
- if (check_interval(req.seqno(), yield_interval)) st_sleep(0);
- if (check_interval(req.seqno(), process_display)) {
- cout << "caught up to req " << req.seqno() << endl;
- }
- }
-}
-
-recovery_t
-make_tpcc_recovery(int mypos, int nnodes, int seqno)
-{
- long long start_time = current_time_millis();
- cout << "serializing recovery, db state is now at seqno "
- << seqno << ":" << endl;
- g_tables->show();
- recovery_t recovery = g_tables->ser(mypos, nnodes, seqno);
- showdatarate("serialized recovery", recovery.size(),
- current_time_millis() - start_time);
- return recovery;
-}
-
-/**
- * Actually do the work of executing a transaction and sending back the reply.
- *
- * \param[in] leader The connection to the leader.
- *
- * \param[in] map The data store.
- *
- * \param[in] seqno The sequence number last seen. This always starts at 0,
- * but may be bumped up by the recovery procedure.
- *
- * \param[in] send_states Channel of snapshots of the database state to send to
- * recovering nodes (sent to recover_joiner).
- *
- * \param[in] backlog The backlog of txns that need to be processed.
- *
- * \param[in] init_seqno The seqno that was sent in the Init message from the
- * leader. The first expected seqno.
- *
- * \param[in] mypos This host's position in the Init message list. Used for
- * calculating the sub-range of the map for which this node is responsible.
- *
- * \param[in] nnodes The total number nodes in the Init message list.
- *
- * \param[in] wal The WAL.
- */
-template<typename Types, typename RTypes>
-void
-process_txns(st_netfd_t leader, mii &map, int &seqno,
- st_channel<recovery_t> &send_states,
- /* XXX st_channel<shared_ptr<pb::Txn> > &backlog */
- st_channel<chunk> &backlog, int init_seqno,
- int mypos, int nnodes)
-{
- typedef typename Types::TxnBatch TxnBatch;
- typedef typename Types::Txn Txn;
- typedef typename Types::Op Op;
- typedef typename RTypes::Response Response;
- typedef typename RTypes::ResponseBatch ResponseBatch;
-
- bool caught_up = init_seqno == 0;
- long long start_time = current_time_millis(),
- time_caught_up = caught_up ? start_time : -1;
- int seqno_caught_up = caught_up ? seqno : -1;
- // Used by joiner only to tell where we actually started (init_seqno is just
- // the seqno reported by the leader in the Init message, but it may have
- // issued more since the Init message).
- int first_seqno = -1;
-
- sized_array<char> rbuf(new char[read_buf_size], read_buf_size);
- commons::array<char> wbuf(buf_size);
- st_reader reader(leader, rbuf.get(), rbuf.size());
- writer w(lambda(const void *buf, size_t len) {
- checkeqnneg(st_write(__ref(leader), buf, len, ST_UTIME_NO_TIMEOUT),
- static_cast<ssize_t>(len));
- }, wbuf.get(), wbuf.size());
- stream s(reader, w);
-
- finally f(lambda () {
- long long now = current_time_millis();
- showtput("processed", now, __ref(start_time), __ref(seqno),
- __ref(init_seqno));
- if (!__ref(caught_up)) {
- cout << "live-processing: never entered this phase (never caught up)" << endl;
- } else {
- showtput("live-processed", now, __ref(time_caught_up), __ref(seqno),
- __ref(seqno_caught_up));
- }
- __ref(send_states).push(recovery_t());
- __ref(w).mark_and_flush();
- });
-
- try {
- scoped_ptr<TxnBatch> pbatch(new_TxnBatch<TxnBatch>(s));
- TxnBatch &batch = *pbatch;
- scoped_ptr<ResponseBatch> presbatch(new_ResponseBatch<ResponseBatch>(s));
- ResponseBatch &resbatch = *presbatch;
- ser_t serbuf;
- char *first_start = reader.start();
- ASSERT(first_start == rbuf.get());
- const size_t headerlen = sizeof(uint32_t) + sizeof(short) + sizeof(int);
- while (true) {
- uint32_t prefix = 0;
- char *start = reader.start();
-
- // Will overflow on next few reads ("header")?
- if (!caught_up && reader.unread() + reader.rem() < headerlen) {
- sized_array<char> buf(new char[read_buf_size], read_buf_size);
- memcpy(buf.get(), reader.start(), reader.unread());
- swap(buf, reader.buf());
- reader.reset_range(reader.buf().get(), reader.buf().get() + reader.unread());
- backlog.push(make_tuple(buf, first_start, start));
- first_start = reader.start();
- }
-
- if (Types::is_pb()) {
- long long before_read = -1;
- if (read_thresh > 0) {
- before_read = current_time_millis();
- }
- {
- st_intr intr(stop_hub);
- readmsg(reader, batch);
- }
- if (read_thresh > 0) {
- long long read_time = current_time_millis() - before_read;
- if (read_time > read_thresh) {
- cout << "thread " << threadname()
- << ": read took " << read_time << " ms" << endl;
- }
- }
- } else {
- st_intr intr(stop_hub);
- prefix = ntohl(reader.read<uint32_t>());
- check(prefix < 10000);
- batch.Clear();
- }
-
- if (batch.txn_size() > 0) {
- const Txn &first_txn = batch.txn(0);
- if (first_txn.seqno() < 0) {
- break;
- } else if (first_txn.seqno() > seqno + 1) {
- // In backlogging mode?
-
- // Skip entire message, pushing it to the thread that's handling
- // recovery for later processing once snapshot is received.
- // TODO: implement and use anchors instead?
- if (first_seqno == -1)
- cout << "first seqno: " << (first_seqno = first_txn.seqno()) << endl;
-
- // Caught up?
- if (first_seqno == seqno + 1) {
- // Rewind so we process accumulated messages.
- reader.reset_range(first_start, reader.end());
- continue;
- }
-
- // About to overflow?
- if (reader.unread() + reader.rem() < prefix + sizeof(uint32_t) - headerlen) {
- // Move current partial message to new buffer.
- sized_array<char> tmp(new char[read_buf_size], read_buf_size);
- raw_writer ser(tmp.get());
- ser.write(prefix);
- ser.write(short(batch.txn_size()));
- ser.write(first_txn.seqno());
- memcpy(tmp.get() + headerlen, reader.start(), reader.unread());
-
- // Swap the buffers.
- swap(tmp, reader.buf());
- reader.reset_range(reader.buf().get() + headerlen, reader.buf().get() + headerlen + reader.unread());
- ASSERT(tmp.get() <= first_start && first_start < tmp.end());
- ASSERT(tmp.get() < start && start < tmp.end());
- ASSERT(first_start < start);
- backlog.push(make_tuple(tmp, first_start, start));
- first_start = reader.buf().get();
- first_seqno = first_txn.seqno();
- }
-
- // Fill up rest of the message
- ASSERT(reader.unread() + reader.rem() >= prefix + sizeof(uint32_t) - headerlen);
- check0x(reader.accum(prefix + sizeof(uint32_t) - headerlen));
- } else {
- // Regular transaction batch.
- if (!caught_up) {
- time_caught_up = current_time_millis();
- seqno_caught_up = seqno;
- showtput("process_txns caught up; backlogged",
- time_caught_up, start_time, seqno_caught_up,
- first_seqno == -1 ? init_seqno - 1 : first_seqno);
- caught_up = true;
- }
- w.mark();
- resbatch.Clear();
- start_res(resbatch);
- for (int t = 0; t < batch.txn_size(); ++t) {
- const Txn &txn = t == 0 ? first_txn : batch.txn(t);
- Response *res = resbatch.add_res();
- process_txn<Types, RTypes>(map, txn, seqno, res);
-#if 0
- if (!sending_recovery) {
- process_txn<Types, RTypes>(map, txn, seqno, res);
- } else {
- process_txn_ext(map, txn, seqno, res, ext);
- }
-#endif
- if (fake_exec && !Types::is_pb()) {
- reader.skip(txn.op_size() * Op_Size);
- }
-
- if (check_interval(txn.seqno(), yield_interval)) st_sleep(0);
- if (check_interval(txn.seqno(), process_display)) {
- cout << "processed txn " << txn.seqno()
- << "; db size = " << map.size()
- << "; seqno = " << seqno
- << "; backlog.size = " << backlog.queue().size() << endl;
- }
- }
- fin_res(resbatch);
- if (RTypes::is_pb() && resbatch.res_size() > 0) {
- serbuf.clear();
- ser(serbuf, resbatch);
- sendbuf(leader, serbuf);
- }
- }
- } else if (multirecover || mypos == 0) {
- // Empty (default) TxnBatch means "generate a snapshot."
- send_states.push(make_recovery(map, mypos, nnodes, seqno));
- }
- }
- } catch (break_exception &ex) {
- }
-
-}
-
-#if 0
-template<typename mii>
-unique_ptr<Recovery>
-make_recovery(const mii &map, int mypos, int nnodes, int seqno)
-{
- // TODO make this faster
- cout << "generating recovery..." << endl;
- unique_ptr<Recovery> recovery(new Recovery);
- typedef ::map<int, int> mii_;
- mii_ map_(map.begin(), map.end());
- mii_::const_iterator begin =
- map_.lower_bound(multirecover ? interp(RAND_MAX, mypos, nnodes) : 0);
- mii_::const_iterator end = multirecover && mypos < nnodes - 1 ?
- map_.lower_bound(interp(RAND_MAX, mypos + 1, nnodes)) : map_.end();
- cout << "generating recovery over " << begin->first << ".."
- << (end == map_.end() ? "end" : lexical_cast<string>(end->first));
- if (multirecover)
- cout << " (node " << mypos << " of " << nnodes << ")";
- cout << endl;
- long long start_snap = current_time_millis();
- foreach (const pii &p, make_iterator_range(begin, end)) {
- Recovery_Pair *pair = recovery->add_pair();
- pair->set_key(p.first);
- pair->set_value(p.second);
- }
- cout << "generating recovery took "
- << current_time_millis() - start_snap << " ms" << endl;
- recovery->set_seqno(seqno);
- return move(recovery);
-}
-#endif
-
-template<typename mii>
-recovery_t
-make_recovery(const mii &map, int mypos, int nnodes, int seqno)
-{
- return recovery_t();
-}
-
-struct recovery_header
-{
- int seqno;
- size_t count;
- size_t total;
- size_t size;
-};
-
-pair<size_t, size_t>
-recovery_range(size_t size, int mypos, int nnodes)
-{
- return make_pair(multirecover ? size * mypos / size_t(nnodes) : 0,
- multirecover ? size * (mypos + 1) / size_t(nnodes) : size);
-}
-
-template<>
-recovery_t
-make_recovery(const snap_map<int, int> &map, int mypos, int nnodes, int seqno)
-{
- const commons::array<entry> &src = map.get_table();
- pair<size_t, size_t> range = recovery_range(src.size(), mypos, nnodes);
- size_t begin = range.first, end = range.second;
- ASSERT(end > begin);
- recovery_header hdr = { seqno, end - begin, src.size(), map.size() };
- size_t bodylen = sizeof(entry) * hdr.count;
- cout << "generating recovery of " << hdr.size << " records in "
- << hdr.count << " slots ("
- << bodylen << " bytes); range is ["
- << begin << ".." << end << "]; seqno is " << hdr.seqno << endl;
- long long start_time = current_time_millis();
- commons::array<char> recovery(sizeof(size_t) + sizeof hdr + bodylen);
- raw_writer ser(recovery.begin());
- ser.write(recovery.size());
- ser.write(hdr);
- memcpy(ser.ptr(), src.begin() + begin, bodylen);
- showdatarate("serialized recovery", recovery.size(),
- current_time_millis() - start_time);
- return recovery;
-}
-
class response_handler
{
public:
@@ -1804,156 +379,6 @@
h.run<Types>();
}
-class tpcc_response_handler
-{
-public:
- tpcc_response_handler(st_netfd_t replica, const int &seqno, int rid,
- st_multichannel<long long> &recover_signals,
- st_channel<st_netfd_t> &delreps, bool caught_up)
- :
- replica(replica),
- seqno(seqno),
- rid(rid),
- recover_signals(recover_signals),
- delreps(delreps),
- caught_up(caught_up),
- sub(recover_signals.subscribe()),
- start_time(current_time_millis()),
- recovery_start_time(caught_up ? -1 : start_time),
- recovery_end_time(-1),
- start_seqno(seqno),
- recovery_start_seqno(caught_up ? -1 : seqno),
- recovery_end_seqno(-1),
- last_seqno(-1)
- {}
-
- void run() {
- finally f(bind(&tpcc_response_handler::cleanup, this));
-
- commons::array<char> rbuf(read_buf_size), wbuf(buf_size);
- st_reader reader(replica, rbuf.get(), rbuf.size());
- writer w(lambda(const void*, size_t) {
- throw not_supported_exception("response handler should not be writing");
- }, wbuf.get(), wbuf.size());
- stream s(reader,w);
-
- long long last_display_time = current_time_millis();
-
- function<void()> loop_cleanup =
- bind(&tpcc_response_handler::loop_cleanup, this);
-
- TpccRes res;
-
- while (true) {
- finally f(loop_cleanup);
-
- // Read the message, but correctly respond to interrupts so that we can
- // cleanly exit (slightly tricky).
- if (stopped_issuing && last_seqno + 1 == seqno) {
- break;
- } else {
- st_intr intr(kill_hub);
- readmsg(reader, res);
- }
-
- if (res.seqno() == -1) {
- st_intr intr(stop_hub);
- cout << "got a failed node" << endl;
- delreps.push(replica);
- readmsg(reader, res);
- last_seqno = seqno - 1;
- } else {
-
- if (res.seqno() < last_seqno)
- throw msg_exception(string("response seqno decreased from ") +
- lexical_cast<string>(last_seqno) + " to " +
- lexical_cast<string>(res.seqno()));
-
- if (!caught_up) {
- long long now = current_time_millis(), time_diff = now - start_time;
- caught_up = true;
- recover_signals.push(now);
- cout << rid << ": " << "recovering node caught up; took "
- << time_diff << " ms" << endl;
- // This will cause the program to exit eventually, but cleanly, such that
- // the recovery time will be set first, before the eventual exit (which
- // may not even happen in the current iteration).
- if (stop_on_recovery) {
- cout << "stopping on recovery" << endl;
- stop_hub.set();
- }
- }
-
- if (check_interval(res.seqno(), handle_responses_display)) {
- cout << rid << ": " << "got response " << res.seqno() << " from "
- << replica << "; ";
- long long display_time = current_time_millis();
- showtput("handling", display_time, last_display_time, res.seqno(),
- res.seqno() - handle_responses_display);
- last_display_time = display_time;
- }
- if (check_interval(res.seqno(), yield_interval)) {
- st_sleep(0);
- }
- last_seqno = res.seqno();
-
- }
- }
- }
-
-private:
- void loop_cleanup() {
- // The first timestamp that comes down the subscription pipeline is the
- // recovery start time, issued by the main thread. The second one is the
- // recovery end time, issued by the response handler associated with the
- // joiner.
- if (recovery_start_time == -1 && !sub.empty()) {
- recovery_start_time = sub.take();
- recovery_start_seqno = last_seqno;
- cout << rid << ": ";
- showtput("before recovery, finished", recovery_start_time, start_time,
- recovery_start_seqno, 0);
- } else if (recovery_end_time == -1 && !sub.empty()) {
- recovery_end_time = sub.take();
- recovery_end_seqno = last_seqno;
- cout << rid << ": ";
- showtput("during recovery, finished roughly", recovery_end_time,
- recovery_start_time, recovery_end_seqno, recovery_start_seqno);
- }
- }
-
- void cleanup() {
- long long end_time = current_time_millis();
- cout << rid << ": ";
- showtput("handled roughly", end_time, start_time, seqno, start_seqno);
- if (recovery_end_time > -1) {
- cout << rid << ": ";
- showtput("after recovery, finished", end_time, recovery_end_time,
- seqno, recovery_end_seqno);
- }
- }
-
- st_netfd_t replica;
- const int &seqno;
- int rid;
- st_multichannel<long long> &recover_signals;
- st_channel<st_netfd_t> &delreps;
- bool caught_up;
- st_channel<long long> ⊂
- long long start_time, recovery_start_time, recovery_end_time;
- int start_seqno, recovery_start_seqno, recovery_end_seqno, last_seqno;
-};
-
-void
-handle_tpcc_responses(st_netfd_t replica, const int &seqno, int rid,
- st_multichannel<long long> &recover_signals,
- st_channel<st_netfd_t> &delreps, bool caught_up)
-{
- tpcc_response_handler h(replica, seqno, rid, recover_signals, delreps,
- caught_up);
- h.run();
-}
-
struct recreq {
int start_seqno, end_seqno;
};
@@ -2048,1305 +473,3 @@
cout << "AAAAAAAAAAAAAAAAAAAAAA" << endl;
}
}
-
-/**
- * Run the leader.
- */
-template<typename Types, typename RTypes>
-void
-run_leader(int minreps, uint16_t leader_port)
-{
- cout << "starting as leader" << endl;
- st_multichannel<long long> recover_signals;
-
- scoped_ptr<txn_wal> twal(new txn_wal(use_twal ? "twal" : "/dev/null"));
- g_twal = twal.get();
- scoped_ptr<wal> pwal(new wal(use_pwal ? "pwal" : "/dev/null"));
- g_wal = pwal.get();
-
- // Wait until all replicas have joined.
- st_netfd_t listener = st_tcp_listen(leader_port);
- st_closing close_listener(listener);
- vector<replica_info> replicas;
- st_closing_all_infos close_replicas(replicas);
- cout << "waiting for at least " << minreps << " replicas to join" << endl;
- for (int i = 0; i < minreps; ++i) {
- st_netfd_t fd;
- {
- st_intr intr(stop_hub);
- fd = checkerr(st_accept(listener, nullptr, nullptr,
- ST_UTIME_NO_TIMEOUT));
- }
- Join join = readmsg<Join>(fd);
- replicas.push_back(replica_info(fd, static_cast<uint16_t>(join.port())));
- }
- cout << "got all " << minreps << " replicas" << endl;
-
- // Construct the initialization message.
- Init init;
- init.set_txnseqno(0);
- init.set_multirecover(multirecover);
- foreach (replica_info r, replicas) {
- SockAddr *psa = init.add_node();
- psa->set_host(r.host());
- psa->set_port(r.port());
- }
-
- // Send init to each initial replica.
- foreach (replica_info r, replicas) {
- init.set_yourhost(r.host());
- sendmsg(r.fd(), init);
- }
-
- // Start dispatching queries.
- st_bool accept_joiner;
- int seqno = 0;
- st_channel<replica_info> newreps;
- st_channel<st_netfd_t> delreps;
- foreach (const replica_info &r, replicas) newreps.push(r);
- function<void()> f;
- if (do_tpcc)
- f = bind(issue_tpcc, ref(newreps), ref(delreps), ref(seqno), ref(accept_joiner));
- else
- f = bind(issue_txns<Types>, ref(newreps), ref(seqno), ref(accept_joiner));
- st_joining join_issue_txns(my_spawn(f, "issue_txns"));
-
- finally fin(bind(summarize, "LEADER", ref(seqno)));
-
- try {
- // Start handling responses.
- st_thread_group handlers;
- int rid = 0;
- foreach (replica_info r, replicas) {
- function<void()> fn;
- if (do_tpcc)
- fn = bind(handle_tpcc_responses, r.fd(), ref(seqno), rid++,
- ref(recover_signals), ref(delreps), true);
- else
- fn = bind(handle_responses<RTypes>, r.fd(), ref(seqno), rid++,
- ref(recover_signals), true);
- handlers.insert(my_spawn(fn, "handle_responses"));
- }
-
- // Accept the recovering node, and tell it about the online replicas.
- st_netfd_t joiner;
- try {
- st_intr intr(stop_hub);
- joiner = checkerr(st_accept(listener, nullptr, nullptr,
- ST_UTIME_NO_TIMEOUT));
- accept_joiner.waitset();
- } catch (std::exception &ex) {
- string s(ex.what());
- if (s.find("Interrupted system call") == s.npos)
- throw;
- else
- throw break_exception();
- }
- Join join = readmsg<Join>(joiner);
- replicas.push_back(replica_info(joiner, static_cast<uint16_t>(join.port())));
- cout << "setting seqno to " << seqno << endl;
- init.set_txnseqno(seqno);
- init.set_yourhost(replicas.back().host());
- sendmsg(joiner, init);
- recover_signals.push(current_time_millis());
-
- // Start streaming txns to joiner.
- cout << "start streaming txns to joiner" << endl;
- function<void()> handle_responses_joiner_fn;
- if (do_tpcc)
- handle_responses_joiner_fn =
- bind(handle_tpcc_responses, joiner, ref(seqno), rid++,
- ref(recover_signals), ref(delreps), false);
- else
- ...
[truncated message content] |
|
From: <yan...@us...> - 2009-03-20 05:35:55
|
Revision: 1312
http://assorted.svn.sourceforge.net/assorted/?rev=1312&view=rev
Author: yangzhang
Date: 2009-03-20 05:35:47 +0000 (Fri, 20 Mar 2009)
Log Message:
-----------
updated macro demo
Modified Paths:
--------------
sandbox/trunk/src/c/macros.c
Modified: sandbox/trunk/src/c/macros.c
===================================================================
--- sandbox/trunk/src/c/macros.c 2009-03-19 22:19:57 UTC (rev 1311)
+++ sandbox/trunk/src/c/macros.c 2009-03-20 05:35:47 UTC (rev 1312)
@@ -12,6 +12,14 @@
// Not really a preprocessor extension.
#define dup(x,y) typeof(x) y = x;
+// Can we define a macro that defines macros? No....
+// If we try to evaluate this we get
+// error: '#' is not followed by a macro parameter
+#if 0
+#define defmacros() \
+#define foo bar
+#endif
+
int
main()
{
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <yan...@us...> - 2009-03-19 22:20:20
|
Revision: 1311
http://assorted.svn.sourceforge.net/assorted/?rev=1311&view=rev
Author: yangzhang
Date: 2009-03-19 22:19:57 +0000 (Thu, 19 Mar 2009)
Log Message:
-----------
- moved a bunch of the code gen helper functions into the CodeGen class as
methods (so that they may know the prefix)
- added the ability to specify a prefix for all the generated class names and
file names
- updated Makefile to remove all garbage and to use the prefix to avoid
inter-test clobbering
- added boost program_options for command-line argument parsing
Modified Paths:
--------------
clamp/trunk/src/CodeGen.cc
clamp/trunk/src/CodeGen.hh
clamp/trunk/src/Makefile
clamp/trunk/src/clamp.y
Modified: clamp/trunk/src/CodeGen.cc
===================================================================
--- clamp/trunk/src/CodeGen.cc 2009-03-19 18:02:16 UTC (rev 1310)
+++ clamp/trunk/src/CodeGen.cc 2009-03-19 22:19:57 UTC (rev 1311)
@@ -113,12 +113,14 @@
// Constructors
////////////////////////////////////////////////////////////////////////
- CodeGen::CodeGen ()
+ CodeGen::CodeGen (const string &prefix)
: mStack ()
, mLambdaCount (0)
- , mImplStream ("lambda_impl.clamp_h")
+ , mImplStream ((prefix + "_lambda_impl.clamp_h").c_str())
+ , mGenrStream ((prefix + "_lambda_genr.clamp_h").c_str())
, mLambdaMap ()
, mGeneratorStream ()
+ , mPrefix (prefix)
{
}
@@ -138,16 +140,17 @@
{
}
- void implTemplateName (ostream &strm, int ccount, int pcount)
+ void CodeGen::implTemplateName (ostream &strm, int ccount, int pcount)
{
strm
- << "lambda_template_"
+ << mPrefix
+ << "_lambda_template_"
<< ccount
<< "_"
<< pcount;
}
- void tList (ostream &strm, int start, int end, bool first = true)
+ void CodeGen::tList (ostream &strm, int start, int end, bool first)
{
for (int count = start; count <= end; ++count)
{
@@ -162,7 +165,7 @@
}
}
- void implFunctionVar (ostream &strm, int ccount, int pcount, const char *n)
+ void CodeGen::implFunctionVar (ostream &strm, int ccount, int pcount, const char *n)
{
strm << "T1 (";
@@ -179,7 +182,7 @@
strm << ")";
}
- void implInstance (ostream &strm, lambdaT const &lam)
+ void CodeGen::implInstance (ostream &strm, lambdaT const &lam)
{
unsigned ccount = count (lam.context.get());
unsigned pcount = count (lam.params.get());
@@ -229,14 +232,15 @@
int CodeGen::LambdaContext::sContextCount = 0;
- stringT bodyFileName (int count)
+ stringT CodeGen::bodyFileName (int count)
{
char name[PATH_MAX];
- sprintf (name, "lambda_body_%d.clamp_h", count);
+ snprintf (name, PATH_MAX, "%s_lambda_body_%d.clamp_h",
+ mPrefix.c_str(), count);
return stringT (name);
}
- void explicitConstructor (ostream &strm, int id, const lambdaT &lam)
+ void CodeGen::explicitConstructor (ostream &strm, int id, const lambdaT &lam)
{
if (lam.onHeap)
{
@@ -279,7 +283,7 @@
strm << ">";
}
- void startExplicitImpl (ostream &strm, unsigned ccount, unsigned pcount)
+ void CodeGen::startExplicitImpl (ostream &strm, unsigned ccount, unsigned pcount)
{
implTemplateDecl (strm, ccount + pcount + 1);
@@ -391,7 +395,7 @@
strm << ");\n }\n";
}
- void declareMemberFn (ostream &strm
+ void CodeGen::declareMemberFn (ostream &strm
, int id
, unsigned ccount
, unsigned pcount)
@@ -410,7 +414,7 @@
strm << ");\n";
}
- void defineMemberFn (ostream &strm
+ void CodeGen::defineMemberFn (ostream &strm
, int id
, unsigned ccount
, unsigned pcount
@@ -508,10 +512,11 @@
strm << "\n}\n";
}
- void implicitConstructor (ostream &strm, int id, const lambdaT &lam)
+ void CodeGen::implicitConstructor (ostream &strm, int id, const lambdaT &lam)
{
strm
- << "lambda_generator_"
+ << mPrefix
+ << "_lambda_generator_"
<< id
<< "<"
<< *lam.ret_type;
@@ -525,7 +530,7 @@
strm <<")";
}
- void implicitTemplateInstance (ostream &strm
+ void CodeGen::implicitTemplateInstance (ostream &strm
, const lambdaT &lam
, unsigned ccount
, unsigned pcount)
@@ -556,7 +561,7 @@
strm << ">";
}
- void writeGenerator (ostream &strm, int id, const lambdaT &lam)
+ void CodeGen::writeGenerator (ostream &strm, int id, const lambdaT &lam)
{
unsigned ccount = count (lam.context.get());
unsigned pcount = count (lam.params.get());
@@ -567,7 +572,7 @@
typenameList (strm, 2 + ccount, 1 + ccount + pcount, false);
strm
- << ">\nstruct lambda_generator_" << id
+ << ">\nstruct " << mPrefix << "_lambda_generator_" << id
<< "\n{";
if (ccount >= 1)
@@ -851,7 +856,7 @@
; iter != list.end()
; ++iter)
{
- defineMemberFn (mImplStream
+ defineMemberFn (mGenrStream
, iter->mLambdaNumber
, group->first.first
, group->first.second
Modified: clamp/trunk/src/CodeGen.hh
===================================================================
--- clamp/trunk/src/CodeGen.hh 2009-03-19 18:02:16 UTC (rev 1310)
+++ clamp/trunk/src/CodeGen.hh 2009-03-19 22:19:57 UTC (rev 1311)
@@ -32,13 +32,16 @@
#include <map>
#include <sstream>
#include "boost/smart_ptr.hpp"
+#include <string>
namespace clamp
{
+ using namespace std;
+
class CodeGen
{
public:
- CodeGen ();
+ CodeGen (const std::string &prefix);
public:
void lambdaType (FILE *, std::auto_ptr<lambdaT>);
@@ -56,6 +59,30 @@
void spitItOut ();
private:
+ void implTemplateName (ostream &strm, int ccount, int pcount);
+ void tList (ostream &strm, int start, int end, bool first = true);
+ void implFunctionVar (ostream &strm, int ccount, int pcount, const char *n);
+ void implInstance (ostream &strm, lambdaT const &lam);
+ stringT bodyFileName (int count);
+ void explicitConstructor (ostream &strm, int id, const lambdaT &lam);
+ void startExplicitImpl (ostream &strm, unsigned ccount, unsigned pcount);
+ void defineMemberFn (ostream &strm
+ , int id
+ , unsigned ccount
+ , unsigned pcount
+ , const lambdaT &lam);
+ void declareMemberFn (ostream &strm
+ , int id
+ , unsigned ccount
+ , unsigned pcount);
+ void implicitConstructor (ostream &strm, int id, const lambdaT &lam);
+ void implicitTemplateInstance (ostream &strm
+ , const lambdaT &lam
+ , unsigned ccount
+ , unsigned pcount);
+ void writeGenerator (ostream &strm, int id, const lambdaT &lam);
+
+ private:
struct LambdaContext {
FILE *mPrevFile;
int mOpenBracePos;
@@ -75,6 +102,7 @@
private:
std::ofstream mImplStream;
+ std::ofstream mGenrStream;
private:
//
@@ -104,7 +132,12 @@
LambdaMap::iterator lambdaGroup (int ccount, int pcount);
// Get the lambda group for the given number of parameters (starting
// a new group if necessary)
+
+ private:
+ std::string mPrefix;
+
};
+
}
#endif // CodeGen_rmg_20020211_included
Modified: clamp/trunk/src/Makefile
===================================================================
--- clamp/trunk/src/Makefile 2009-03-19 18:02:16 UTC (rev 1310)
+++ clamp/trunk/src/Makefile 2009-03-19 22:19:57 UTC (rev 1311)
@@ -35,7 +35,8 @@
LFLAGS =
YFLAGS = -v -d
-LDLIBS = -lstdc++
+LDLIBS = -lboost_program_options-gcc43-mt
+LD = $(CXX)
clamp.tab.h: clamp.cc
mv y.tab.h clamp.tab.h
@@ -61,10 +62,10 @@
clamp: LDLIBS := $(LDLIBS) -lfl
clamp: clamp.o clamplex.o clamp_support.o CodeGen.o
-clean:
- -rm -f *.o clamp *.exe clamp.cc y.output clamp.tab.h lex.yy.cc lambda*.clamp_h test.cc
+clean:
+ -rm -f *.o clamp *.exe clamp.cc y.output clamp.tab.h lex.yy.cc *_lambda_*.clamp_h test.cc curry test eg curry.cc test.cc eg.cc
.PRECIOUS: %.cc
%.cc: %.clamp clamp
- ./clamp <$< >$@
+ ./clamp --prefix $(basename $<) <$< >$@
Modified: clamp/trunk/src/clamp.y
===================================================================
--- clamp/trunk/src/clamp.y 2009-03-19 18:02:16 UTC (rev 1310)
+++ clamp/trunk/src/clamp.y 2009-03-19 22:19:57 UTC (rev 1311)
@@ -27,12 +27,15 @@
#include "clamplex.h"
#include "CodeGen.hh"
+#include <string>
#include <iostream>
-#include <stdio.h>
+#include <cstdio>
#include <malloc.h>
#include <set>
+#include <boost/program_options.hpp>
using namespace clamp;
+ using namespace std;
CodeGen *gGenPtr = 0;
@@ -79,7 +82,7 @@
;
lambda: lambda_type {
- gGenPtr->lambdaType (yyout, std::auto_ptr<lambdaT>($1));
+ gGenPtr->lambdaType (yyout, auto_ptr<lambdaT>($1));
// We read a lookahead token in lambda mode, and it wasn't
// a left brace - put the char back and let the lexer reprocess
@@ -99,7 +102,7 @@
lambda_def_start: lambda_type '{' {
resume_initial = 1;
- if (!gGenPtr->startLambda (yyout, openbraces, std::auto_ptr<lambdaT>($1)))
+ if (!gGenPtr->startLambda (yyout, openbraces, auto_ptr<lambdaT>($1)))
{
YYABORT;
}
@@ -152,7 +155,7 @@
| cvqualopt genericid cvqualopt { $$ = appendstr ($1, $2, $3); }
| typename genericid { $$ = appendstr ($1, $2); }
| lambda_type {
- $$ = gGenPtr->lambdaTypeName (std::auto_ptr<lambdaT>($1)).release();
+ $$ = gGenPtr->lambdaTypeName (auto_ptr<lambdaT>($1)).release();
}
;
@@ -192,7 +195,7 @@
;
context: ctxorref '(' genericexpr ')' {
- if (!gGenPtr->contextRef (yyout, std::auto_ptr<stringT> ($3), $1))
+ if (!gGenPtr->contextRef (yyout, auto_ptr<stringT> ($3), $1))
{
YYABORT;
}
@@ -231,13 +234,13 @@
BlockInfo *prev;
BlockInfo *next;
void *caller;
- std::size_t size;
+ size_t size;
};
struct Stats
{
- std::size_t blocks;
- std::size_t bytes;
+ size_t blocks;
+ size_t bytes;
};
Stats gAlloc = {0};
@@ -247,7 +250,7 @@
}
#ifdef TRACK_MEMORY
-void *operator new (std::size_t size) throw (std::bad_alloc)
+void *operator new (size_t size) throw (bad_alloc)
{
++gAlloc.blocks;
gAlloc.bytes += size;
@@ -299,7 +302,7 @@
}
}
-void *operator new[] (std::size_t size) throw (std::bad_alloc)
+void *operator new[] (size_t size) throw (bad_alloc)
{
return operator new (size);
}
@@ -348,6 +351,39 @@
{
int result;
+ string prefix;
+
+ namespace po = boost::program_options;
+ po::options_description desc("Allowed options");
+ desc.add_options()
+ ("help,h", "show this help message")
+ ("debug,d", "enable yydebug")
+#if 0
+ ("exit-on-recovery,x", po::bool_switch(&stop_on_recovery),
+ "exit after the joiner fully recovers (for leader)")
+ ("batch-size,b", po::value<int>(&batch_size)->default_value(100),
+ "number of txns to batch up in each msg (for leader)")
+#endif
+ ("prefix,p", po::value<string>(&prefix)->default_value(""),
+ "hostname or address of the leader");
+
+ po::variables_map vm;
+ try {
+ po::store(po::parse_command_line(argc, argv, desc), vm);
+ po::notify(vm);
+
+ yydebug = vm.count("debug");
+
+ if (vm.count("help")) {
+ cout << desc << endl;
+ return 0;
+ }
+ } catch (std::exception &ex) {
+ cerr << ex.what() << "\n\n" << desc << endl;
+ return 1;
+ }
+
+#if 0
if (argc > 1)
{
yyin = fopen (argv[1], "r");
@@ -362,15 +398,16 @@
++argv;
--argc;
}
+#endif
- yydebug = argc - 1;
+ cout << "#include \"" << prefix << "_lambda_impl.clamp_h\"\n";
- printf ("#include \"lambda_impl.clamp_h\"\n");
+ gGenPtr = new CodeGen(prefix);
- gGenPtr = new CodeGen;
-
result = yyparse();
+ cout << "#include \"" << prefix << "_lambda_genr.clamp_h\"\n";
+
if (result == 0)
{
gGenPtr->spitItOut ();
@@ -385,8 +422,6 @@
if (gAlloc.blocks != gDealloc.blocks)
{
- using namespace std;
-
fprintf (stderr
, "Memory leak detected. Dumping still allocated blocks:\n");
@@ -396,7 +431,7 @@
while (scan)
{
- std::cerr
+ cerr
<< "Address " << static_cast<void *>(scan)
<< " size " << scan->size
<< " caller " << scan->caller
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <yan...@us...> - 2009-03-19 18:02:29
|
Revision: 1310
http://assorted.svn.sourceforge.net/assorted/?rev=1310&view=rev
Author: yangzhang
Date: 2009-03-19 18:02:16 +0000 (Thu, 19 Mar 2009)
Log Message:
-----------
reformatted; tweaked Makefile
Modified Paths:
--------------
clamp/trunk/src/CodeGen.cc
clamp/trunk/src/CodeGen.hh
clamp/trunk/src/Makefile
clamp/trunk/src/clamp.y
clamp/trunk/src/clamp_support.cc
clamp/trunk/src/clamp_support.hh
clamp/trunk/src/clamplex.lex
clamp/trunk/src/curry.clamp
clamp/trunk/src/eg.clamp
clamp/trunk/src/home_page_examples.clamp
clamp/trunk/src/test.clamp
Modified: clamp/trunk/src/CodeGen.cc
===================================================================
--- clamp/trunk/src/CodeGen.cc 2009-03-19 10:18:05 UTC (rev 1309)
+++ clamp/trunk/src/CodeGen.cc 2009-03-19 18:02:16 UTC (rev 1310)
@@ -15,7 +15,7 @@
//
// History
// =======
-// 2002/ 2/11 rmg File creation
+// 2002/ 2/11 rmg File creation
//
#include "CodeGen.hh"
@@ -38,12 +38,12 @@
{
if (pListp)
{
- return pListp->size();
+ return pListp->size();
}
else
{
- return 0;
+ return 0;
}
}
@@ -51,35 +51,35 @@
{
if (pListp)
{
- for (paramListT::const_iterator iter = pListp->begin()
- ; iter != pListp->end()
- ; ++iter)
- {
- if (iter->get()->initialiser.get())
- {
- if (!first)
- {
- strm << ", ";
- }
+ for (paramListT::const_iterator iter = pListp->begin()
+ ; iter != pListp->end()
+ ; ++iter)
+ {
+ if (iter->get()->initialiser.get())
+ {
+ if (!first)
+ {
+ strm << ", ";
+ }
- strm << *iter->get()->initialiser;
+ strm << *iter->get()->initialiser;
- first = false;
- }
+ first = false;
+ }
- else if (iter->get()->type.get())
- {
- if (!first)
- {
- strm << ", ";
- }
+ else if (iter->get()->type.get())
+ {
+ if (!first)
+ {
+ strm << ", ";
+ }
// Default initialize
- strm << "(" << *iter->get()->type << ")0";
+ strm << "(" << *iter->get()->type << ")0";
- first = false;
- }
- }
+ first = false;
+ }
+ }
}
}
@@ -87,22 +87,22 @@
{
if (pListp)
{
- for (paramListT::const_iterator iter = pListp->begin()
- ; iter != pListp->end()
- ; ++iter)
- {
- if (iter->get()->type.get())
- {
- if (!first)
- {
- strm << ", ";
- }
+ for (paramListT::const_iterator iter = pListp->begin()
+ ; iter != pListp->end()
+ ; ++iter)
+ {
+ if (iter->get()->type.get())
+ {
+ if (!first)
+ {
+ strm << ", ";
+ }
- strm << *iter->get()->type;
+ strm << *iter->get()->type;
- first = false;
- }
- }
+ first = false;
+ }
+ }
}
}
}
@@ -123,8 +123,8 @@
}
CodeGen::LambdaContext::LambdaContext (FILE *f
- , int b
- , std::auto_ptr<lambdaT> lamp)
+ , int b
+ , std::auto_ptr<lambdaT> lamp)
: mPrevFile (f)
, mOpenBracePos (b)
, mLamp (lamp.release())
@@ -151,14 +151,14 @@
{
for (int count = start; count <= end; ++count)
{
- if (!first)
- {
- strm << ", ";
- }
+ if (!first)
+ {
+ strm << ", ";
+ }
- strm << "T" << count;
+ strm << "T" << count;
- first = false;
+ first = false;
}
}
@@ -168,9 +168,9 @@
if (ccount > 0)
{
- // Implementation via pointer to member function
- implTemplateName (strm, ccount, pcount);
- strm << "::";
+ // Implementation via pointer to member function
+ implTemplateName (strm, ccount, pcount);
+ strm << "::";
}
// else implementation via pointer to ordinary function
@@ -240,7 +240,7 @@
{
if (lam.onHeap)
{
- strm << "new ";
+ strm << "new ";
}
implInstance (strm, lam);
@@ -252,20 +252,20 @@
}
void typenameList (ostream &strm
- , unsigned start
- , unsigned end
- , bool first = true)
+ , unsigned start
+ , unsigned end
+ , bool first = true)
{
for (unsigned count = start; count <= end; ++count)
{
- if (!first)
- {
- strm << ", ";
- }
+ if (!first)
+ {
+ strm << ", ";
+ }
- strm << "typename T" << count;
+ strm << "typename T" << count;
- first = false;
+ first = false;
}
}
@@ -295,41 +295,41 @@
for (unsigned count = 1; count <= ccount; ++count)
{
- strm << "\n T" << count + 1 << " mContext" << count << ";";
+ strm << "\n T" << count + 1 << " mContext" << count << ";";
}
strm << "\n\n typedef T1 result_type;";
for (unsigned count = 1; count <= ccount; ++count)
{
- strm
- << "\n typedef T" << count + 1
- << " context_type_" << count
- << ";";
+ strm
+ << "\n typedef T" << count + 1
+ << " context_type_" << count
+ << ";";
}
for (unsigned count = 1; count <= pcount; ++count)
{
- strm
- << "\n typedef T" << count + ccount + 1
- << " argument_type_" << count
- << ";";
+ strm
+ << "\n typedef T" << count + ccount + 1
+ << " argument_type_" << count
+ << ";";
}
// Additional typedefs in special cases for standard binder compatability
if (pcount == 1)
{
- strm
- << "\n typedef T" << ccount + 2
- << " argument_type;";
+ strm
+ << "\n typedef T" << ccount + 2
+ << " argument_type;";
}
else if (pcount == 2)
{
- strm
- << "\n typedef T" << ccount + 2
- << " first_argument_type;\n typedef T" << ccount + 3
- << " second_argument_type;";
+ strm
+ << "\n typedef T" << ccount + 2
+ << " first_argument_type;\n typedef T" << ccount + 3
+ << " second_argument_type;";
}
strm << "\n\n // Constructor\n ";
@@ -342,14 +342,14 @@
for (unsigned count = 1; count <= ccount; ++count)
{
- strm << ", T" << count + 1 << " c" << count;
+ strm << ", T" << count + 1 << " c" << count;
}
strm << ")\n : mFn (fn)";
for (unsigned count = 1; count <= ccount; ++count)
{
- strm << "\n , mContext" << count << " (c" << count << ")";
+ strm << "\n , mContext" << count << " (c" << count << ")";
}
strm
@@ -359,20 +359,20 @@
for (unsigned count = 1; count <= pcount; ++count)
{
- if (count != 1)
- {
- strm << ", ";
- }
+ if (count != 1)
+ {
+ strm << ", ";
+ }
- strm << "T" << count + ccount + 1 << " p" << count;
+ strm << "T" << count + ccount + 1 << " p" << count;
}
strm << ")\n {\n return (";
if (ccount > 0)
{
- // Have closure parameters - use pointer to member syntax
- strm << "this->";
+ // Have closure parameters - use pointer to member syntax
+ strm << "this->";
}
// else static function
@@ -380,27 +380,27 @@
for (unsigned count = 1; count <= pcount; ++count)
{
- if (count != 1)
- {
- strm << ", ";
- }
+ if (count != 1)
+ {
+ strm << ", ";
+ }
- strm << "p" << count;
+ strm << "p" << count;
}
strm << ");\n }\n";
}
void declareMemberFn (ostream &strm
- , int id
- , unsigned ccount
- , unsigned pcount)
+ , int id
+ , unsigned ccount
+ , unsigned pcount)
{
strm << "\n ";
if (ccount == 0)
{
- strm << "static ";
+ strm << "static ";
}
strm << "T1 fn" << id << " (";
@@ -411,10 +411,10 @@
}
void defineMemberFn (ostream &strm
- , int id
- , unsigned ccount
- , unsigned pcount
- , const lambdaT &lam)
+ , int id
+ , unsigned ccount
+ , unsigned pcount
+ , const lambdaT &lam)
{
implTemplateDecl (strm, ccount + pcount + 1);
@@ -430,54 +430,54 @@
if (lam.params.get())
{
- unsigned count = 1;
+ unsigned count = 1;
- for (paramListT::const_iterator iter = lam.params->begin()
- ; iter != lam.params->end()
- ; ++iter)
- {
- if (count != 1)
- {
- strm << ", ";
- }
+ for (paramListT::const_iterator iter = lam.params->begin()
+ ; iter != lam.params->end()
+ ; ++iter)
+ {
+ if (count != 1)
+ {
+ strm << ", ";
+ }
- strm << "T" << count + ccount + 1;
+ strm << "T" << count + ccount + 1;
- const stringT *name = iter->get()->name.get();
+ const stringT *name = iter->get()->name.get();
- if (name)
- {
- strm
- << " " << *name;
- }
+ if (name)
+ {
+ strm
+ << " " << *name;
+ }
- ++count;
- }
+ ++count;
+ }
}
strm << ")\n{";
if (lam.explicitContext && lam.context.get())
{
- // Context parameters require renaming
- unsigned count = 1;
+ // Context parameters require renaming
+ unsigned count = 1;
- for (paramListT::const_iterator iter = lam.context->begin()
- ; iter != lam.context->end()
- ; ++iter)
- {
- const stringT *name = iter->get()->name.get();
+ for (paramListT::const_iterator iter = lam.context->begin()
+ ; iter != lam.context->end()
+ ; ++iter)
+ {
+ const stringT *name = iter->get()->name.get();
- if (name)
- {
- strm
- << "\n#define " << *name
- << " mContext"
+ if (name)
+ {
+ strm
+ << "\n#define " << *name
+ << " mContext"
<< count;
- }
+ }
- ++count;
- }
+ ++count;
+ }
}
strm
@@ -487,22 +487,22 @@
if (lam.explicitContext && lam.context.get())
{
- unsigned count = 1;
+ unsigned count = 1;
- for (paramListT::const_iterator iter = lam.context->begin()
- ; iter != lam.context->end()
- ; ++iter)
- {
- const stringT *name = iter->get()->name.get();
+ for (paramListT::const_iterator iter = lam.context->begin()
+ ; iter != lam.context->end()
+ ; ++iter)
+ {
+ const stringT *name = iter->get()->name.get();
- if (name)
- {
- strm
- << "\n#undef " << *name;
- }
+ if (name)
+ {
+ strm
+ << "\n#undef " << *name;
+ }
- ++count;
- }
+ ++count;
+ }
}
strm << "\n}\n";
@@ -526,9 +526,9 @@
}
void implicitTemplateInstance (ostream &strm
- , const lambdaT &lam
- , unsigned ccount
- , unsigned pcount)
+ , const lambdaT &lam
+ , unsigned ccount
+ , unsigned pcount)
{
implTemplateName (strm, ccount, pcount);
@@ -536,19 +536,19 @@
if (ccount)
{
- paramListT::const_iterator iter = lam.context->begin();
+ paramListT::const_iterator iter = lam.context->begin();
- for (unsigned count = 1; count <= ccount; ++count)
- {
- strm << ", T" << count + 1;
+ for (unsigned count = 1; count <= ccount; ++count)
+ {
+ strm << ", T" << count + 1;
- if ((*iter)->byReference)
- {
- strm << " &";
- }
+ if ((*iter)->byReference)
+ {
+ strm << " &";
+ }
- ++iter;
- }
+ ++iter;
+ }
}
tList (strm, 2 + ccount, 1 + ccount + pcount, false);
@@ -572,11 +572,11 @@
if (ccount >= 1)
{
- strm << "\n template <";
+ strm << "\n template <";
- typenameList (strm, 2, 1 + ccount);
+ typenameList (strm, 2, 1 + ccount);
- strm << ">";
+ strm << ">";
}
strm << "\n static ";
@@ -585,23 +585,23 @@
if (lam.onHeap)
{
- strm << " *";
+ strm << " *";
}
strm << "\n generate (";
if (ccount)
{
- paramListT::const_iterator iter = lam.context->begin();
+ paramListT::const_iterator iter = lam.context->begin();
- for (unsigned count = 1; count <= ccount; ++count)
- {
- if (count != 1)
- {
- strm << ", ";
- }
+ for (unsigned count = 1; count <= ccount; ++count)
+ {
+ if (count != 1)
+ {
+ strm << ", ";
+ }
- strm << "T" << count + 1 << " ";
+ strm << "T" << count + 1 << " ";
if ((*iter)->byReference)
{
@@ -610,15 +610,15 @@
strm << "p" << count;
- ++iter;
- }
+ ++iter;
+ }
}
strm << ")\n {\n return ";
if (lam.onHeap)
{
- strm << "new ";
+ strm << "new ";
}
implicitTemplateInstance (strm, lam, ccount, pcount);
@@ -650,7 +650,7 @@
bool mByRef;
MatchInitialiser (const stringT &target
- , bool byRef)
+ , bool byRef)
: mTarget (target)
, mByRef (byRef)
{
@@ -663,8 +663,8 @@
};
bool CodeGen::contextRef (FILE *outFile
- , std::auto_ptr<stringT> strp
- , bool byRef)
+ , std::auto_ptr<stringT> strp
+ , bool byRef)
{
// Handles context by value and by reference
@@ -672,15 +672,15 @@
if (lam.explicitContext)
{
- cerr
- << "__ctx or __ref expression in lambda with explicit context\n";
+ cerr
+ << "__ctx or __ref expression in lambda with explicit context\n";
- return false;
+ return false;
}
if (!lam.context.get())
{
- lam.context.reset (new paramListT);
+ lam.context.reset (new paramListT);
}
paramListT &context (*lam.context);
@@ -688,32 +688,32 @@
paramListT::iterator param;
param = std::find_if (context.begin()
- , context.end()
- , MatchInitialiser (*strp, byRef));
+ , context.end()
+ , MatchInitialiser (*strp, byRef));
if (param == context.end())
{
- int ccount = context.size() + 1;
+ int ccount = context.size() + 1;
- char newName[128];
+ char newName[128];
- sprintf (newName, "mContext%d", ccount);
+ sprintf (newName, "mContext%d", ccount);
- char pseudoType[128];
+ char pseudoType[128];
- sprintf (pseudoType, "T%d", ccount);
+ sprintf (pseudoType, "T%d", ccount);
- boost::shared_ptr<paramT>
- newParamPtr (new paramT (makestr(pseudoType)
- , makestr (newName)
- , strp.release()
+ boost::shared_ptr<paramT>
+ newParamPtr (new paramT (makestr(pseudoType)
+ , makestr (newName)
+ , strp.release()
, ccount
- , byRef));
+ , byRef));
- context.push_back (newParamPtr);
+ context.push_back (newParamPtr);
- param = context.end();
- --param;
+ param = context.end();
+ --param;
}
fprintf (outFile, "%s", contextVarExpression (**param).c_str());
@@ -722,8 +722,8 @@
}
bool CodeGen::startLambda (FILE *&f
- , int openBracePos
- , std::auto_ptr<lambdaT> lamp)
+ , int openBracePos
+ , std::auto_ptr<lambdaT> lamp)
{
mStack.push (LambdaContext (f, openBracePos, lamp));
@@ -733,15 +733,15 @@
if (!f)
{
- cerr
- << "Failed to create new body file "
- << newName
- << "\n";
+ cerr
+ << "Failed to create new body file "
+ << newName
+ << "\n";
- f = mStack.top().mPrevFile;
- mStack.pop ();
+ f = mStack.top().mPrevFile;
+ mStack.pop ();
- return false;
+ return false;
}
return true;
@@ -755,14 +755,14 @@
if (mapIter == mLambdaMap.end())
{
- pair<LambdaMap::iterator, bool> inserted
- = mLambdaMap.insert (std::make_pair (typeId, LambdaGroup()));
+ pair<LambdaMap::iterator, bool> inserted
+ = mLambdaMap.insert (std::make_pair (typeId, LambdaGroup()));
- assert (inserted.second);
+ assert (inserted.second);
- mapIter = inserted.first;
+ mapIter = inserted.first;
- startExplicitImpl (*mapIter->second.mImplStream, ccount, pcount);
+ startExplicitImpl (*mapIter->second.mImplStream, ccount, pcount);
}
return mapIter;
@@ -784,21 +784,21 @@
mapIter->second.mLambdaList.push_back (ctx);
declareMemberFn (*mapIter->second.mImplStream
- , ctx.mLambdaNumber
- , ccount
- , pcount);
+ , ctx.mLambdaNumber
+ , ccount
+ , pcount);
std::ostringstream temp;
if (ctx.mLamp->explicitContext)
{
- explicitConstructor (temp, ctx.mLambdaNumber, *ctx.mLamp);
+ explicitConstructor (temp, ctx.mLambdaNumber, *ctx.mLamp);
}
else
{
- writeGenerator (mGeneratorStream, ctx.mLambdaNumber, *ctx.mLamp);
- implicitConstructor (temp, ctx.mLambdaNumber, *ctx.mLamp);
+ writeGenerator (mGeneratorStream, ctx.mLambdaNumber, *ctx.mLamp);
+ implicitConstructor (temp, ctx.mLambdaNumber, *ctx.mLamp);
}
fprintf (f, "%s", temp.str().c_str());
@@ -818,10 +818,10 @@
;
for (LambdaMap::const_iterator iter = mLambdaMap.begin()
- ; iter != mLambdaMap.end()
- ; ++iter)
+ ; iter != mLambdaMap.end()
+ ; ++iter)
{
- mImplStream
+ mImplStream
<< iter->second.mImplStream->rdbuf()
<< "\nprivate:\n ";
@@ -842,21 +842,21 @@
mImplStream << "\n" << mGeneratorStream.str() << "\n";
for (LambdaMap::const_iterator group = mLambdaMap.begin()
- ; group != mLambdaMap.end()
- ; ++group)
+ ; group != mLambdaMap.end()
+ ; ++group)
{
- const LambdaList &list (group->second.mLambdaList);
+ const LambdaList &list (group->second.mLambdaList);
- for (LambdaList::const_iterator iter = list.begin()
- ; iter != list.end()
- ; ++iter)
- {
- defineMemberFn (mImplStream
- , iter->mLambdaNumber
- , group->first.first
- , group->first.second
- , *iter->mLamp);
- }
+ for (LambdaList::const_iterator iter = list.begin()
+ ; iter != list.end()
+ ; ++iter)
+ {
+ defineMemberFn (mImplStream
+ , iter->mLambdaNumber
+ , group->first.first
+ , group->first.second
+ , *iter->mLamp);
+ }
}
}
}
Modified: clamp/trunk/src/CodeGen.hh
===================================================================
--- clamp/trunk/src/CodeGen.hh 2009-03-19 10:18:05 UTC (rev 1309)
+++ clamp/trunk/src/CodeGen.hh 2009-03-19 18:02:16 UTC (rev 1310)
@@ -15,7 +15,7 @@
//
// History
// =======
-// 2002/ 2/11 rmg File creation
+// 2002/ 2/11 rmg File creation
//
// version: $Id: CodeGen.hh,v 1.9 2003/09/30 22:42:56 Raoul Exp $
//
Modified: clamp/trunk/src/Makefile
===================================================================
--- clamp/trunk/src/Makefile 2009-03-19 10:18:05 UTC (rev 1309)
+++ clamp/trunk/src/Makefile 2009-03-19 18:02:16 UTC (rev 1310)
@@ -62,7 +62,7 @@
clamp: clamp.o clamplex.o clamp_support.o CodeGen.o
clean:
- -rm *.o *.exe clamp.cc y.output clamp.tab.h lex.yy.cc lambda*.clamp_h
+ -rm -f *.o clamp *.exe clamp.cc y.output clamp.tab.h lex.yy.cc lambda*.clamp_h test.cc
.PRECIOUS: %.cc
Modified: clamp/trunk/src/clamp.y
===================================================================
--- clamp/trunk/src/clamp.y 2009-03-19 10:18:05 UTC (rev 1309)
+++ clamp/trunk/src/clamp.y 2009-03-19 18:02:16 UTC (rev 1310)
@@ -15,7 +15,7 @@
*
* History
* =======
- * 2002/ 2/ 7 rmg File creation
+ * 2002/ 2/ 7 rmg File creation
*
* $Id: clamp.y,v 1.31 2003/09/30 22:06:25 Raoul Exp $
*/
@@ -110,11 +110,11 @@
| lambda_start '(' decls ')' {
if (plistInitialisers ($3))
{
- yywarning ("default parameters ignored");
+ yywarning ("default parameters ignored");
}
- $$ = shiftlambda ($1, $3); // Adjust to explicit-context lambda
- }
+ $$ = shiftlambda ($1, $3); // Adjust to explicit-context lambda
+ }
;
lambda_start: TOK_LAMBDA typeidopt '(' decls ')' {
@@ -281,19 +281,19 @@
gDealloc.bytes += myPtr->size;
if (myPtr->next)
- {
- myPtr->next->prev = myPtr->prev;
- }
+ {
+ myPtr->next->prev = myPtr->prev;
+ }
if (myPtr->prev)
- {
- myPtr->prev->next = myPtr->next;
- }
+ {
+ myPtr->prev->next = myPtr->next;
+ }
else
- {
- first = myPtr->next;
- }
+ {
+ first = myPtr->next;
+ }
free (myPtr);
}
@@ -317,10 +317,10 @@
fflush (stdout);
fprintf (stderr
- , "\nclamp: %s on line %d with token %s\n"
- , s
- , yylineno
- , yytext);
+ , "\nclamp: %s on line %d with token %s\n"
+ , s
+ , yylineno
+ , yytext);
return 0;
}
@@ -331,9 +331,9 @@
fflush (stdout);
fprintf (stderr
- , "clamp: %s on line %d\n"
- , s
- , yylineno);
+ , "clamp: %s on line %d\n"
+ , s
+ , yylineno);
return 0;
}
@@ -353,11 +353,11 @@
yyin = fopen (argv[1], "r");
if (!yyin)
- {
- perror ("fopen");
+ {
+ perror ("fopen");
- return 1;
- }
+ return 1;
+ }
++argv;
--argc;
@@ -388,22 +388,22 @@
using namespace std;
fprintf (stderr
- , "Memory leak detected. Dumping still allocated blocks:\n");
+ , "Memory leak detected. Dumping still allocated blocks:\n");
fflush (stderr);
BlockInfo *scan = first;
while (scan)
- {
- std::cerr
+ {
+ std::cerr
<< "Address " << static_cast<void *>(scan)
<< " size " << scan->size
<< " caller " << scan->caller
- << "\n";
+ << "\n";
- scan = scan->next;
- }
+ scan = scan->next;
+ }
}
return result;
Modified: clamp/trunk/src/clamp_support.cc
===================================================================
--- clamp/trunk/src/clamp_support.cc 2009-03-19 10:18:05 UTC (rev 1309)
+++ clamp/trunk/src/clamp_support.cc 2009-03-19 18:02:16 UTC (rev 1310)
@@ -16,7 +16,7 @@
//
// History
// =======
-// 2002/ 2/10 rmg File creation
+// 2002/ 2/10 rmg File creation
//
// version: $Id: clamp_support.cc,v 1.11 2002/09/16 17:09:45 Raoul Exp $
//
@@ -43,19 +43,19 @@
{
if (from || force)
{
- to.reset (from);
+ to.reset (from);
}
}
lambdaT *makelambda (lambdaT *lamp
- , stringT *typestr
- , paramListT *ctx
- , paramListT *params
- , bool onh)
+ , stringT *typestr
+ , paramListT *ctx
+ , paramListT *params
+ , bool onh)
{
if (lamp == 0)
{
- lamp = new lambdaT;
+ lamp = new lambdaT;
}
safeAssign (lamp->ret_type, typestr);
@@ -67,7 +67,7 @@
}
lambdaT *shiftlambda (lambdaT *lamp
- , paramListT *params)
+ , paramListT *params)
{
// Release should be safe because it is only called if lamp is
// already non-null, in which case makelambda guarantees no-throw
@@ -83,16 +83,16 @@
{
if (front)
{
- if (back)
- {
- front->splice (front->end(), *back);
- delete back;
- }
+ if (back)
+ {
+ front->splice (front->end(), *back);
+ delete back;
+ }
}
else if (back)
{
- front = back;
+ front = back;
}
return front;
@@ -113,12 +113,12 @@
if (list)
{
- for (paramListT::const_iterator iter = list->begin()
- ; (iter != list->end()) && (!foundInit)
- ; ++iter)
- {
- foundInit = (*iter)->initialiser.get();
- }
+ for (paramListT::const_iterator iter = list->begin()
+ ; (iter != list->end()) && (!foundInit)
+ ; ++iter)
+ {
+ foundInit = (*iter)->initialiser.get();
+ }
}
return foundInit;
@@ -128,18 +128,18 @@
{
if (s1)
{
- if (s2)
- {
- s1->append (" ");
- s1->append (*s2);
+ if (s2)
+ {
+ s1->append (" ");
+ s1->append (*s2);
- delete s2;
- }
+ delete s2;
+ }
}
else if (s2)
{
- s1 = s2;
+ s1 = s2;
}
return s1;
@@ -156,10 +156,10 @@
}
stringT *appendstr (stringT *s1
- , stringT *s2
- , stringT *s3
- , stringT *s4
- , stringT *s5)
+ , stringT *s2
+ , stringT *s3
+ , stringT *s4
+ , stringT *s5)
{
return appendstr (appendstr (s1, s2, s3, s4), s5);
}
@@ -168,12 +168,12 @@
{
if (cstr)
{
- return new stringT (cstr);
+ return new stringT (cstr);
}
else
{
- return 0;
+ return 0;
}
}
@@ -181,12 +181,12 @@
{
if (cstr)
{
- return new stringT (cstr, length);
+ return new stringT (cstr, length);
}
else
{
- return 0;
+ return 0;
}
}
@@ -199,10 +199,10 @@
if (lam.explicitContext)
{
- strm
- << " context "
- << lam.context.get()
- << " and";
+ strm
+ << " context "
+ << lam.context.get()
+ << " and";
}
strm
@@ -225,15 +225,15 @@
std::ostream &operator<< (std::ostream &strm, const paramListT &pList)
{
for (paramListT::const_iterator iter = pList.begin()
- ; iter != pList.end()
- ; ++iter)
+ ; iter != pList.end()
+ ; ++iter)
{
- if (iter != pList.begin())
- {
- strm << ", ";
- }
+ if (iter != pList.begin())
+ {
+ strm << ", ";
+ }
- strm << iter->get();
+ strm << iter->get();
}
return strm;
Modified: clamp/trunk/src/clamp_support.hh
===================================================================
--- clamp/trunk/src/clamp_support.hh 2009-03-19 10:18:05 UTC (rev 1309)
+++ clamp/trunk/src/clamp_support.hh 2009-03-19 18:02:16 UTC (rev 1310)
@@ -15,7 +15,7 @@
//
// History
// =======
-// 2002/ 2/ 7 rmg File creation
+// 2002/ 2/ 7 rmg File creation
//
// version: $Id: clamp_support.hh,v 1.14 2003/09/30 22:06:22 Raoul Exp $
//
@@ -65,10 +65,10 @@
};
lambdaT *makelambda (lambdaT *lamp
- , stringT *typestr
- , paramListT *ctx
- , paramListT *params
- , bool onHeap = false);
+ , stringT *typestr
+ , paramListT *ctx
+ , paramListT *params
+ , bool onHeap = false);
//
// Convert an implicit-context lambda into an explicit-context version
@@ -76,7 +76,7 @@
// and adding the given parameters
//
lambdaT *shiftlambda (lambdaT *lamp
- , paramListT *params);
+ , paramListT *params);
paramListT *makeplist (stringT *type, stringT *name, stringT *init);
@@ -99,12 +99,12 @@
{
if (tptr)
{
- return strm << *tptr;
+ return strm << *tptr;
}
else
{
- return strm << "(null)";
+ return strm << "(null)";
}
}
*/
Modified: clamp/trunk/src/clamplex.lex
===================================================================
--- clamp/trunk/src/clamplex.lex 2009-03-19 10:18:05 UTC (rev 1309)
+++ clamp/trunk/src/clamplex.lex 2009-03-19 18:02:16 UTC (rev 1310)
@@ -15,7 +15,7 @@
*
* History
* =======
- * 2002/ 2/ 6 rmg File creation
+ * 2002/ 2/ 6 rmg File creation
*
* version: $Id: clamplex.lex,v 1.13 2002/11/18 17:04:00 Raoul Exp $
*/
@@ -111,12 +111,12 @@
{RBRACE} {
if (parserBraceMatch (openbraces--))
{
- return '}';
+ return '}';
}
else
{
- ECHO;
+ ECHO;
}
}
}
Modified: clamp/trunk/src/curry.clamp
===================================================================
--- clamp/trunk/src/curry.clamp 2009-03-19 10:18:05 UTC (rev 1309)
+++ clamp/trunk/src/curry.clamp 2009-03-19 18:02:16 UTC (rev 1310)
@@ -15,7 +15,7 @@
//
// History
// =======
-// 2002/11/14 rmg File creation
+// 2002/11/14 rmg File creation
//
int foo (int p1, int p2) { return p1 * p2; }
Modified: clamp/trunk/src/eg.clamp
===================================================================
--- clamp/trunk/src/eg.clamp 2009-03-19 10:18:05 UTC (rev 1309)
+++ clamp/trunk/src/eg.clamp 2009-03-19 18:02:16 UTC (rev 1310)
@@ -15,7 +15,7 @@
//
// History
// =======
-// 2002/ 2/12 rmg File creation
+// 2002/ 2/12 rmg File creation
//
// version: $Id: eg.clamp,v 1.13 2003/09/30 21:58:28 Raoul Exp $
//
Modified: clamp/trunk/src/home_page_examples.clamp
===================================================================
--- clamp/trunk/src/home_page_examples.clamp 2009-03-19 10:18:05 UTC (rev 1309)
+++ clamp/trunk/src/home_page_examples.clamp 2009-03-19 18:02:16 UTC (rev 1310)
@@ -6,7 +6,7 @@
//
// History
// =======
-// 2002/ 2/15 rmg File creation
+// 2002/ 2/15 rmg File creation
//
#include <vector>
Modified: clamp/trunk/src/test.clamp
===================================================================
--- clamp/trunk/src/test.clamp 2009-03-19 10:18:05 UTC (rev 1309)
+++ clamp/trunk/src/test.clamp 2009-03-19 18:02:16 UTC (rev 1310)
@@ -15,7 +15,7 @@
//
// History
// =======
-// 2002/ 2/12 rmg File creation
+// 2002/ 2/12 rmg File creation
//
// version: $Id: test.clamp,v 1.12 2002/11/18 17:03:27 Raoul Exp $
//
@@ -45,11 +45,11 @@
FN apply_if (ITER begin, ITER end, COND cond, FN fn)
{
for_each (begin
- , end
- , lambda (typename iterator_traits<ITER>::value_type &v) {
+ , end
+ , lambda (typename iterator_traits<ITER>::value_type &v) {
if (__ref(cond)(v))
{
- __ref(fn) (v);
+ __ref(fn) (v);
}
});
@@ -70,9 +70,9 @@
//
for_each (vs.begin(), vs.end(), lambda (string &str) {
apply_if (str.begin()
- , str.end()
- , match ('m')
- , lambda (char &c) { --c; });
+ , str.end()
+ , match ('m')
+ , lambda (char &c) { --c; });
});
//
@@ -81,8 +81,8 @@
string temp;
for_each (vs.begin()
- , vs.end()
- , lambda (const string &str) { __ref(temp) += str; } );
+ , vs.end()
+ , lambda (const string &str) { __ref(temp) += str; } );
cout << temp;
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <yan...@us...> - 2009-03-19 10:18:20
|
Revision: 1309
http://assorted.svn.sourceforge.net/assorted/?rev=1309&view=rev
Author: yangzhang
Date: 2009-03-19 10:18:05 +0000 (Thu, 19 Mar 2009)
Log Message:
-----------
- added friendlier versions of st_read, st_write
- added array ctor
- made stream_writer::reserve return the current pointer
Modified Paths:
--------------
cpp-commons/trunk/src/commons/array.h
cpp-commons/trunk/src/commons/st/st.h
cpp-commons/trunk/src/commons/streamwriter.h
Modified: cpp-commons/trunk/src/commons/array.h
===================================================================
--- cpp-commons/trunk/src/commons/array.h 2009-03-19 10:16:33 UTC (rev 1308)
+++ cpp-commons/trunk/src/commons/array.h 2009-03-19 10:18:05 UTC (rev 1309)
@@ -52,6 +52,7 @@
friend void swap<>(array<T> &a, array<T> &b);
public:
explicit array(size_t n) : p_(new T[n]), n_(n) {}
+ explicit array(T *p, size_t n) : p_(p), n_(n) {}
array() : p_(), n_(0) {}
array(array<T> &&src) : p_(src.release()), n_(src.n_) {}
#if 0
Modified: cpp-commons/trunk/src/commons/st/st.h
===================================================================
--- cpp-commons/trunk/src/commons/st/st.h 2009-03-19 10:16:33 UTC (rev 1308)
+++ cpp-commons/trunk/src/commons/st/st.h 2009-03-19 10:18:05 UTC (rev 1309)
@@ -50,6 +50,33 @@
st_mutex_t mx_;
};
+ void
+ st_read(st_netfd_t fd, void *buf, size_t len)
+ {
+ checkeqnneg(st_read_fully(fd, buf, len, ST_UTIME_NO_TIMEOUT), ssize_t(len));
+ }
+
+ template<typename T>
+ void
+ st_read(st_netfd_t fd, T &x)
+ {
+ st_read(fd, &x, sizeof x);
+ }
+
+ void
+ st_write(st_netfd_t fd, const void *buf, size_t len)
+ {
+ checkeqnneg(st_write(fd, buf, len, ST_UTIME_NO_TIMEOUT), ssize_t(len));
+ }
+
+ template<typename T>
+ void
+ st_write(st_netfd_t fd, const T &x)
+ {
+ st_write(fd, &x, sizeof x);
+ }
+
+
/**
* Run a function in pthread.
* \return The new pthread_t on success, 0 on failure.
Modified: cpp-commons/trunk/src/commons/streamwriter.h
===================================================================
--- cpp-commons/trunk/src/commons/streamwriter.h 2009-03-19 10:16:33 UTC (rev 1308)
+++ cpp-commons/trunk/src/commons/streamwriter.h 2009-03-19 10:18:05 UTC (rev 1309)
@@ -60,7 +60,7 @@
}
}
void reset() { p_ = mark_; }
- void reserve(size_t n) { reserve(n, p_); }
+ char *reserve(size_t n) { reserve(n, p_); return p_; }
void mark_and_flush() {
mark();
flush();
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <yan...@us...> - 2009-03-19 10:16:38
|
Revision: 1308
http://assorted.svn.sourceforge.net/assorted/?rev=1308&view=rev
Author: yangzhang
Date: 2009-03-19 10:16:33 +0000 (Thu, 19 Mar 2009)
Log Message:
-----------
- main:
- skip read-only txns when process_tpcc with no response
- added twal to process_tpccs
- added twal recovery with (broken) network recovery
- analysis:
- tolerate missing aries-log in scaling()
- fixed and simplified rec
- updates notes
Modified Paths:
--------------
ydb/trunk/README
ydb/trunk/src/main.lzz.clamp
ydb/trunk/src/tpcc/tpccclient.cc
ydb/trunk/src/tpcc/tpcctables.cc.cog
ydb/trunk/src/tpcc/tpcctables.h
ydb/trunk/tools/analysis.py
Modified: ydb/trunk/README
===================================================================
--- ydb/trunk/README 2009-03-18 20:08:05 UTC (rev 1307)
+++ ydb/trunk/README 2009-03-19 10:16:33 UTC (rev 1308)
@@ -752,6 +752,17 @@
- DONE prelim measurements on cluster
- TODO skip processing readonly txns on catch-up
+- some high level issues in this project
+ - meeting each week with random new todo's
+ - many of the todo's - like improving the baseline performance - are pointless
+ - no clear objective/end goal
+ - no clear idea of what we're doing
+ - what's the high-level model of usage? e.g. are we considering disk-based
+ no-network/no-replica methods?
+ - is there really a vldb-worthy project here?
+ - if not, why did we set down this road a year ago?
+ - cut our losses?
+
- TODO PAPER!!!
- TODO related work
Modified: ydb/trunk/src/main.lzz.clamp
===================================================================
--- ydb/trunk/src/main.lzz.clamp 2009-03-18 20:08:05 UTC (rev 1307)
+++ ydb/trunk/src/main.lzz.clamp 2009-03-19 10:16:33 UTC (rev 1308)
@@ -22,6 +22,7 @@
#include <cstring> // strsignal
#include <fstream> // ofstream
#include <google/dense_hash_map>
+#include <google/protobuf/io/zero_copy_stream_impl.h>
#include <gtest/gtest.h>
#include <inttypes.h> // PRId64
#include <iostream>
@@ -32,7 +33,7 @@
#include <sys/socket.h> // getpeername
#include <sys/types.h> // ssize_t
#include <tr1/unordered_map>
-#include <unistd.h> // pipe, write
+#include <unistd.h> // pipe, write, sync
#include <vector>
#include "ser.h"
#include "tpcc/clock.h"
@@ -53,6 +54,7 @@
using namespace boost::archive;
using namespace commons;
using namespace google;
+using namespace google::protobuf::io;
using namespace std;
using namespace std::tr1;
using namespace testing;
@@ -86,20 +88,24 @@
// Configuration.
st_utime_t timeout;
int yield_interval, accept_joiner_seqno, issuing_interval, min_ops, max_ops,
- stop_on_seqno, batch_size, handle_responses_display,
+ stop_on_seqno, batch_size, handle_responses_display, fail_seqno,
catch_up_display, issue_display, nwarehouses,
process_display;
size_t accept_joiner_size, buf_size, read_buf_size;
bool yield_during_build_up, yield_during_catch_up, dump, show_updates,
count_updates, stop_on_recovery, general_txns, profile_threads,
debug_threads, multirecover, disk, debug_memory, use_pwal, use_twal,
- use_pb, use_pb_res, g_caught_up, rec_pwal, do_tpcc,
- suppress_txn_msgs, fake_bcast, force_ser, fake_exec;
+ use_pb, use_pb_res, g_caught_up, rec_pwal, rec_twal, do_tpcc,
+ suppress_txn_msgs, fake_bcast, force_ser, fake_exec, ship_log;
long long timelim, read_thresh, write_thresh;
// Control.
st_intr_bool stop_hub, kill_hub;
st_bool do_pause;
+// On leader, signifies that a node is in fail mode. On replica, signifies that a node is in fail mode/recovering from the twal.
+st_bool failed;
+// The seqno on which we should resume.
+st_channel<int> resume;
bool stopped_issuing;
// Statistics.
@@ -584,6 +590,34 @@
check(msg.ParseFromArray(checkpass(src.read(len)), len));
}
+template<typename T>
+inline void
+readmsg(istream &src, T &msg)
+{
+ uint32_t len;
+ src.read(reinterpret_cast<char*>(&len), sizeof len);
+ len = ntohl(len);
+#if 0
+ IstreamInputStream iis(&src);
+ LimitingInputStream lis(&iis, len);
+ check(msg.ParseFromZeroCopyStream(&lis));
+#else
+ char buf[len];
+ src.read(buf, len);
+ check(msg.ParseFromArray(buf, len));
+#endif
+}
+
+inline uint32_t
+readlen(istream &src)
+{
+ uint32_t len;
+ src.read(reinterpret_cast<char*>(&len), sizeof len);
+ len = ntohl(len);
+ ASSERT(len < 10000);
+ return len;
+}
+
enum { op_del, op_write, op_commit };
/**
@@ -611,14 +645,30 @@
int op = op_commit;
out & op;
}
+ void flush() { of.flush(); }
private:
ofstream of;
binary_oarchive out;
};
+// TODO?
+class txn_wal {
+public:
+ txn_wal(const string &fname) : of(fname.c_str()) {}
+ void logbuf(const ser_t &s) { logbuf(s.data(), s.size()); }
+ void logbuf(const void *buf, size_t len) {
+ of.write(reinterpret_cast<const char*>(buf), len);
+ }
+ void flush() { of.flush(); }
+private:
+ ofstream of;
+};
+
// Globals
mii g_map;
wal *g_wal;
+txn_wal *g_twal;
+//tpcc_wal *g_tpcc_wal;
/**
* Keep issuing transactions to the replicas.
@@ -645,7 +695,7 @@
reader r(nullptr, rbuf.get(), rbuf.size());
function<void(const void*, size_t)> fn;
if (use_twal)
- fn = bind(&wal::logbuf, g_wal, _1, _2);
+ fn = bind(&txn_wal::logbuf, g_twal, _1, _2);
else
fn = lambda(const void *buf, size_t len) {
foreach (st_netfd_t dst, __ref(fds))
@@ -754,7 +804,7 @@
serbuf.clear();
ser(serbuf, batch);
if (do_bcast) bcastbuf(fds, serbuf);
- if (use_twal) g_wal->logbuf(serbuf);
+ if (use_twal) g_twal->logbuf(serbuf);
}
} else {
// Reset if we have nobody to send to (incl. disk) or if we actually have
@@ -858,14 +908,14 @@
showdatarate(const char *action, streamoff len, long long time)
{
cout << action << " of " << len << " bytes in " << time << " ms ("
- << len / time / 1000 << " MB/s)" << endl;
+ << double(len) / double(time) / 1000 << " MB/s)" << endl;
}
void
showdatarate(const char *action, size_t len, long long time)
{
cout << action << " of " << len << " bytes in " << time << " ms ("
- << len / time / 1000 << " MB/s)" << endl;
+ << double(len) / double(time) / 1000 << " MB/s)" << endl;
}
void
@@ -991,23 +1041,29 @@
res->Clear();
res->set_seqno(seqno);
}
+ // First three are read-only txns, so doesn't make sense to exec if no res to
+ // put results. They constitute only 8% of the workload.
if (req.has_stock_level()) {
- const StockLevelMsg &sl = req.stock_level();
- int result = g_tables->stockLevel(sl.warehouse_id(), sl.district_id(), sl.threshold());
if (res != nullptr) {
+ const StockLevelMsg &sl = req.stock_level();
+ int result = g_tables->stockLevel(sl.warehouse_id(), sl.district_id(), sl.threshold());
StockLevelOutputMsg &msg = *res->mutable_stock_level();
msg.set_result(result);
}
} else if (req.has_order_status_1()) {
- const OrderStatusMsg1 &os = req.order_status_1();
- OrderStatusOutput output;
- g_tables->orderStatus(os.warehouse_id(), os.district_id(), os.customer_id(), &output);
- if (res != nullptr) mkres(res, output);
+ if (res != nullptr) {
+ const OrderStatusMsg1 &os = req.order_status_1();
+ OrderStatusOutput output;
+ g_tables->orderStatus(os.warehouse_id(), os.district_id(), os.customer_id(), &output);
+ mkres(res, output);
+ }
} else if (req.has_order_status_2()) {
- const OrderStatusMsg2 &os = req.order_status_2();
- OrderStatusOutput output;
- g_tables->orderStatus(os.warehouse_id(), os.district_id(), os.c_last().c_str(), &output);
- if (res != nullptr) mkres(res, output);
+ if (res != nullptr) {
+ const OrderStatusMsg2 &os = req.order_status_2();
+ OrderStatusOutput output;
+ g_tables->orderStatus(os.warehouse_id(), os.district_id(), os.c_last().c_str(), &output);
+ mkres(res, output);
+ }
} else if (req.has_new_order()) {
const NewOrderMsg &no = req.new_order();
vector<NewOrderItem> items(no.item_size());
@@ -1079,7 +1135,11 @@
int mypos, int nnodes)
{
bool caught_up = init_seqno == 0;
+ // Means we're currently ignoring the incoming txns until we see a fail-ack
+ // from the leader.
+ bool depleting = false;
long long start_time = current_time_millis(),
+ time_failed = -1,
time_caught_up = caught_up ? start_time : -1;
int seqno_caught_up = caught_up ? seqno : -1;
// Used by joiner only to tell where we actually started (init_seqno is just
@@ -1090,6 +1150,7 @@
int first_seqno_in_chunk = -1;
TpccReq req;
TpccRes res;
+ txn_wal &wal = *g_twal;
function<void(anchored_stream_reader& reader)> overflow_fn =
lambda(anchored_stream_reader &reader) {
@@ -1124,8 +1185,7 @@
anchored_stream_reader reader(st_read_fn(leader), st_read_fully_fn(leader),
overflow_fn, rbuf.get(), rbuf.size());
writer w(lambda(const void *buf, size_t len) {
- checkeqnneg(st_write(__ref(leader), buf, len, ST_UTIME_NO_TIMEOUT),
- static_cast<ssize_t>(len));
+ st_write(__ref(leader), buf, len);
}, wbuf.get(), wbuf.size());
finally f(lambda () {
@@ -1142,14 +1202,17 @@
__ref(w).mark_and_flush();
});
+ function<void()> send_failure_msg = lambda() {
+ TpccRes &res = __ref(res);
+ writer &w = __ref(w);
+ res.Clear();
+ res.set_seqno(-1);
+ ser(w, res);
+ w.mark_and_flush();
+ };
+
while (true)
{
- // Has replayer just caught up to the start of the chunk?
- if (first_seqno_in_chunk == seqno + 1) {
- process_buf(reader.anchor(), reader.start(), req, seqno);
- reader.set_anchor();
- }
-
marker = reader.start();
{
@@ -1164,33 +1227,80 @@
// Prepare recovery msg.
send_states.push(make_tpcc_recovery(mypos, nnodes, seqno));
} else {
- // Backlog (auto/implicit) or process.
- if (!caught_up) {
- // If we were at the start of a new buffer (our chunk was recently reset).
- if (reader.buf().get() == marker)
- first_seqno_in_chunk = req.seqno();
- // If we fully caught up.
- if (req.seqno() == seqno + 1) {
- time_caught_up = current_time_millis();
- seqno_caught_up = seqno;
- showtput("process_tpccs caught up; backlogged",
- time_caught_up, start_time, seqno_caught_up,
- first_seqno == -1 ? init_seqno - 1 : first_seqno);
- caught_up = true;
+
+ if (depleting) {
+ if (req.seqno() == -3) {
+ // Fail-ack. Should not be receiving anything until we resume.
+ failed.waitreset();
+ send_failure_msg();
+ // Note that we don't reset depleting; we want the next iteration to
+ // fall through to the next case in this if-else chain....
+
+ // Adjust reader so that the next xact (the first one after failure)
+ // will go to the start of the buffer; this is necessary for
+ // backlogging.
+ reader.set_anchor();
+ shift_reader(reader);
+ } else if (!failed) {
+ // This is the first txn after resuming. Tell the recoverer task
+ // that this is the seqno to build up to (from another replica's
+ // log).
+ resume.push(req.seqno());
+ depleting = false;
}
+ // Ignore all other messages.
}
- if (caught_up) {
- // Process.
- process_tpcc(req, seqno, &res);
- ser(w, res);
- reader.set_anchor();
+
+ if (!depleting) {
+ if (req.seqno() == -3) {
+ // Ignore the fail-ack.
+ } else {
+ if (use_twal) wal.logbuf(marker, reader.start() - marker);
+
+ // Backlog (auto/implicit) or process.
+ if (!caught_up) {
+ // If we were at the start of a new buffer (our chunk was recently reset).
+ if (reader.buf().get() == marker)
+ first_seqno_in_chunk = req.seqno();
+ // If we fully caught up.
+ if (req.seqno() == seqno + 1) {
+ time_caught_up = current_time_millis();
+ seqno_caught_up = seqno;
+ showtput("process_tpccs caught up; backlogged",
+ time_caught_up, start_time, seqno_caught_up,
+ first_seqno == -1 ? init_seqno - 1 : first_seqno);
+ caught_up = true;
+ }
+ }
+ if (caught_up) {
+ // Process.
+ process_tpcc(req, seqno, &res);
+ ser(w, res);
+ reader.set_anchor();
+ }
+
+ // Display/yield.
+ if (check_interval(req.seqno(), process_display))
+ cout << (caught_up ? "processed req " : "backlogged req ")
+ << req.seqno() << endl;
+ if (check_interval(req.seqno(), yield_interval)) st_sleep(0);
+
+ // Die.
+ if (fail_seqno > 0 && req.seqno() == fail_seqno) {
+ cout << "process_tpccs failing on seqno " << fail_seqno;
+ time_failed = current_time_millis();
+ showtput("; live-processed ", time_failed, start_time, seqno, 0);
+ ASSERT(init_seqno == 0);
+ caught_up = false;
+ depleting = true;
+ seqno = -1;
+
+ failed.set();
+ send_failure_msg();
+ }
+ }
}
- // Display/yield.
- if (check_interval(req.seqno(), process_display))
- cout << (caught_up ? "processed req " : "backlogged req ")
- << req.seqno() << endl;
- if (check_interval(req.seqno(), yield_interval)) st_sleep(0);
}
}
@@ -1698,12 +1808,14 @@
{
public:
tpcc_response_handler(st_netfd_t replica, const int &seqno, int rid,
- st_multichannel<long long> &recover_signals, bool caught_up)
+ st_multichannel<long long> &recover_signals,
+ st_channel<st_netfd_t> &delreps, bool caught_up)
:
replica(replica),
seqno(seqno),
rid(rid),
recover_signals(recover_signals),
+ delreps(delreps),
caught_up(caught_up),
sub(recover_signals.subscribe()),
start_time(current_time_millis()),
@@ -1744,38 +1856,48 @@
readmsg(reader, res);
}
- if (res.seqno() < last_seqno)
- throw msg_exception(string("response seqno decreased from ") +
- lexical_cast<string>(last_seqno) + " to " +
- lexical_cast<string>(res.seqno()));
+ if (res.seqno() == -1) {
+ st_intr intr(stop_hub);
+ cout << "got a failed node" << endl;
+ delreps.push(replica);
+ readmsg(reader, res);
+ last_seqno = seqno - 1;
+ } else {
- if (!caught_up) {
- long long now = current_time_millis(), time_diff = now - start_time;
- caught_up = true;
- recover_signals.push(now);
- cout << rid << ": " << "recovering node caught up; took "
- << time_diff << " ms" << endl;
- // This will cause the program to exit eventually, but cleanly, such that
- // the recovery time will be set first, before the eventual exit (which
- // may not even happen in the current iteration).
- if (stop_on_recovery) {
- cout << "stopping on recovery" << endl;
- stop_hub.set();
+ if (res.seqno() < last_seqno)
+ throw msg_exception(string("response seqno decreased from ") +
+ lexical_cast<string>(last_seqno) + " to " +
+ lexical_cast<string>(res.seqno()));
+
+ if (!caught_up) {
+ long long now = current_time_millis(), time_diff = now - start_time;
+ caught_up = true;
+ recover_signals.push(now);
+ cout << rid << ": " << "recovering node caught up; took "
+ << time_diff << " ms" << endl;
+ // This will cause the program to exit eventually, but cleanly, such that
+ // the recovery time will be set first, before the eventual exit (which
+ // may not even happen in the current iteration).
+ if (stop_on_recovery) {
+ cout << "stopping on recovery" << endl;
+ stop_hub.set();
+ }
}
- }
- if (check_interval(res.seqno(), handle_responses_display)) {
- cout << rid << ": " << "got response " << res.seqno() << " from "
- << replica << "; ";
- long long display_time = current_time_millis();
- showtput("handling", display_time, last_display_time, res.seqno(),
- res.seqno() - handle_responses_display);
- last_display_time = display_time;
+ if (check_interval(res.seqno(), handle_responses_display)) {
+ cout << rid << ": " << "got response " << res.seqno() << " from "
+ << replica << "; ";
+ long long display_time = current_time_millis();
+ showtput("handling", display_time, last_display_time, res.seqno(),
+ res.seqno() - handle_responses_display);
+ last_display_time = display_time;
+ }
+ if (check_interval(res.seqno(), yield_interval)) {
+ st_sleep(0);
+ }
+ last_seqno = res.seqno();
+
}
- if (check_interval(res.seqno(), yield_interval)) {
- st_sleep(0);
- }
- last_seqno = res.seqno();
}
}
@@ -1815,6 +1937,7 @@
const int &seqno;
int rid;
st_multichannel<long long> &recover_signals;
+ st_channel<st_netfd_t> &delreps;
bool caught_up;
st_channel<long long> ⊂
long long start_time, recovery_start_time, recovery_end_time;
@@ -1823,12 +1946,18 @@
void
handle_tpcc_responses(st_netfd_t replica, const int &seqno, int rid,
- st_multichannel<long long> &recover_signals, bool caught_up)
+ st_multichannel<long long> &recover_signals,
+ st_channel<st_netfd_t> &delreps, bool caught_up)
{
- tpcc_response_handler h(replica, seqno, rid, recover_signals, caught_up);
+ tpcc_response_handler h(replica, seqno, rid, recover_signals, delreps,
+ caught_up);
h.run();
}
+struct recreq {
+ int start_seqno, end_seqno;
+};
+
/**
* Help the recovering node.
*
@@ -1846,32 +1975,69 @@
recover_joiner(st_netfd_t listener,
st_channel<recovery_t> &send_states)
{
+ cout << "waiting for joiner" << endl;
+ recovery_t recovery;
st_netfd_t joiner;
- recovery_t recovery;
- {
- st_intr intr(stop_hub);
- // Wait for the snapshot.
- recovery = send_states.take();
- if (recovery == nullptr) {
- return;
+ if (ship_log) {
+ {
+ st_intr intr(stop_hub);
+ joiner = checkerr(st_accept(listener, nullptr, nullptr,
+ ST_UTIME_NO_TIMEOUT));
}
- // Wait for the new joiner.
- joiner = checkerr(st_accept(listener, nullptr, nullptr,
- ST_UTIME_NO_TIMEOUT));
+ st_closing closing(joiner);
+ recreq r;
+ st_read(joiner, r);
+ commons::array<char> wbuf(buf_size);
+ writer writer(lambda(const void *buf, size_t len) {
+ st_write(__ref(joiner), buf, len);
+ }, wbuf.get(), wbuf.size());
+ cout << "got joiner's connection, sending log from seqnos "
+ << r.start_seqno << " to " << r.end_seqno << endl;
+
+ g_twal->flush();
+ sync();
+ ifstream inf("twal");
+ long long start_time = current_time_millis();
+ for (int seqno = 0; seqno < r.start_seqno; ++seqno) {
+ ASSERT(inf.good());
+ inf.seekg(readlen(inf), ios::cur);
+ }
+ long long mid_time = current_time_millis();
+ streamoff mid_off = inf.tellg();
+ showdatarate("scanned log", mid_off, mid_time - start_time);
+ for (int seqno = r.start_seqno; seqno < r.end_seqno; ++seqno) {
+ ASSERT(inf.good());
+ uint32_t len = readlen(inf);
+ inf.read(writer.reserve(len), len);
+ writer.mark();
+ cout << seqno << ' ' << len << endl;
+ if (check_interval(seqno, yield_interval)) st_sleep(0);
+ }
+ writer.mark_and_flush();
+ long long end_time = current_time_millis();
+ streamoff end_off = inf.tellg();
+ showdatarate("shipped log", end_off - mid_off, end_time - mid_time);
+ } else {
+ {
+ st_intr intr(stop_hub);
+ // Wait for the snapshot.
+ recovery = send_states.take();
+ if (recovery == nullptr) {
+ return;
+ }
+ // Wait for the new joiner.
+ joiner = checkerr(st_accept(listener, nullptr, nullptr,
+ ST_UTIME_NO_TIMEOUT));
+ }
+
+ st_closing closing(joiner);
+ cout << "got joiner's connection, sending recovery of "
+ << recovery.size() << " bytes" << endl;
+ long long start_time = current_time_millis();
+ st_write(joiner, recovery.get(), recovery.size());
+ long long diff = current_time_millis() - start_time;
+ showdatarate("sent recovery", recovery.size(), diff);
}
-
- st_closing closing(joiner);
- cout << "got joiner's connection, sending recovery of "
- << recovery.size() << " bytes" << endl;
- long long start_time = current_time_millis();
- checkeqnneg(st_write(joiner, recovery.get(), recovery.size(),
- ST_UTIME_NO_TIMEOUT),
- ssize_t(recovery.size()));
- long long diff = current_time_millis() - start_time;
-#if 0
- sendmsg(joiner, *recovery);
-#endif
- showdatarate("sent recovery", recovery.size(), diff);
}
void
@@ -1893,8 +2059,10 @@
cout << "starting as leader" << endl;
st_multichannel<long long> recover_signals;
- scoped_ptr<wal> twal(new wal(use_twal ? "twal" : "/dev/null"));
- g_wal = twal.get();
+ scoped_ptr<txn_wal> twal(new txn_wal(use_twal ? "twal" : "/dev/null"));
+ g_twal = twal.get();
+ scoped_ptr<wal> pwal(new wal(use_pwal ? "pwal" : "/dev/null"));
+ g_wal = pwal.get();
// Wait until all replicas have joined.
st_netfd_t listener = st_tcp_listen(leader_port);
@@ -1934,10 +2102,13 @@
st_bool accept_joiner;
int seqno = 0;
st_channel<replica_info> newreps;
+ st_channel<st_netfd_t> delreps;
foreach (const replica_info &r, replicas) newreps.push(r);
- const function<void()> f = do_tpcc ?
- bind(issue_tpcc, ref(newreps), ref(seqno), ref(accept_joiner)) :
- bind(issue_txns<Types>, ref(newreps), ref(seqno), ref(accept_joiner));
+ function<void()> f;
+ if (do_tpcc)
+ f = bind(issue_tpcc, ref(newreps), ref(delreps), ref(seqno), ref(accept_joiner));
+ else
+ f = bind(issue_txns<Types>, ref(newreps), ref(seqno), ref(accept_joiner));
st_joining join_issue_txns(my_spawn(f, "issue_txns"));
finally fin(bind(summarize, "LEADER", ref(seqno)));
@@ -1950,7 +2121,7 @@
function<void()> fn;
if (do_tpcc)
fn = bind(handle_tpcc_responses, r.fd(), ref(seqno), rid++,
- ref(recover_signals), true);
+ ref(recover_signals), ref(delreps), true);
else
fn = bind(handle_responses<RTypes>, r.fd(), ref(seqno), rid++,
ref(recover_signals), true);
@@ -1985,7 +2156,7 @@
if (do_tpcc)
handle_responses_joiner_fn =
bind(handle_tpcc_responses, joiner, ref(seqno), rid++,
- ref(recover_signals), false);
+ ref(recover_signals), ref(delreps), false);
else
handle_responses_joiner_fn =
bind(handle_responses<RTypes>, joiner, ref(seqno), rid++,
@@ -2076,6 +2247,7 @@
<< current_time_millis() - start_time << " ms" << endl;
tables->show();
}
+ recovery_t orig = rec_twal ? g_tables->ser(0, 0, seqno) : recovery_t();
finally f(bind(summarize, "REPLICA", ref(seqno)));
st_channel<recovery_t> send_states;
@@ -2115,14 +2287,16 @@
INET_ADDRSTRLEN))
<< ':' << sa.port() << (is_self ? " (self)" : "") << endl;
if (is_self) mypos = i;
- if (!is_self && init.txnseqno() > 0) {
+ if (!is_self && (init.txnseqno() > 0 || rec_twal)) {
replicas.push_back(st_tcp_connect(host,
static_cast<uint16_t>(sa.port()),
timeout));
}
}
- // Initialize physical log.
+ // Initialize physical or txn log.
+ scoped_ptr<txn_wal> twal(new txn_wal(use_twal ? "twal" : "/dev/null"));
+ g_twal = twal.get();
scoped_ptr<wal> pwal(new wal(use_pwal ? "pwal" : "/dev/null"));
g_wal = pwal.get();
@@ -2137,29 +2311,99 @@
ref(send_states), ref(backlog), init.txnseqno(), mypos,
init.node_size());
st_joining join_proc(my_spawn(process_fn, "process_txns"));
- if (init.txnseqno() == 0 && multirecover || mypos == 0)
- st_joining join_rec(my_spawn(bind(recover_joiner, listener,
- ref(send_states)),
- "recover_joiner"));
+ st_joining join_rec(init.txnseqno() == 0 && (multirecover || mypos == 0) ?
+ my_spawn(bind(recover_joiner, listener, ref(send_states)),
+ "recover_joiner") :
+ nullptr);
try {
// If there's anything to recover.
- if (init.txnseqno() > 0) {
+ if (init.txnseqno() > 0 || fail_seqno > 0) {
if (do_tpcc) {
//
// TPCC txns
//
- g_tables.reset(new TPCCTables);
+ function<void()> rec_twal_fn = lambda() {
+ int &seqno = __ref(seqno);
+ cout << "recovering from twal" << endl;
+ long long start_time = current_time_millis();
+ g_twal->flush();
+ sync();
+ ifstream inf("twal");
+ TpccReq req;
+ while (inf.peek() != ifstream::traits_type::eof()) {
+ ASSERT(inf.good());
+ readmsg(inf, req);
+ process_tpcc(req, seqno, nullptr);
+ if (check_interval(seqno, yield_interval)) st_sleep(0);
+ }
+ showdatarate("recovered from twal", inf.tellg(),
+ current_time_millis() - start_time);
+ cout << "now at seqno " << seqno << endl;
+ };
+ function<void()> recv_log_fn = lambda() {
+ st_netfd_t src = __ref(replicas[0]);
+ int &seqno = __ref(seqno);
+ ASSERT(fail_seqno == seqno);
+ recreq r = { fail_seqno + 1, resume.take() };
+ st_write(src, r);
+ sized_array<char> rbuf(new char[read_buf_size], read_buf_size);
+ function<void(anchored_stream_reader &reader)> overflow_fn =
+ lambda(anchored_stream_reader &reader) {
+ shift_reader(reader);
+ };
+ anchored_stream_reader reader(st_read_fn(src),
+ st_read_fully_fn(src),
+ overflow_fn, rbuf.get(), rbuf.size());
+ TpccReq req;
+ while (seqno < r.end_seqno) {
+ { st_intr intr(stop_hub); readmsg(reader, req); }
+ process_tpcc(req, seqno, nullptr);
+ reader.set_anchor();
+ if (check_interval(seqno, yield_interval)) st_sleep(0);
+ }
+ };
+
+ if (rec_twal) {
+ failed.waitset();
+ g_tables.reset(new TPCCTables);
+ tpcc_recovery_header &hdr = *reinterpret_cast<tpcc_recovery_header*>(orig.begin());
+ commons::array<char> body(orig.begin() + sizeof(tpcc_recovery_header),
+ orig.size() - sizeof(tpcc_recovery_header));
+ g_tables->deser(mypos, init.node_size(), hdr, body);
+ body.release();
+ rec_twal_fn();
+ failed.reset();
+ recv_log_fn();
+ }
+
+#if 0
+ st_thread_t rec_twal_thread = my_spawn(rec_twal_fn, "rec_twal");
+ st_thread_t recv_log_thread = my_spawn(recv_log_fn, "recv_log");
+
+ st_join(rec_twal_thread);
+ st_join(recv_log_thread);
+#endif
+
if (rec_pwal) {
+ // Recover from phy log.
+ } else if (rec_twal) {
+ // Recover from txn log.
} else {
+ g_tables.reset(new TPCCTables);
+
//
// Build-up
//
+ if (ship_log) {
+ } else {
+ // XXX indent
+
cout << "waiting for recovery message" << (multirecover ? "s" : "")
<< endl;
long long before_recv = current_time_millis();
@@ -2237,6 +2481,8 @@
foreach (st_thread_t t, recovery_builders) {
st_join(t);
}
+
+ }
}
//
@@ -2558,6 +2804,13 @@
void flush() { writer_.mark_and_flush(); }
void set_seqno(int seqno) { seqno_ = seqno; }
+ void sendStop() {
+ req_.Clear();
+ req_.set_seqno(-1);
+ ser(writer_, req_);
+ writer_.mark_and_flush();
+ }
+
void sendRec() {
req_.Clear();
req_.set_seqno(-2);
@@ -2565,9 +2818,9 @@
writer_.mark_and_flush();
}
- void sendStop() {
+ void sendFailAck() {
req_.Clear();
- req_.set_seqno(-1);
+ req_.set_seqno(-3);
ser(writer_, req_);
writer_.mark_and_flush();
}
@@ -2704,7 +2957,9 @@
};
void
-issue_tpcc(st_channel<replica_info> &newreps, int &seqno,
+issue_tpcc(st_channel<replica_info> &newreps,
+ st_channel<st_netfd_t> &delreps,
+ int &seqno,
st_bool &accept_joiner)
{
vector<st_netfd_t> fds;
@@ -2732,7 +2987,7 @@
// one) to prepare to send recovery information (by sending an
// empty/default Txn).
// XXX rec_pwal
- if (!newreps.empty() && seqno > 0 && !rec_pwal) {
+ if (!newreps.empty() && seqno > 0 && !rec_pwal && !rec_twal) {
tables.sendRec();
}
@@ -2741,6 +2996,10 @@
while (!newreps.empty()) {
fds.push_back(newreps.take().fd());
}
+ while (!delreps.empty()) {
+ tables.sendFailAck();
+ fds.erase( find(fds.begin(), fds.end(), delreps.take()) );
+ }
tables.set_seqno(seqno);
client.doOne();
@@ -2863,10 +3122,14 @@
"yield periodically during catch-up phase of recovery (for recoverer)")
("multirecover,m", po::bool_switch(&multirecover),
"recover from multiple hosts, instead of just one (specified via leader)")
+ ("rec-twal", po::bool_switch(&rec_twal),
+ "recover from twal")
("rec-pwal", po::bool_switch(&rec_pwal),
"recover from pwal")
("disk,k", po::bool_switch(&disk),
"use disk-based recovery")
+ ("ship-log", po::bool_switch(&ship_log),
+ "ship the log instead of the complete database state")
("dump,D", po::bool_switch(&dump),
"replicas should finally dump their state to a tmp file for "
"inspection/diffing")
@@ -2930,6 +3193,9 @@
("warehouses,w",
po::value<int>(&nwarehouses)->default_value(1),
"number of warehouses to model")
+ ("fail-seqno",
+ po::value<int>(&fail_seqno)->default_value(0),
+ "fail after processing this seqno (for replica only)")
("accept-joiner-seqno,j",
po::value<int>(&accept_joiner_seqno)->default_value(0),
"accept recovering joiner (start recovery) after this seqno (for leader "
Modified: ydb/trunk/src/tpcc/tpccclient.cc
===================================================================
--- ydb/trunk/src/tpcc/tpccclient.cc 2009-03-18 20:08:05 UTC (rev 1307)
+++ ydb/trunk/src/tpcc/tpccclient.cc 2009-03-19 10:16:33 UTC (rev 1308)
@@ -68,11 +68,13 @@
vector<DeliveryOrderInfo> orders;
db_->delivery(generateWarehouse(), carrier, now, &orders);
+#if 0
#ifndef NDEBUG
if (orders.size() != District::NUM_PER_WAREHOUSE) {
printf("Only delivered from %zd districts\n", orders.size());
}
#endif
+#endif
}
void TPCCClient::doPayment() {
Modified: ydb/trunk/src/tpcc/tpcctables.cc.cog
===================================================================
--- ydb/trunk/src/tpcc/tpcctables.cc.cog 2009-03-18 20:08:05 UTC (rev 1307)
+++ ydb/trunk/src/tpcc/tpcctables.cc.cog 2009-03-19 10:16:33 UTC (rev 1308)
@@ -194,6 +194,8 @@
// Modify the order id to assign it
d->d_next_o_id += 1;
+ //XXX pwal.write(d);
+ //XXX pwal.write(d->d_next_o_id);
Warehouse* w = findWarehouse(warehouse_id);
output->w_tax = w->w_tax;
Modified: ydb/trunk/src/tpcc/tpcctables.h
===================================================================
--- ydb/trunk/src/tpcc/tpcctables.h 2009-03-18 20:08:05 UTC (rev 1307)
+++ ydb/trunk/src/tpcc/tpcctables.h 2009-03-19 10:16:33 UTC (rev 1308)
@@ -152,6 +152,8 @@
std::vector<const History*> history_;
commons::sized_array<char> serbuf_;
+
+ //XXX raw_writer pwal;
};
#endif
Modified: ydb/trunk/tools/analysis.py
===================================================================
--- ydb/trunk/tools/analysis.py 2009-03-18 20:08:05 UTC (rev 1307)
+++ ydb/trunk/tools/analysis.py 2009-03-19 10:16:33 UTC (rev 1308)
@@ -117,27 +117,44 @@
r'=== n=(?P<n>-?\d+) ',
r'issued .*\((?P<tps>[.\d]+) tps\)' ])
- print 'file:', getname(ariespath)
- res2 = logextract(ariespath, 'n', [
- r'=== n=(?P<n>-?\d+) ',
- r'issued .*\((?P<tps>[.\d]+) tps\)' ])
+ if path(ariespath).exists():
+ print 'file:', getname(ariespath)
+ res2 = logextract(ariespath, 'n', [
+ r'=== n=(?P<n>-?\d+) ',
+ r'issued .*\((?P<tps>[.\d]+) tps\)' ])
- print hstack([res2['n'], res0['n'][:1], res['n']])
- print hstack([res2['tps mean'], res0['tps mean'][:1], res['tps mean']])
- print hstack([res2['tps sd'], res0['tps sd'][:1], res['tps sd']])
+ print hstack([res2['n'], res0['n'][:1], res['n']])
+ print hstack([res2['tps mean'], res0['tps mean'][:1], res['tps mean']])
+ print hstack([res2['tps sd'], res0['tps sd'][:1], res['tps sd']])
- errorbar(
- hstack([res2['n'], res0['n'][:1], res['n']]),
- hstack([res2['tps mean'], res0['tps mean'][:1], res['tps mean']]),
- hstack([res2['tps sd'], res0['tps sd'][:1], res['tps sd']]))
- title('Scaling of baseline throughput with number of nodes')
- xlabel('Node count')
- ylabel('Mean TPS (stdev error bars)')
- xlim(hstack([res2['n'], res['n']]).min() - .5,
- hstack([res2['n'], res['n']]).max() + .5)
- ylim(ymin = 0)
- savefig('scaling.png')
+ errorbar(
+ hstack([res2['n'], res0['n'][:1], res['n']]),
+ hstack([res2['tps mean'], res0['tps mean'][:1], res['tps mean']]),
+ hstack([res2['tps sd'], res0['tps sd'][:1], res['tps sd']]))
+ title('Scaling of baseline throughput with number of nodes')
+ xlabel('Node count')
+ ylabel('Mean TPS (stdev error bars)')
+ xlim(hstack([res2['n'], res['n']]).min() - .5,
+ hstack([res2['n'], res['n']]).max() + .5)
+ ylim(ymin = 0)
+ savefig('scaling.png')
+ else:
+ print hstack([res0['n'][:1], res['n']])
+ print hstack([res0['tps mean'][:1], res['tps mean']])
+ print hstack([res0['tps sd'][:1], res['tps sd']])
+ errorbar(
+ hstack([res0['n'][:1], res['n']]),
+ hstack([res0['tps mean'][:1], res['tps mean']]),
+ hstack([res0['tps sd'][:1], res['tps sd']]))
+ title('Scaling of baseline throughput with number of nodes')
+ xlabel('Node count')
+ ylabel('Mean TPS (stdev error bars)')
+ xlim(hstack([res['n']]).min() - .5,
+ hstack([res['n']]).max() + .5)
+ ylim(ymin = 0)
+ savefig('scaling.png')
+
def run(singlepath, multipath):
singlepath, multipath = map(path, [singlepath, multipath])
ress = []
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <yan...@us...> - 2009-03-18 20:08:22
|
Revision: 1307
http://assorted.svn.sourceforge.net/assorted/?rev=1307&view=rev
Author: yangzhang
Date: 2009-03-18 20:08:05 +0000 (Wed, 18 Mar 2009)
Log Message:
-----------
- main: fixed starting of extraneous recover_joiner's
- test.bash: added whos, updated line-counts
- removed wc.bash
Modified Paths:
--------------
ydb/trunk/README
ydb/trunk/src/main.lzz.clamp
ydb/trunk/tools/test.bash
Removed Paths:
-------------
ydb/trunk/tools/wc.bash
Modified: ydb/trunk/README
===================================================================
--- ydb/trunk/README 2009-03-18 19:25:42 UTC (rev 1306)
+++ ydb/trunk/README 2009-03-18 20:08:05 UTC (rev 1307)
@@ -749,8 +749,8 @@
- DONE full TPC-C txn workload
- readonly: order_status, stock_level
- overwhelming majority are payment/neworder
+- DONE prelim measurements on cluster
- TODO skip processing readonly txns on catch-up
-- TODO prelim measurements on cluster
- TODO PAPER!!!
Modified: ydb/trunk/src/main.lzz.clamp
===================================================================
--- ydb/trunk/src/main.lzz.clamp 2009-03-18 19:25:42 UTC (rev 1306)
+++ ydb/trunk/src/main.lzz.clamp 2009-03-18 20:08:05 UTC (rev 1307)
@@ -2137,9 +2137,10 @@
ref(send_states), ref(backlog), init.txnseqno(), mypos,
init.node_size());
st_joining join_proc(my_spawn(process_fn, "process_txns"));
- st_joining join_rec(my_spawn(bind(recover_joiner, listener,
- ref(send_states)),
- "recover_joiner"));
+ if (init.txnseqno() == 0 && multirecover || mypos == 0)
+ st_joining join_rec(my_spawn(bind(recover_joiner, listener,
+ ref(send_states)),
+ "recover_joiner"));
try {
// If there's anything to recover.
Modified: ydb/trunk/tools/test.bash
===================================================================
--- ydb/trunk/tools/test.bash 2009-03-18 19:25:42 UTC (rev 1306)
+++ ydb/trunk/tools/test.bash 2009-03-18 20:08:05 UTC (rev 1307)
@@ -315,6 +315,15 @@
"
}
+whos() {
+ xargs= parssh "
+ echo
+ hostname
+ echo =====
+ w
+ "
+}
+
times() {
parssh date +%s.%N
}
@@ -350,7 +359,7 @@
# Recovery experient.
exp-rec() {
- for seqno in 500000 400000 300000 200000 100000 ; do # configurations
+ for seqno in 300000 200000 100000 ; do # configurations
stop
for i in {1..3} ; do # trials
echo === seqno=$seqno i=$i ===
@@ -429,7 +438,7 @@
tagssh $leader "CPUPROFILE=ydb.prof ydb/src/ydb -q -l -n $# -X 100000 --tpcc ${extraargs:-}" &
sleep .1
for rep in "$@"
- do tagssh $rep "CPUPROFILE=ydb.prof --args ydb/src/ydb -q -n $# -H $leader --tpcc ${extraargs:-}" &
+ do tagssh $rep "CPUPROFILE=ydb.prof ydb/src/ydb -q -n $# -H $leader --tpcc ${extraargs:-}" &
done
wait
}
@@ -566,7 +575,7 @@
}
line-counts() {
- wc -l "$(dirname "$0")/../src/"{main.lzz.clamp,ser.{h,cc},p2.cc,ydb.proto,Makefile,serperf.cc} \
+ wc -l "$(dirname "$0")/../src/"{main.lzz.clamp,ser.{h,cc},p2.cc,ydb.proto,Makefile,serperf.cc,tpcc/{Makefile,*.{h,cc,cog}}} \
~/ccom/src/{commons/{,st/}*.h,test/{*.*,Makefile}}
}
Deleted: ydb/trunk/tools/wc.bash
===================================================================
--- ydb/trunk/tools/wc.bash 2009-03-18 19:25:42 UTC (rev 1306)
+++ ydb/trunk/tools/wc.bash 2009-03-18 20:08:05 UTC (rev 1307)
@@ -1,11 +0,0 @@
-#!/usr/bin/env bash
-
-cd "$(dirname "$0")/../src/"
-wc -l main.lzz.clamp
-{
- wc -l main.lzz.clamp
-
- cat main.lzz.clamp |
- perl -n -e 'if (m /#include <commons/s) { s/.*<(.+)>.*/$1/; print }' |
- xargs -I_ wc -l /home/yang/ccom/src/_
-} | cut -f1 -d' ' | numsum | xargs -I_ echo _ total
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <yan...@us...> - 2009-03-18 19:25:57
|
Revision: 1306
http://assorted.svn.sourceforge.net/assorted/?rev=1306&view=rev
Author: yangzhang
Date: 2009-03-18 19:25:42 +0000 (Wed, 18 Mar 2009)
Log Message:
-----------
- tpccdb: fixed buf overflow, h_data should have size MAX_DATA + 1
- general:
- fixed the tpcc_response_handler to have proper termination condition
- fixed optimized build issues
- tested and verified that scaling, rec fully work
- test.bash:
- added cog to setups
- adjusted rec, scaling to use tpcc (and lowered some constants)
- added locking
- tpcctables: safely delete neworders, avoiding deletion if in serbuf
- btree: added conditional compilation of NODE_INNER, NODE_LEAF just to be safe
- tpccclient: fixed inverted NDEBUG
- tpcc/Makefile: handle OPT
- Makefile: use g++ for linking
Modified Paths:
--------------
ydb/trunk/src/Makefile
ydb/trunk/src/main.lzz.clamp
ydb/trunk/src/tpcc/Makefile
ydb/trunk/src/tpcc/btree.h
ydb/trunk/src/tpcc/tpccclient.cc
ydb/trunk/src/tpcc/tpccdb.h
ydb/trunk/src/tpcc/tpcctables.cc.cog
ydb/trunk/tools/test.bash
Modified: ydb/trunk/src/Makefile
===================================================================
--- ydb/trunk/src/Makefile 2009-03-18 11:40:56 UTC (rev 1305)
+++ ydb/trunk/src/Makefile 2009-03-18 19:25:42 UTC (rev 1306)
@@ -32,12 +32,12 @@
PPROF := -lprofiler
endif
ifneq ($(OPT),)
- OPT := -O3 -Wdisabled-optimization -DNDEBUG
+ OPT := -g3 -O3 -Wdisabled-optimization -DNDEBUG
else
OPT := -g3
endif
# CXX := $(WTF) ag++ -k --Xcompiler # $(CXX)
-CXX := $(WTF) $(CXX) -pipe
+CXX := $(WTF) ccache $(CXX) -pipe
LDFLAGS := -pthread $(GPROF)
LDLIBS := -lstx -lst -lresolv -lprotobuf -lgtest \
-lboost_program_options-gcc43-mt -lboost_thread-gcc43-mt \
@@ -94,7 +94,7 @@
all: $(TARGET)
$(TARGET): $(OBJS)
- $(LINK.o) $^ $(LOADLIBES) $(LDLIBS) -o $@
+ $(LINK.cc) $^ $(LOADLIBES) $(LDLIBS) -o $@
%.pb.o: %.pb.cc %.pb.h
$(CXX) -c $(PBCXXFLAGS) $(OUTPUT_OPTION) $<
@@ -137,6 +137,7 @@
clean:
rm -f $(GENSRCS) $(GENHDRS) $(OBJS) $(TARGET) main.lzz *.clamp_h
+ make -C tpcc/ clean
distclean: clean
rm -f all.h all.h.gch
Modified: ydb/trunk/src/main.lzz.clamp
===================================================================
--- ydb/trunk/src/main.lzz.clamp 2009-03-18 11:40:56 UTC (rev 1305)
+++ ydb/trunk/src/main.lzz.clamp 2009-03-18 19:25:42 UTC (rev 1306)
@@ -1130,7 +1130,6 @@
finally f(lambda () {
long long now = current_time_millis();
- stopped_issuing = true;
showtput("processed", now, __ref(start_time), __ref(seqno),
__ref(init_seqno));
if (!__ref(caught_up)) {
@@ -1738,9 +1737,8 @@
// Read the message, but correctly respond to interrupts so that we can
// cleanly exit (slightly tricky).
- if (stopped_issuing) {
- st_intr intr(stop_hub);
- readmsg(reader, res);
+ if (stopped_issuing && last_seqno + 1 == seqno) {
+ break;
} else {
st_intr intr(kill_hub);
readmsg(reader, res);
@@ -2712,6 +2710,7 @@
long long start_time = current_time_millis();
finally f(lambda () {
+ stopped_issuing = true;
showtput("issued", current_time_millis(), __ref(start_time), __ref(seqno),
0);
});
Modified: ydb/trunk/src/tpcc/Makefile
===================================================================
--- ydb/trunk/src/tpcc/Makefile 2009-03-18 11:40:56 UTC (rev 1305)
+++ ydb/trunk/src/tpcc/Makefile 2009-03-18 19:25:42 UTC (rev 1306)
@@ -1,12 +1,15 @@
WARNINGS = -Werror -Wall -Wextra -Wconversion -Wpointer-arith -Wcast-qual -Wcast-align -Wwrite-strings -Woverloaded-virtual -Wno-sign-compare -Wno-unused-parameter
+CXX := ccache $(CXX)
+
# Debug flags
-CXXFLAGS = -g -MD $(WARNINGS) -I.. -std=gnu++0x
-# Optimization flags
-#CXXFLAGS = -g -O3 -DNDEBUG -MD $(WARNINGS) -std=gnu++0x
+ifeq ($(OPT),)
+ CXXFLAGS = -g3 -MD $(WARNINGS) -std=gnu++0x
+else
+ CXXFLAGS = -g3 -O3 -DNDEBUG -MD $(WARNINGS) -std=gnu++0x
+endif
# Link withthe C++ standard library
-#LDFLAGS=-lstdc++
LDLIBS = -lgtest
BINARIES = btree_test.o tpccclient.o tpccgenerator.o tpcctables.o tpccdb.o clock.o randomgenerator.o
@@ -18,7 +21,7 @@
%.cc: %.cc.cog
cog.py $< > $@
-clean :
+clean:
rm -f *.o *.d $(BINARIES)
-include *.d
Modified: ydb/trunk/src/tpcc/btree.h
===================================================================
--- ydb/trunk/src/tpcc/btree.h 2009-03-18 11:40:56 UTC (rev 1305)
+++ ydb/trunk/src/tpcc/btree.h 2009-03-18 19:25:42 UTC (rev 1306)
@@ -280,7 +280,9 @@
private:
// Used when debugging
+#ifndef NDEBUG
enum NodeType {NODE_INNER=0xDEADBEEF, NODE_LEAF=0xC0FFEE};
+#endif
// Leaf nodes store pairs of keys and values.
struct LeafNode {
Modified: ydb/trunk/src/tpcc/tpccclient.cc
===================================================================
--- ydb/trunk/src/tpcc/tpccclient.cc 2009-03-18 11:40:56 UTC (rev 1305)
+++ ydb/trunk/src/tpcc/tpccclient.cc 2009-03-18 19:25:42 UTC (rev 1306)
@@ -68,7 +68,7 @@
vector<DeliveryOrderInfo> orders;
db_->delivery(generateWarehouse(), carrier, now, &orders);
-#ifdef NDEBUG
+#ifndef NDEBUG
if (orders.size() != District::NUM_PER_WAREHOUSE) {
printf("Only delivered from %zd districts\n", orders.size());
}
Modified: ydb/trunk/src/tpcc/tpccdb.h
===================================================================
--- ydb/trunk/src/tpcc/tpccdb.h 2009-03-18 11:40:56 UTC (rev 1305)
+++ ydb/trunk/src/tpcc/tpccdb.h 2009-03-18 19:25:42 UTC (rev 1306)
@@ -211,7 +211,7 @@
int32_t h_w_id;
float h_amount;
char h_date[DATETIME_SIZE+1];
- char h_data[MAX_DATA];
+ char h_data[MAX_DATA+1];
};
// Data returned by the "order status" transaction.
Modified: ydb/trunk/src/tpcc/tpcctables.cc.cog
===================================================================
--- ydb/trunk/src/tpcc/tpcctables.cc.cog 2009-03-18 11:40:56 UTC (rev 1305)
+++ ydb/trunk/src/tpcc/tpcctables.cc.cog 2009-03-18 19:25:42 UTC (rev 1306)
@@ -351,7 +351,7 @@
assert(neworder->no_d_id == d_id && neworder->no_w_id == warehouse_id);
int32_t o_id = neworder->no_o_id;
neworders_.erase(iterator);
- delete neworder;
+ if (!within(serbuf_, neworder)) delete neworder;
DeliveryOrderInfo order;
order.d_id = d_id;
Modified: ydb/trunk/tools/test.bash
===================================================================
--- ydb/trunk/tools/test.bash 2009-03-18 11:40:56 UTC (rev 1305)
+++ ydb/trunk/tools/test.bash 2009-03-18 19:25:42 UTC (rev 1306)
@@ -92,6 +92,37 @@
}
#
+# Access control
+#
+
+node-lock() {
+ check-remote
+ if ! node-islocked ; then
+ sudo su - -c '
+ echo AllowUsers root yang >> /etc/ssh/sshd_config &&
+ /etc/init.d/ssh restart
+ '
+ fi
+}
+
+node-unlock() {
+ check-remote
+ sudo su - -c '
+ sed -i "/^AllowUsers root yang$/ d" /etc/ssh/sshd_config &&
+ /etc/init.d/ssh restart
+ '
+}
+
+node-islocked() {
+ check-remote
+ sudo grep '^AllowUsers root yang$' /etc/ssh/sshd_config
+}
+
+lock() { parremote node-lock ; }
+unlock() { parremote node-unlock ; }
+islocked() { parremote node-islocked ; }
+
+#
# Setup
#
@@ -191,6 +222,11 @@
PPROF=1 OPT=1 make WTF= ydb
}
+node-setup-cog() {
+ check-remote
+ toast --quiet arm 'http://nedbatchelder.com/code/cog/cog-2.1.tar.gz'
+}
+
init-setup() {
parremote node-init-setup
}
@@ -227,6 +263,7 @@
parremote node-setup-clamp
parremote node-setup-gtest
parremote node-setup-ghash
+ parremote node-setup-cog
}
setup-ydb() {
@@ -289,17 +326,17 @@
rec-helper() {
local leader=$1
shift
- : ${seqno:=1000000} ${extraargs:=}
- tagssh $leader "ydb/src/ydb -l --exit-on-recovery --accept-joiner-seqno $seqno -n $(( $# - 1 )) $extraargs" &
+ : ${seqno:=100000} ${extraargs:=}
+ tagssh $leader "set -x; ydb/src/ydb --tpcc -l --exit-on-recovery --accept-joiner-seqno $seqno -n $(( $# - 1 )) $extraargs" &
sleep .1
# Run initial replicas.
while (( $# > 1 )) ; do
- tagssh $1 "ydb/src/ydb -H $leader" &
+ tagssh $1 "set -x; ydb/src/ydb --tpcc -H $leader" &
shift
done
sleep .1
# Run joiner.
- tagssh $1 "ydb/src/ydb -H $leader --yield-build-up --yield-catch-up $extraargs" &
+ tagssh $1 "set -x; ydb/src/ydb --tpcc -H $leader --yield-build-up --yield-catch-up $extraargs" &
if false ; then
if [[ ${wait2:-} ]]
then sleep $wait2
@@ -389,10 +426,10 @@
scaling-helper() {
local leader=$1
shift
- tagssh $leader "CPUPROFILE=ydb.prof ydb/src/ydb -q -l -n $# -X 1000000 ${extraargs:-}" &
+ tagssh $leader "CPUPROFILE=ydb.prof ydb/src/ydb -q -l -n $# -X 100000 --tpcc ${extraargs:-}" &
sleep .1
for rep in "$@"
- do tagssh $rep "CPUPROFILE=ydb.prof ydb/src/ydb -q -n $# -H $leader ${extraargs:-}" &
+ do tagssh $rep "CPUPROFILE=ydb.prof --args ydb/src/ydb -q -n $# -H $leader --tpcc ${extraargs:-}" &
done
wait
}
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <yan...@us...> - 2009-03-18 11:41:05
|
Revision: 1305
http://assorted.svn.sourceforge.net/assorted/?rev=1305&view=rev
Author: yangzhang
Date: 2009-03-18 11:40:56 +0000 (Wed, 18 Mar 2009)
Log Message:
-----------
- finished adding all tpcc txns
Modified Paths:
--------------
ydb/trunk/README
ydb/trunk/src/main.lzz.clamp
ydb/trunk/src/tpcc/tpccclient.cc
ydb/trunk/src/ydb.proto
Modified: ydb/trunk/README
===================================================================
--- ydb/trunk/README 2009-03-18 09:59:25 UTC (rev 1304)
+++ ydb/trunk/README 2009-03-18 11:40:56 UTC (rev 1305)
@@ -746,7 +746,10 @@
has caught up to the first seqno in the buffer
- DONE implement TPC-C network recovery
-- TODO full TPC-C txn workload
+- DONE full TPC-C txn workload
+ - readonly: order_status, stock_level
+ - overwhelming majority are payment/neworder
+- TODO skip processing readonly txns on catch-up
- TODO prelim measurements on cluster
- TODO PAPER!!!
Modified: ydb/trunk/src/main.lzz.clamp
===================================================================
--- ydb/trunk/src/main.lzz.clamp 2009-03-18 09:59:25 UTC (rev 1304)
+++ ydb/trunk/src/main.lzz.clamp 2009-03-18 11:40:56 UTC (rev 1305)
@@ -908,11 +908,107 @@
unique_ptr<TPCCTables> g_tables;
void
+mkres(TpccRes *res, const OrderStatusOutput &output)
+{
+ OrderStatusOutputMsg &msg = *res->mutable_order_status();
+ msg.set_c_id(output.c_id);
+ msg.set_c_balance(output.c_balance);
+ msg.set_o_id(output.o_id);
+ msg.set_o_carrier_id(output.o_carrier_id);
+ foreach (const OrderStatusOutput::OrderLineSubset &src, output.lines) {
+ OrderLineSubsetMsg &dst = *msg.add_line();
+ dst.set_ol_i_id(src.ol_i_id);
+ dst.set_ol_supply_w_id(src.ol_supply_w_id);
+ dst.set_ol_quantity(src.ol_quantity);
+ dst.set_ol_amount(src.ol_amount);
+ dst.set_ol_delivery_d(src.ol_delivery_d);
+ }
+ msg.set_c_first(output.c_first);
+ msg.set_c_middle(output.c_middle);
+ msg.set_c_last(output.c_last);
+ msg.set_o_entry_d(output.o_entry_d);
+}
+
+void
+mkres(TpccRes *res, const PaymentOutput &output)
+{
+ PaymentOutputMsg &msg = *res->mutable_payment();
+
+ WarehouseMsg &w = *msg.mutable_warehouse();
+ w.set_w_id(output.warehouse.w_id);
+ w.set_w_tax(output.warehouse.w_tax);
+ w.set_w_ytd(output.warehouse.w_ytd);
+ w.set_w_name(output.warehouse.w_name);
+ w.set_w_street_1(output.warehouse.w_street_1);
+ w.set_w_street_2(output.warehouse.w_street_2);
+ w.set_w_city(output.warehouse.w_city);
+ w.set_w_state(output.warehouse.w_state);
+ w.set_w_zip(output.warehouse.w_zip);
+
+ DistrictMsg &d = *msg.mutable_district();
+ d.set_d_id(output.district.d_id);
+ d.set_d_w_id(output.district.d_w_id);
+ d.set_d_tax(output.district.d_tax);
+ d.set_d_ytd(output.district.d_ytd);
+ d.set_d_next_o_id(output.district.d_next_o_id);
+ d.set_d_name(output.district.d_name);
+ d.set_d_street_1(output.district.d_street_1);
+ d.set_d_street_2(output.district.d_street_2);
+ d.set_d_city(output.district.d_city);
+ d.set_d_state(output.district.d_state);
+ d.set_d_zip(output.district.d_zip);
+
+ CustomerMsg &c = *msg.mutable_customer();
+ c.set_c_id(output.customer.c_id);
+ c.set_c_d_id(output.customer.c_d_id);
+ c.set_c_w_id(output.customer.c_w_id);
+ c.set_c_credit_lim(output.customer.c_credit_lim);
+ c.set_c_discount(output.customer.c_discount);
+ c.set_c_balance(output.customer.c_balance);
+ c.set_c_ytd_payment(output.customer.c_ytd_payment);
+ c.set_c_payment_cnt(output.customer.c_payment_cnt);
+ c.set_c_delivery_cnt(output.customer.c_delivery_cnt);
+ c.set_c_first(output.customer.c_first);
+ c.set_c_middle(output.customer.c_middle);
+ c.set_c_last(output.customer.c_last);
+ c.set_c_street_1(output.customer.c_street_1);
+ c.set_c_street_2(output.customer.c_street_2);
+ c.set_c_city(output.customer.c_city);
+ c.set_c_state(output.customer.c_state);
+ c.set_c_zip(output.customer.c_zip);
+ c.set_c_phone(output.customer.c_phone);
+ c.set_c_since(output.customer.c_since);
+ c.set_c_credit(output.customer.c_credit);
+ c.set_c_data(output.customer.c_data);
+}
+
+void
process_tpcc(const TpccReq &req, int &seqno, TpccRes *res)
{
checkeq(req.seqno(), seqno + 1);
++seqno;
- if (req.has_new_order()) {
+ if (res != nullptr) {
+ res->Clear();
+ res->set_seqno(seqno);
+ }
+ if (req.has_stock_level()) {
+ const StockLevelMsg &sl = req.stock_level();
+ int result = g_tables->stockLevel(sl.warehouse_id(), sl.district_id(), sl.threshold());
+ if (res != nullptr) {
+ StockLevelOutputMsg &msg = *res->mutable_stock_level();
+ msg.set_result(result);
+ }
+ } else if (req.has_order_status_1()) {
+ const OrderStatusMsg1 &os = req.order_status_1();
+ OrderStatusOutput output;
+ g_tables->orderStatus(os.warehouse_id(), os.district_id(), os.customer_id(), &output);
+ if (res != nullptr) mkres(res, output);
+ } else if (req.has_order_status_2()) {
+ const OrderStatusMsg2 &os = req.order_status_2();
+ OrderStatusOutput output;
+ g_tables->orderStatus(os.warehouse_id(), os.district_id(), os.c_last().c_str(), &output);
+ if (res != nullptr) mkres(res, output);
+ } else if (req.has_new_order()) {
const NewOrderMsg &no = req.new_order();
vector<NewOrderItem> items(no.item_size());
for (int i = 0; i < no.item_size(); ++i) {
@@ -927,28 +1023,52 @@
no.customer_id(), items, no.now().c_str(),
&output);
if (res != nullptr) {
- res->Clear();
- res->set_seqno(seqno);
- NewOrderOutputMsg &outmsg = *res->mutable_new_order();
- outmsg.set_w_tax(output.w_tax);
- outmsg.set_d_tax(output.d_tax);
- outmsg.set_o_id(output.o_id);
- outmsg.set_c_discount(output.c_discount);
- outmsg.set_total(output.total);
+ NewOrderOutputMsg &msg = *res->mutable_new_order();
+ msg.set_w_tax(output.w_tax);
+ msg.set_d_tax(output.d_tax);
+ msg.set_o_id(output.o_id);
+ msg.set_c_discount(output.c_discount);
+ msg.set_total(output.total);
foreach (const NewOrderOutput::ItemInfo &src, output.items) {
- ItemInfoMsg &dst = *outmsg.add_item();
+ ItemInfoMsg &dst = *msg.add_item();
dst.set_s_quantity(src.s_quantity);
dst.set_i_price(src.i_price);
dst.set_ol_amount(src.ol_amount);
dst.set_brand_generic(src.brand_generic);
dst.set_i_name(src.i_name);
}
- outmsg.set_c_last(output.c_last);
- outmsg.set_c_credit(output.c_credit);
- outmsg.set_status(output.status);
+ msg.set_c_last(output.c_last);
+ msg.set_c_credit(output.c_credit);
+ msg.set_status(output.status);
}
+ } else if (req.has_payment_1()) {
+ const PaymentMsg1 &p = req.payment_1();
+ PaymentOutput output;
+ g_tables->payment(p.warehouse_id(), p.district_id(), p.c_warehouse_id(),
+ p.c_district_id(), p.customer_id(), p.h_amount(),
+ p.now().c_str(), &output);
+ if (res != nullptr) mkres(res, output);
+ } else if (req.has_payment_2()) {
+ const PaymentMsg2 &p = req.payment_2();
+ PaymentOutput output;
+ g_tables->payment(p.warehouse_id(), p.district_id(), p.c_warehouse_id(),
+ p.c_district_id(), p.c_last().c_str(), p.h_amount(),
+ p.now().c_str(), &output);
+ if (res != nullptr) mkres(res, output);
+ } else if (req.has_delivery()) {
+ const DeliveryMsg &d = req.delivery();
+ vector<DeliveryOrderInfo> orders;
+ g_tables->delivery(d.warehouse_id(), d.carrier_id(), d.now().c_str(), &orders);
+ if (res != nullptr) {
+ DeliveryOutputMsg &msg = *res->mutable_delivery();
+ foreach (const DeliveryOrderInfo &src, orders) {
+ DeliveryOrderInfoMsg &dst = *msg.add_order();
+ dst.set_d_id(src.d_id);
+ dst.set_o_id(src.o_id);
+ }
+ }
} else {
- throw_not_implemented();
+ ASSERT(false);
}
}
@@ -2456,24 +2576,48 @@
// Executes the TPC-C "slev" transaction. From the last 20 orders, returns the number of rows in
// the STOCK table that have S_QUANTITY < threshold. See TPC-C 2.8 (page 43).
int stockLevel(int32_t warehouse_id, int32_t district_id, int32_t threshold) {
- warehouse_id = district_id = threshold = 0;
- throw_not_implemented();
+ req_.Clear();
+ req_.set_seqno(seqno_);
+
+ StockLevelMsg &sl = *req_.mutable_stock_level();
+ sl.set_warehouse_id(warehouse_id);
+ sl.set_district_id(district_id);
+ sl.set_threshold(threshold);
+
+ ser(writer_, req_);
+ return 0;
}
// Executes the TPC-C order status transaction. Find the customer's last order and check the
// delivery date of each item on the order. See TPC-C 2.6 (page 36).
void orderStatus(int32_t warehouse_id, int32_t district_id, int32_t customer_id,
OrderStatusOutput* output) {
- warehouse_id = district_id = customer_id = 0; output = 0;
- throw_not_implemented();
+ req_.Clear();
+ req_.set_seqno(seqno_);
+
+ OrderStatusMsg1 &os = *req_.mutable_order_status_1();
+ os.set_warehouse_id(warehouse_id);
+ os.set_district_id(district_id);
+ os.set_customer_id(customer_id);
+
+ ser(writer_, req_);
+ output = nullptr;
}
// Executes the TPC-C order status transaction. Find the customer's last order and check the
// delivery date of each item on the order. See TPC-C 2.6 (page 36).
void orderStatus(int32_t warehouse_id, int32_t district_id, const char* c_last,
OrderStatusOutput* output) {
- warehouse_id = district_id = 0; c_last = 0; output = 0;
- throw_not_implemented();
+ req_.Clear();
+ req_.set_seqno(seqno_);
+
+ OrderStatusMsg2 &os = *req_.mutable_order_status_2();
+ os.set_warehouse_id(warehouse_id);
+ os.set_district_id(district_id);
+ os.set_c_last(c_last);
+
+ ser(writer_, req_);
+ output = nullptr;
}
// Executes the TPC-C new order transaction. Enter the new order for customer_id into the
@@ -2506,9 +2650,20 @@
void payment(int32_t warehouse_id, int32_t district_id, int32_t c_warehouse_id,
int32_t c_district_id, int32_t customer_id, float h_amount, const char* now,
PaymentOutput* output) {
- warehouse_id = district_id = c_district_id = c_warehouse_id = customer_id = 0;
- h_amount = 0; now = 0; output = 0;
- throw_not_implemented();
+ req_.Clear();
+ req_.set_seqno(seqno_);
+
+ PaymentMsg1 &p = *req_.mutable_payment_1();
+ p.set_warehouse_id(warehouse_id);
+ p.set_district_id(district_id);
+ p.set_c_warehouse_id(c_warehouse_id);
+ p.set_c_district_id(c_district_id);
+ p.set_customer_id(customer_id);
+ p.set_h_amount(h_amount);
+ p.set_now(now);
+
+ ser(writer_, req_);
+ output = nullptr;
}
// Executes the TPC-C payment transaction. Add h_amount to the customer's account.
@@ -2516,17 +2671,36 @@
void payment(int32_t warehouse_id, int32_t district_id, int32_t c_warehouse_id,
int32_t c_district_id, const char* c_last, float h_amount, const char* now,
PaymentOutput* output) {
- warehouse_id = district_id = c_warehouse_id = c_district_id = 0;
- h_amount = 0; c_last = now = 0; output = 0;
- throw_not_implemented();
+ req_.Clear();
+ req_.set_seqno(seqno_);
+
+ PaymentMsg2 &p = *req_.mutable_payment_2();
+ p.set_warehouse_id(warehouse_id);
+ p.set_district_id(district_id);
+ p.set_c_warehouse_id(c_warehouse_id);
+ p.set_c_district_id(c_district_id);
+ p.set_c_last(c_last);
+ p.set_h_amount(h_amount);
+ p.set_now(now);
+
+ ser(writer_, req_);
+ output = nullptr;
}
// Executes the TPC-C delivery transaction. Delivers the oldest undelivered transaction in each
// district in warehouse_id. See TPC-C 2.7 (page 39).
void delivery(int32_t warehouse_id, int32_t carrier_id, const char* now,
vector<DeliveryOrderInfo>* orders) {
- warehouse_id = carrier_id = 0; now = 0; orders = 0;
- throw_not_implemented();
+ req_.Clear();
+ req_.set_seqno(seqno_);
+
+ DeliveryMsg &d = *req_.mutable_delivery();
+ d.set_warehouse_id(warehouse_id);
+ d.set_carrier_id(carrier_id);
+ d.set_now(now);
+
+ ser(writer_, req_);
+ orders = nullptr;
}
};
Modified: ydb/trunk/src/tpcc/tpccclient.cc
===================================================================
--- ydb/trunk/src/tpcc/tpccclient.cc 2009-03-18 09:59:25 UTC (rev 1304)
+++ ydb/trunk/src/tpcc/tpccclient.cc 2009-03-18 11:40:56 UTC (rev 1305)
@@ -68,9 +68,11 @@
vector<DeliveryOrderInfo> orders;
db_->delivery(generateWarehouse(), carrier, now, &orders);
+#ifdef NDEBUG
if (orders.size() != District::NUM_PER_WAREHOUSE) {
printf("Only delivered from %zd districts\n", orders.size());
}
+#endif
}
void TPCCClient::doPayment() {
@@ -144,22 +146,24 @@
// This is not strictly accurate: The requirement is for certain *minimum* percentages to be
// maintained. This is close to the right thing, but not precisely correct.
// See TPC-C 5.2.4 (page 68).
- doNewOrder();
-#if 0
int x = generator_->number(1, 100);
if (x <= 4) { // 4%
+ //printf("stocklevel\n");
doStockLevel();
} else if (x <= 8) { // 4%
+ //printf("delivery\n");
doDelivery();
} else if (x <= 12) { // 4%
+ //printf("orderstatus\n");
doOrderStatus();
} else if (x <= 12+43) { // 43%
+ //printf("payment\n");
doPayment();
} else { // 45%
+ //printf("neworder\n");
ASSERT(x > 100-45);
doNewOrder();
}
-#endif
}
int32_t TPCCClient::generateWarehouse() {
Modified: ydb/trunk/src/ydb.proto
===================================================================
--- ydb/trunk/src/ydb.proto 2009-03-18 09:59:25 UTC (rev 1304)
+++ ydb/trunk/src/ydb.proto 2009-03-18 11:40:56 UTC (rev 1305)
@@ -87,9 +87,27 @@
}
//
-// TPCC messages
+// TPCC request messages
//
+message StockLevelMsg {
+ required int32 warehouse_id = 1;
+ required int32 district_id = 2;
+ required int32 threshold = 3;
+}
+
+message OrderStatusMsg1 {
+ required int32 warehouse_id = 1;
+ required int32 district_id = 2;
+ required int32 customer_id = 3;
+}
+
+message OrderStatusMsg2 {
+ required int32 warehouse_id = 1;
+ required int32 district_id = 2;
+ required string c_last = 3;
+}
+
message NewOrderItemMsg {
required int32 i_id = 1;
required int32 ol_supply_w_id = 2;
@@ -104,6 +122,71 @@
required string now = 5;
}
+message PaymentMsg1 {
+ required int32 warehouse_id = 1;
+ required int32 district_id = 2;
+ required int32 c_warehouse_id = 3;
+ required int32 c_district_id = 4;
+ required int32 customer_id = 5;
+ required float h_amount = 6;
+ required string now = 7;
+}
+
+message PaymentMsg2 {
+ required int32 warehouse_id = 1;
+ required int32 district_id = 2;
+ required int32 c_warehouse_id = 3;
+ required int32 c_district_id = 4;
+ required string c_last = 5;
+ required float h_amount = 6;
+ required string now = 7;
+}
+
+message DeliveryMsg {
+ required int32 warehouse_id = 1;
+ required int32 carrier_id = 2;
+ required string now = 3;
+}
+
+message TpccReq {
+ required int32 seqno = 1;
+ optional StockLevelMsg stock_level = 2;
+ optional OrderStatusMsg1 order_status_1 = 3;
+ optional OrderStatusMsg2 order_status_2 = 4;
+ optional NewOrderMsg new_order = 5;
+ optional PaymentMsg1 payment_1 = 6;
+ optional PaymentMsg2 payment_2 = 7;
+ optional DeliveryMsg delivery = 8;
+}
+
+//
+// TPCC response messages.
+//
+
+message StockLevelOutputMsg {
+ required int32 result = 1;
+}
+
+message OrderLineSubsetMsg {
+ required int32 ol_i_id = 1;
+ required int32 ol_supply_w_id = 2;
+ required int32 ol_quantity = 3;
+ required float ol_amount = 4;
+ required string ol_delivery_d = 5;
+}
+
+message OrderStatusOutputMsg {
+ required int32 c_id = 1;
+ required float c_balance = 2;
+ required int32 o_id = 3;
+ required int32 o_carrier_id = 4;
+ repeated OrderLineSubsetMsg line = 5;
+ required string c_first = 6;
+ required string c_middle = 7;
+ required string c_last = 8;
+ required string o_entry_d = 9;
+}
+
message ItemInfoMsg {
required int32 s_quantity = 1;
required float i_price = 2;
@@ -136,12 +219,76 @@
required string status = 9;
}
-message TpccReq {
- required int32 seqno = 1;
- optional NewOrderMsg new_order = 2;
+message WarehouseMsg {
+ required int32 w_id = 1;
+ required float w_tax = 2;
+ required float w_ytd = 3;
+ required string w_name = 4;
+ required string w_street_1 = 5;
+ required string w_street_2 = 6;
+ required string w_city = 7;
+ required string w_state = 8;
+ required string w_zip = 9;
}
+message DistrictMsg {
+ required int32 d_id = 1;
+ required int32 d_w_id = 2;
+ required float d_tax = 3;
+ required float d_ytd = 4;
+ required int32 d_next_o_id = 5;
+ required string d_name = 6;
+ required string d_street_1 = 7;
+ required string d_street_2 = 8;
+ required string d_city = 9;
+ required string d_state = 10;
+ required string d_zip = 11;
+}
+
+message CustomerMsg {
+ required int32 c_id = 1;
+ required int32 c_d_id = 2;
+ required int32 c_w_id = 3;
+ required float c_credit_lim = 4;
+ required float c_discount = 5;
+ required float c_balance = 6;
+ required float c_ytd_payment = 7;
+ required int32 c_payment_cnt = 8;
+ required int32 c_delivery_cnt = 9;
+ required string c_first = 10;
+ required string c_middle = 11;
+ required string c_last = 12;
+ required string c_street_1 = 13;
+ required string c_street_2 = 14;
+ required string c_city = 15;
+ required string c_state = 16;
+ required string c_zip = 17;
+ required string c_phone = 18;
+ required string c_since = 19;
+ required string c_credit = 20;
+ required string c_data = 21;
+}
+
+message PaymentOutputMsg {
+ required WarehouseMsg warehouse = 1;
+ required DistrictMsg district = 2;
+ required CustomerMsg customer = 3;
+}
+
+message DeliveryOrderInfoMsg {
+ required int32 d_id = 1;
+ required int32 o_id = 2;
+}
+
+message DeliveryOutputMsg {
+ repeated DeliveryOrderInfoMsg order = 1;
+}
+
message TpccRes {
required int32 seqno = 1;
- optional NewOrderOutputMsg new_order = 2;
+ optional StockLevelOutputMsg stock_level = 2;
+ optional OrderStatusOutputMsg order_status = 3;
+ optional NewOrderOutputMsg new_order = 4;
+ optional PaymentOutputMsg payment = 5;
+ optional DeliveryOutputMsg delivery = 6;
}
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <yan...@us...> - 2009-03-18 09:59:32
|
Revision: 1304
http://assorted.svn.sourceforge.net/assorted/?rev=1304&view=rev
Author: yangzhang
Date: 2009-03-18 09:59:25 +0000 (Wed, 18 Mar 2009)
Log Message:
-----------
cleaned up macros, cog
Modified Paths:
--------------
ydb/trunk/src/ser.h
ydb/trunk/src/tpcc/tpcctables.cc.cog
Modified: ydb/trunk/src/ser.h
===================================================================
--- ydb/trunk/src/ser.h 2009-03-18 09:58:41 UTC (rev 1303)
+++ ydb/trunk/src/ser.h 2009-03-18 09:59:25 UTC (rev 1304)
@@ -10,9 +10,6 @@
#include <iostream>
#include "ydb.pb.h"
-#define BEGIN_NAMESPACE(ns) namespace ns {
-#define END_NAMESPACE }
-
#define MAKE_START_FIN_HELPER(MsgType, field, action) \
template<typename T> inline void action##_##field(T &msg); \
template<> inline void action##_##field(ydb::pb::MsgType&) {} \
Modified: ydb/trunk/src/tpcc/tpcctables.cc.cog
===================================================================
--- ydb/trunk/src/tpcc/tpcctables.cc.cog 2009-03-18 09:58:41 UTC (rev 1303)
+++ ydb/trunk/src/tpcc/tpcctables.cc.cog 2009-03-18 09:59:25 UTC (rev 1304)
@@ -1,3 +1,13 @@
+//[[[cog
+// allfields = 'items warehouses stock districts customers orders orderlines neworders history'.split()
+// treepairs = 'warehouses/Warehouse stock/Stock districts/District customers/Customer orders/Order orderlines/OrderLine'.split()
+// allpairs = treepairs + ['neworders/NewOrder']
+// def typedefs():
+// for name in allfields:
+// cog.outl(r'typedef typeof(%s_) type_%s;' % (name, name))
+//]]]
+//[[[end]]]
+
#include "tpcctables.h"
#include <algorithm>
@@ -605,7 +615,6 @@
using namespace std;
cout
//[[[cog
- // import cog
// for name in 'items warehouses stock districts customers orders orders_by_customer orderlines customers_by_name neworders history'.split():
// cog.outl(r'<< " |%s| = " << %s_.size() << "\n"' % (name, name))
//]]]
@@ -625,11 +634,10 @@
bzero(&hdr, sizeof hdr);
hdr.seqno = seqno;
//[[[cog
- // import cog
- // for name in 'items warehouses stock districts customers orders orderlines neworders history'.split():
- // cog.outl(r'typedef typeof(%s_) type_%s;' % (name, name))
+ // typedefs()
+ // for name in allfields:
// cog.outl(r'hdr.n%s = uint32_t(%s_.size());' % (name, name))
- // for pair in 'warehouses/Warehouse stock/Stock districts/District customers/Customer orders/Order orderlines/OrderLine neworders/NewOrder'.split():
+ // for pair in allpairs:
// name, struct = pair.split('/')
// cog.outl(r'hdr.len += uint32_t(hdr.n%s * (sizeof(type_%s::key_type) + sizeof(%s)));' % (name, name, struct))
//]]]
@@ -651,7 +659,7 @@
}
//[[[cog
- // for pair in 'warehouses/Warehouse stock/Stock districts/District customers/Customer orders/Order orderlines/OrderLine'.split():
+ // for pair in treepairs:
// name, struct = pair.split('/')
// cog.outl(r'''
// {
@@ -691,19 +699,14 @@
raw_reader reader(arr.get());
- //[[[cog
- // for name in 'items warehouses stock districts customers orders orderlines neworders history'.split():
- // cog.outl(r'typedef typeof(%s_) type_%s;' % (name, name))
- //]]]
- //[[[end]]]
-
items_.reserve(hdr.nitems);
for (uint32_t i = 0; i < hdr.nitems; ++i) {
items_.push_back(reader.read<Item>());
}
//[[[cog
- // for pair in 'warehouses/Warehouse stock/Stock districts/District customers/Customer orders/Order orderlines/OrderLine'.split():
+ // typedefs()
+ // for pair in treepairs:
// name, struct = pair.split('/')
// # Generate customers_by_name_
// cbn = r'customers_by_name_.insert(reinterpret_cast<Customer*>(reader.ptr()));' if name == 'customers' else ''
@@ -730,7 +733,7 @@
history_.reserve(hdr.nhistory);
for (uint32_t i = 0; i < hdr.nhistory; ++i) {
- history_.push_back(reader.readptr<const History>()); // reinterpret_cast<const History*>(reader.ptr()));
+ history_.push_back(reader.readptr<const History>());
}
serbuf_.reset(arr.get(), arr.size());
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <yan...@us...> - 2009-03-18 09:58:55
|
Revision: 1303
http://assorted.svn.sourceforge.net/assorted/?rev=1303&view=rev
Author: yangzhang
Date: 2009-03-18 09:58:41 +0000 (Wed, 18 Mar 2009)
Log Message:
-----------
- added anchored_stream_reader and associated functions
- added default ctor for sized_array
- added within
- tweaks to stream_writer
- simplified and error-check current_time_millis()
- added unused warning for static functions die, _vcheck, _check, swap
- added BEGIN_NAMESPACE, END_NAMESPACE
- added readptr, ptr, skip to raw_reader, raw_writer
- added todos to readme
Modified Paths:
--------------
cpp-commons/trunk/README
cpp-commons/trunk/src/commons/algo.h
cpp-commons/trunk/src/commons/array.h
cpp-commons/trunk/src/commons/check.h
cpp-commons/trunk/src/commons/die.h
cpp-commons/trunk/src/commons/memory.h
cpp-commons/trunk/src/commons/streamreader.h
cpp-commons/trunk/src/commons/streamwriter.h
cpp-commons/trunk/src/commons/time.h
cpp-commons/trunk/src/commons/utility.h
Modified: cpp-commons/trunk/README
===================================================================
--- cpp-commons/trunk/README 2009-03-18 09:36:57 UTC (rev 1302)
+++ cpp-commons/trunk/README 2009-03-18 09:58:41 UTC (rev 1303)
@@ -104,3 +104,8 @@
[GNU Common C++]: http://www.gnu.org/software/commoncpp/
[dlib]: http://dclib.sourceforge.net/
[FC++]: http://www.cc.gatech.edu/~yannis/fc++/
+
+Todo
+----
+
+- Get to the bottom of statics, inlines, unused
Modified: cpp-commons/trunk/src/commons/algo.h
===================================================================
--- cpp-commons/trunk/src/commons/algo.h 2009-03-18 09:36:57 UTC (rev 1302)
+++ cpp-commons/trunk/src/commons/algo.h 2009-03-18 09:58:41 UTC (rev 1303)
@@ -5,7 +5,8 @@
namespace commons {
- void swap(size_t &x, size_t &y) {
+ __attribute__((unused)) inline void
+ swap(size_t &x, size_t &y) {
x = x ^ y;
y = x ^ y;
x = x ^ y;
Modified: cpp-commons/trunk/src/commons/array.h
===================================================================
--- cpp-commons/trunk/src/commons/array.h 2009-03-18 09:36:57 UTC (rev 1302)
+++ cpp-commons/trunk/src/commons/array.h 2009-03-18 09:58:41 UTC (rev 1303)
@@ -27,6 +27,7 @@
friend void swap<>(sized_array<T> &a, sized_array<T> &b);
public:
explicit sized_array(char *p, size_t n) : p_(p), n_(n) {}
+ sized_array() : p_(nullptr), n_(0) {}
size_t size() const { return n_; }
T *get() const { return p_; }
T *begin() const { return p_; }
@@ -125,6 +126,14 @@
bool scoped_;
};
+ /**
+ * Checks if a pointer is in an array.
+ */
+ template<typename T>
+ inline bool within(T &a, const void *p) {
+ return a.begin() <= p && p < a.end();
+ }
+
}
#endif
Modified: cpp-commons/trunk/src/commons/check.h
===================================================================
--- cpp-commons/trunk/src/commons/check.h 2009-03-18 09:36:57 UTC (rev 1302)
+++ cpp-commons/trunk/src/commons/check.h 2009-03-18 09:58:41 UTC (rev 1303)
@@ -47,7 +47,7 @@
const string name;
};
- __attribute__((format(printf, 4, 0))) inline void
+ __attribute__((format(printf, 4, 0))) inline static void
_vcheck(bool cond, const char *file, int line, const char *fmt, va_list ap)
{
if (!cond) {
@@ -62,7 +62,7 @@
}
}
- __attribute__((format(printf, 4, 5))) void
+ __attribute__((format(printf, 4, 5))) static void
_check(bool cond, const char *file, int line, const char *fmt, ...)
{
va_list ap;
@@ -71,7 +71,7 @@
va_end(ap);
}
- inline void
+ inline static void
_check(bool cond, const char *file, int line)
{
_check(cond, file, line, NULL);
Modified: cpp-commons/trunk/src/commons/die.h
===================================================================
--- cpp-commons/trunk/src/commons/die.h 2009-03-18 09:36:57 UTC (rev 1302)
+++ cpp-commons/trunk/src/commons/die.h 2009-03-18 09:58:41 UTC (rev 1303)
@@ -15,7 +15,7 @@
*
* TODO: move into C Commons.
*/
- __attribute__((format(printf, 1, 2))) void
+ __attribute__((format(printf, 1, 2),unused)) static void
die(const char *format, ...)
{
const char *errstr;
Modified: cpp-commons/trunk/src/commons/memory.h
===================================================================
--- cpp-commons/trunk/src/commons/memory.h 2009-03-18 09:36:57 UTC (rev 1302)
+++ cpp-commons/trunk/src/commons/memory.h 2009-03-18 09:58:41 UTC (rev 1303)
@@ -38,6 +38,8 @@
void write(const T &x) { memput(p_, x); p_ += sizeof(T); }
/** Get the current value of the pointer. */
void *ptr() const { return p_; }
+ /** Skip n bytes. */
+ void skip(size_t n) { p_ += n; }
};
class raw_reader
@@ -49,7 +51,12 @@
template<typename T>
void read(T& x) { memget(p_, x); p_ += sizeof(T); }
template<typename T>
- T &read() { void *p = p_; p_ += sizeof(T); return memget<T>(p_); }
+ T &read() { void *p = p_; p_ += sizeof(T); return memget<T>(p); }
+ template<typename T>
+ T *readptr() { return reinterpret_cast<T*>(readptr(sizeof(T))); }
+ void *readptr(size_t n) { void *p = p_; p_ += n; return p; }
+ void *ptr() const { return p_; }
+ void skip(size_t n) { p_ += n; }
};
}
Modified: cpp-commons/trunk/src/commons/streamreader.h
===================================================================
--- cpp-commons/trunk/src/commons/streamreader.h 2009-03-18 09:36:57 UTC (rev 1302)
+++ cpp-commons/trunk/src/commons/streamreader.h 2009-03-18 09:58:41 UTC (rev 1303)
@@ -264,6 +264,252 @@
char *anchor_;
};
+ /**
+ * General-purpose stream reader for arbitrary data source streams, plus the
+ * ability to hold on to (don't discard) parts of the stream.
+ *
+ * TODO actually don't need anchor!!! factor that out and rename, and adjust
+ * the external functions accordingly. also look into replacing the
+ * pointer-ref-returning functions with just reset() method(s) (overload this
+ * to replace/not replace the buffer as well).
+ */
+ class anchored_stream_reader
+ {
+ NONCOPYABLE(anchored_stream_reader)
+ public:
+ anchored_stream_reader(boost::function<size_t(char*, size_t)> reader,
+ boost::function<void(anchored_stream_reader&)> overflow,
+ char *buf, size_t bufsize) :
+ reader_(reader),
+ full_reader_(repeat_reader(reader_)),
+ overflow_(overflow),
+ buf_(buf, bufsize),
+ start_(buf),
+ end_(buf),
+ anchor_(buf)
+ {}
+
+ anchored_stream_reader(boost::function<size_t(char*, size_t)> reader,
+ boost::function<void(char*, size_t)> full_reader,
+ boost::function<void(anchored_stream_reader&)> overflow,
+ char *buf, size_t bufsize) :
+ reader_(reader),
+ full_reader_(full_reader),
+ overflow_(overflow),
+ buf_(buf, bufsize),
+ start_(buf),
+ end_(buf),
+ anchor_(buf)
+ {}
+
+ /**
+ * The size of the unconsumed range of bytes.
+ */
+ size_t unread() { return end_ - start_; }
+ size_t anchored() { return end_ - anchor_; }
+
+ /**
+ * The remaining number of bytes in the buffer
+ */
+ size_t rem() { return buf_.end() - end_; }
+
+ /**
+ * The entire read buffer.
+ */
+ sized_array<char> &buf() { return buf_; }
+
+ void set_anchor() { anchor_ = start_; }
+
+ /**
+ * Reset the pointers, emptying the buffer.
+ */
+ void reset() {
+ start_ = end_ = anchor_ = buf_.get();
+ }
+
+ char *start() const { return start_; }
+ char *end() const { return end_; }
+ char *anchor() const { return anchor_; }
+ char *&start() { return start_; }
+ char *&end() { return end_; }
+ char *&anchor() { return anchor_; }
+
+#if 0
+ /**
+ * Discard the requested number of bytes. Disregards anchor.
+ */
+ void skip(size_t req) {
+ if (unread() >= req) {
+ // We have more unconsumed bytes than requested, so we're done.
+ start_ += req;
+ return;
+ }
+
+ // We have more requested bytes than unconsumed, so need to keep
+ // reading. Skip over bytes that are immediately available...
+ req -= unread();
+ // ...and reset pointers to discard current buffer.
+ reset();
+
+ // Keep reading until we have enough.
+ while (true) {
+ size_t res = reader_(end_, rem());
+ if (res == 0) throw eof_exception();
+ if (res == req) break;
+ if (res > req) {
+ // Now skip over the rest of the bytes, which weren't available.
+ start_ += req;
+ end_ += res;
+ break;
+ }
+ req -= res;
+ }
+ }
+#endif
+
+ /**
+ * Read req bytes; if we hit an error or EOF, then an exception is
+ * thrown. Returns old value of start() normally (this call then
+ * increments start()), or returns nullptr if req too large for buffer.
+ */
+ char *read(size_t req) {
+ // Do we need to perform a read (or do we already
+ // have requested data)?
+ if (unread() < req) {
+ // Do we have enough space?
+ if (req > buf_.size()) return nullptr;
+
+ // Shift things down if necessary.
+ if (req > rem()) overflow_(*this);
+ ASSERT(req <= rem());
+
+ // Keep reading until we have enough.
+ while (unread() < req) {
+ size_t res = reader_(end_, rem());
+ if (res == 0) break;
+ else end_ += res;
+ }
+
+ // If we got a premature EOF.
+ if (unread() < req)
+ throw eof_exception();
+ }
+
+ char *ret = start_;
+ start_ += req;
+ return ret;
+ }
+
+ template<typename T>
+ T read()
+ {
+ size_t req = sizeof(T);
+
+ // Do we need to perform a read (or do we already
+ // have the requested data)?
+ if (unread() < req) {
+ ASSERT(req <= buf_.size());
+
+ // Shift things down if necessary.
+ if (req > rem()) overflow_(*this);
+ ASSERT(req <= rem());
+
+ // Keep reading until we have enough.
+ while (unread() < req) {
+ size_t res = reader_(end_, rem());
+ if (res == 0) break;
+ else end_ += res;
+ }
+
+ // If we got a premature EOF.
+ if (unread() < req)
+ throw eof_exception();
+ }
+
+ T x = *reinterpret_cast<const T*>(start_);
+ start_ += req;
+ return x;
+ }
+
+#if 0
+ /**
+ * Shift the unread bytes down to the start of the buffer.
+ */
+ void shift() {
+ memmove(buf_.get(), start_, unread());
+ size_t diff = start_ - buf_.get();
+ start_ -= diff;
+ end_ -= diff;
+ }
+#endif
+
+ private:
+ boost::function<size_t(char*, size_t)> reader_;
+
+ boost::function<void(char*, size_t)> full_reader_;
+
+ boost::function<void(anchored_stream_reader&)> overflow_;
+
+ /**
+ * The temporary storage buffer.
+ */
+ sized_array<char> buf_;
+
+ /**
+ * The start of the unconsumed range of bytes.
+ */
+ char *start_;
+
+ /**
+ * The end of the unconsumed range of bytes.
+ */
+ char *end_;
+
+ /**
+ * Do not discard bytes >= this.
+ */
+ char *anchor_;
+ };
+
+ template<typename T>
+ inline void
+ reset_reader(T &reader, char *anchor, char *start, char *end) {
+ reader.anchor() = anchor;
+ reader.start() = start;
+ reader.end() = end;
+ }
+
+ template<typename T>
+ inline void
+ reset_reader(T &reader, sized_array<char> &buf, char *anchor, char *start, char *end) {
+ swap(reader.buf(), buf);
+ reset_reader(reader, anchor, start, end);
+ }
+
+ template<typename T>
+ inline void
+ shift_reader(T &reader)
+ {
+ memmove(reader.buf().get(), reader.anchor(), reader.anchored());
+ ptrdiff_t diff = reader.anchor() - reader.buf().get();
+ reset_reader(reader,
+ reader.anchor() - diff,
+ reader.start() - diff,
+ reader.end() - diff);
+ }
+
+ template<typename T>
+ inline void
+ replace_reader(T &reader)
+ {
+ sized_array<char> buf(new char[reader.buf().size()], reader.buf().size());
+ memcpy(buf.get(), reader.anchor(), reader.anchored());
+ reset_reader(reader, buf,
+ buf.get(),
+ buf.get() + (reader.start() - reader.anchor()),
+ buf.get() + (reader.end() - reader.anchor()));
+ }
+
}
#endif
Modified: cpp-commons/trunk/src/commons/streamwriter.h
===================================================================
--- cpp-commons/trunk/src/commons/streamwriter.h 2009-03-18 09:36:57 UTC (rev 1302)
+++ cpp-commons/trunk/src/commons/streamwriter.h 2009-03-18 09:58:41 UTC (rev 1303)
@@ -23,7 +23,7 @@
char *mark_;
char *p_;
boost::function<void(void*, size_t)> flushcb;
- char *reserve(int n, char *p) {
+ char *reserve(size_t n, char *p) {
if (p + n > a_.end()) {
// check that the reserved space will fit
ASSERT(size_t(p - mark_ + n + sizeof(uint32_t)) <= a_.size());
@@ -51,15 +51,16 @@
size_t pos() { return p_ - mark_; }
size_t size() { return a_.size(); }
void mark() {
+ ASSERT(p_ >= mark_);
if (p_ > mark_) {
// prefix last segment with its length
- *reinterpret_cast<uint32_t*>(prefix()) = uint32_t(p_ - mark_);
+ *reinterpret_cast<uint32_t*>(prefix()) = htonl(uint32_t(pos()));
// start new segment
mark_ = (p_ += sizeof(uint32_t));
}
}
void reset() { p_ = mark_; }
- void reserve(int n) { reserve(n, p_); }
+ void reserve(size_t n) { reserve(n, p_); }
void mark_and_flush() {
mark();
flush();
@@ -71,7 +72,8 @@
unsent_ = prefix();
}
}
- template<typename T> void skip() { reserve(sizeof(T)); p_ += sizeof(T); }
+ void skip(size_t n) { reserve(n); p_ += n; }
+ template<typename T> void skip() { skip(sizeof(T)); }
template<typename T> void write(T x) { write_(x, p_); p_ += sizeof x; }
template<typename T> void write(T x, size_t off) { write_(x, mark_ + off); }
void show() {
Modified: cpp-commons/trunk/src/commons/time.h
===================================================================
--- cpp-commons/trunk/src/commons/time.h 2009-03-18 09:36:57 UTC (rev 1302)
+++ cpp-commons/trunk/src/commons/time.h 2009-03-18 09:58:41 UTC (rev 1303)
@@ -7,6 +7,8 @@
#include <sys/time.h>
#include <time.h>
+#include <commons/check.h>
+
namespace commons
{
@@ -19,15 +21,9 @@
inline long long
current_time_millis()
{
- long long t;
struct timeval tv;
-
- gettimeofday(&tv, 0);
-
- t = tv.tv_sec;
- t = (t *1000) + (tv.tv_usec/1000);
-
- return t;
+ check0x(gettimeofday(&tv, 0));
+ return 1000LL * tv.tv_sec + tv.tv_usec / 1000;
}
/**
Modified: cpp-commons/trunk/src/commons/utility.h
===================================================================
--- cpp-commons/trunk/src/commons/utility.h 2009-03-18 09:36:57 UTC (rev 1302)
+++ cpp-commons/trunk/src/commons/utility.h 2009-03-18 09:58:41 UTC (rev 1303)
@@ -11,4 +11,8 @@
* expand the given type. */
#define EXPAND(type)
+/** To avoid indentation. */
+#define BEGIN_NAMESPACE(ns) namespace ns {
+#define END_NAMESPACE }
+
#endif
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
