[Cpptool-cvs] CppParser/src/cppparser commandstream.cpp,NONE,1.1 commandstream.h,NONE,1.1 commandstreamtest.cpp,NONE,1.1 commandstreamtest.h,NONE,1.1 conststringtest.cpp,NONE,1.1 conststringtest.h,NONE,1.1 cpp_grammar.txt,NONE,1.1 cpplexer.cpp,NONE,1.1 cpplexer.h,NONE,1.1 cpplexertest.cpp,NONE,1.1 cpplexertest.h,NONE,1.1 cppnodetestbase.cpp,NONE,1.1 cppnodetestbase.h,NONE,1.1 cppparser.cpp,NONE,1.1 cppparser.vcproj,NONE,1.1 cppparserfacade.cpp,NONE,1.1 cppparsertest.cpp,NONE,1.1 cppparsertest.h,NONE,1.1 grammar_tree.txt,NONE,1.1 grammarbuilder.cpp,NONE,1.1 grammarbuilder.h,NONE,1.1 grammarbuildertest.cpp,NONE,1.1 grammarbuildertest.h,NONE,1.1 grammarparser.cpp,NONE,1.1 grammarparser.h,NONE,1.1 grammarparsertest.cpp,NONE,1.1 grammarparsertest.h,NONE,1.1 node.cpp,NONE,1.1 nodeiterator.h,NONE,1.1 nodeprocessor.cpp,NONE,1.1 nodetest.cpp,NONE,1.1 nodetest.h,NONE,1.1 nodetesthelper.cpp,NONE,1.1 nodetesthelper.h,NONE,1.1 nodetracker.cpp,NONE,1.1 nodetrackertest.cpp,NONE,1.1 nodetreetestbase.cpp,NONE,1.1 nodetreetestbase.h,NONE,1.1 parsecontext.h,NONE,1.1 parser.h,NONE,1.1 parsertest.cpp,NONE,1.1 parsertest.h,NONE,1.1 parsertesthelper.cpp,NONE,1.1 parsertesthelper.h,NONE,1.1 parsingtracker.cpp,NONE,1.1 parsingtracker.h,NONE,1.1 preprocessortest.cpp,NONE,1.1 preprocessortest.h,NONE,1.1 stlhelper.h,NONE,1.1 testtools.h,NONE,1.1 token.h,NONE,1.1 tokentesthelper.h,NONE,1.1 unittesting.h,NONE,1.1

[Baptiste L. <bl...@us...>]

Update of /cvsroot/cpptool/CppParser/src/cppparser
In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv9448/src/cppparser

Added Files:
	commandstream.cpp commandstream.h commandstreamtest.cpp 
	commandstreamtest.h conststringtest.cpp conststringtest.h 
	cpp_grammar.txt cpplexer.cpp cpplexer.h cpplexertest.cpp 
	cpplexertest.h cppnodetestbase.cpp cppnodetestbase.h 
	cppparser.cpp cppparser.vcproj cppparserfacade.cpp 
	cppparsertest.cpp cppparsertest.h grammar_tree.txt 
	grammarbuilder.cpp grammarbuilder.h grammarbuildertest.cpp 
	grammarbuildertest.h grammarparser.cpp grammarparser.h 
	grammarparsertest.cpp grammarparsertest.h node.cpp 
	nodeiterator.h nodeprocessor.cpp nodetest.cpp nodetest.h 
	nodetesthelper.cpp nodetesthelper.h nodetracker.cpp 
	nodetrackertest.cpp nodetreetestbase.cpp nodetreetestbase.h 
	parsecontext.h parser.h parsertest.cpp parsertest.h 
	parsertesthelper.cpp parsertesthelper.h parsingtracker.cpp 
	parsingtracker.h preprocessortest.cpp preprocessortest.h 
	stlhelper.h testtools.h token.h tokentesthelper.h 
	unittesting.h 
Log Message:
* made a library out of the c++ parser (cppparser)
* exposed the c++ parser to python (pycppparser)
* migrating refactoring framework to python (pyrfta)


--- NEW FILE: cppparser.vcproj ---
<?xml version="1.0" encoding="Windows-1252"?>
<VisualStudioProject
	ProjectType="Visual C++"
	Version="7.10"
	Name="cppparser"
	ProjectGUID="{EA6D6F9C-4BA7-4A0D-BD46-7ACA6F3D6E72}"
	Keyword="Win32Proj">
	<Platforms>
		<Platform
			Name="Win32"/>
	</Platforms>
	<Configurations>
		<Configuration
			Name="Debug|Win32"
			OutputDirectory="../../build/cppparser/debug"
			IntermediateDirectory="../../build/cppparser/debug"
			ConfigurationType="2"
			CharacterSet="2">
			<Tool
				Name="VCCLCompilerTool"
				Optimization="0"
				AdditionalIncludeDirectories="../../include"
				PreprocessorDefinitions="PARSER_ENABLE_TRACKING;RFTA_CPPPARSER_DLL_BUILD"
				MinimalRebuild="TRUE"
				BasicRuntimeChecks="3"
				RuntimeLibrary="3"
				EnableFunctionLevelLinking="FALSE"
				ForceConformanceInForLoopScope="TRUE"
				RuntimeTypeInfo="TRUE"
				UsePrecompiledHeader="0"
				WarningLevel="3"
				Detect64BitPortabilityProblems="TRUE"
				DebugInformationFormat="4"
				DisableSpecificWarnings="4267;4244;4018;4251"/>
			<Tool
				Name="VCCustomBuildTool"/>
			<Tool
				Name="VCLinkerTool"
				AdditionalDependencies="cpput_vc6_mds.lib"
				OutputFile="$(OutDir)/rfta_cppparser_vc7_mdd.dll"
				AdditionalLibraryDirectories="../../build/cpput/debug;../../libs/vc71/boost"
				GenerateDebugInformation="TRUE"
				SubSystem="2"
				TargetMachine="1"/>
			<Tool
				Name="VCMIDLTool"/>
			<Tool
				Name="VCPostBuildEventTool"/>
			<Tool
				Name="VCPreBuildEventTool"/>
			<Tool
				Name="VCPreLinkEventTool"/>
			<Tool
				Name="VCResourceCompilerTool"/>
			<Tool
				Name="VCWebServiceProxyGeneratorTool"/>
			<Tool
				Name="VCXMLDataGeneratorTool"/>
			<Tool
				Name="VCWebDeploymentTool"/>
			<Tool
				Name="VCManagedWrapperGeneratorTool"/>
			<Tool
				Name="VCAuxiliaryManagedWrapperGeneratorTool"/>
		</Configuration>
		<Configuration
			Name="Release|Win32"
			OutputDirectory="../../build/cppparser/release"
			IntermediateDirectory="../../build/cppparser/release"
			ConfigurationType="4"
			CharacterSet="2">
			<Tool
				Name="VCCLCompilerTool"
				AdditionalIncludeDirectories="../../include"
				PreprocessorDefinitions="PARSER_ENABLE_TRACKING;RFTA_CPPPARSER_DLL_BUILD"
				RuntimeLibrary="2"
				ForceConformanceInForLoopScope="TRUE"
				RuntimeTypeInfo="TRUE"
				UsePrecompiledHeader="0"
				WarningLevel="3"
				Detect64BitPortabilityProblems="TRUE"
				DebugInformationFormat="3"
				DisableSpecificWarnings="4267;4244;4018;4251"/>
			<Tool
				Name="VCCustomBuildTool"/>
			<Tool
				Name="VCLibrarianTool"
				OutputFile="$(OutDir)/cppparser.lib"
				AdditionalLibraryDirectories="../../libs/vc71/boost;../../build/cpput/release"/>
			<Tool
				Name="VCMIDLTool"/>
			<Tool
				Name="VCPostBuildEventTool"/>
			<Tool
				Name="VCPreBuildEventTool"/>
			<Tool
				Name="VCPreLinkEventTool"/>
			<Tool
				Name="VCResourceCompilerTool"/>
			<Tool
				Name="VCWebServiceProxyGeneratorTool"/>
			<Tool
				Name="VCXMLDataGeneratorTool"/>
			<Tool
				Name="VCManagedWrapperGeneratorTool"/>
			<Tool
				Name="VCAuxiliaryManagedWrapperGeneratorTool"/>
		</Configuration>
	</Configurations>
	<References>
	</References>
	<Files>
		<Filter
			Name="parser"
			Filter="">
			<File
				RelativePath=".\commandstream.cpp">
			</File>
			<File
				RelativePath=".\commandstream.h">
			</File>
			<File
				RelativePath=".\cpplexer.cpp">
			</File>
			<File
				RelativePath=".\cpplexer.h">
			</File>
			<File
				RelativePath=".\cppparser.cpp">
			</File>
			<File
				RelativePath=".\cppparserfacade.cpp">
			</File>
			<File
				RelativePath=".\grammarbuilder.cpp">
			</File>
			<File
				RelativePath=".\grammarbuilder.h">
			</File>
			<File
				RelativePath=".\grammarparser.cpp">
			</File>
			<File
				RelativePath=".\grammarparser.h">
			</File>
			<File
				RelativePath=".\grammarparsertest.cpp">
			</File>
			<File
				RelativePath=".\grammarparsertest.h">
			</File>
			<File
				RelativePath=".\node.cpp">
			</File>
			<File
				RelativePath=".\nodeiterator.h">
			</File>
			<File
				RelativePath=".\nodeprocessor.cpp">
			</File>
			<File
				RelativePath=".\parsecontext.h">
			</File>
			<File
				RelativePath=".\parser.h">
			</File>
			<File
				RelativePath=".\parsingtracker.cpp">
			</File>
			<File
				RelativePath=".\parsingtracker.h">
			</File>
			<File
				RelativePath=".\stlhelper.h">
			</File>
			<File
				RelativePath=".\tokenmanager.h">
			</File>
			<Filter
				Name="interface"
				Filter="">
				<File
					RelativePath="..\..\include\rfta\cppparser\autolink.h">
				</File>
				<File
					RelativePath="..\..\include\rfta\cppparser\config.h">
				</File>
				<File
					RelativePath="..\..\include\rfta\cppparser\conststring.h">
				</File>
				<File
					RelativePath="..\..\include\rfta\cppparser\cppparser.h">
				</File>
				<File
					RelativePath="..\..\include\rfta\cppparser\cppparserfacade.h">
				</File>
				<File
					RelativePath="..\..\include\rfta\cppparser\cppparsersettings.h">
				</File>
				<File
					RelativePath="..\..\include\rfta\cppparser\forwards.h">
				</File>
				<File
					RelativePath="..\..\include\rfta\cppparser\node.h">
				</File>
				<File
					RelativePath="..\..\include\rfta\cppparser\nodeprocessor.h">
				</File>
				<File
					RelativePath="..\..\include\rfta\cppparser\token.h">
				</File>
			</Filter>
		</Filter>
		<Filter
			Name="tests"
			Filter="">
			<File
				RelativePath=".\commandstreamtest.cpp">
			</File>
			<File
				RelativePath=".\commandstreamtest.h">
			</File>
			<File
				RelativePath=".\conststringtest.cpp">
			</File>
			<File
				RelativePath=".\conststringtest.h">
			</File>
			<File
				RelativePath=".\cpplexertest.cpp">
			</File>
			<File
				RelativePath=".\cpplexertest.h">
			</File>
			<File
				RelativePath=".\cppnodetestbase.cpp">
			</File>
			<File
				RelativePath=".\cppnodetestbase.h">
			</File>
			<File
				RelativePath=".\cppparsertest.cpp">
			</File>
			<File
				RelativePath=".\cppparsertest.h">
			</File>
			<File
				RelativePath=".\grammarbuildertest.cpp">
			</File>
			<File
				RelativePath=".\grammarbuildertest.h">
			</File>
			<File
				RelativePath=".\nodetest.cpp">
			</File>
			<File
				RelativePath=".\nodetest.h">
			</File>
			<File
				RelativePath=".\nodetesthelper.cpp">
			</File>
			<File
				RelativePath=".\nodetesthelper.h">
			</File>
			<File
				RelativePath=".\nodetreetestbase.cpp">
			</File>
			<File
				RelativePath=".\nodetreetestbase.h">
			</File>
			<File
				RelativePath=".\parsertest.cpp">
			</File>
			<File
				RelativePath=".\parsertest.h">
			</File>
			<File
				RelativePath=".\parsertesthelper.cpp">
			</File>
			<File
				RelativePath=".\parsertesthelper.h">
			</File>
			<File
				RelativePath=".\preprocessortest.cpp">
			</File>
			<File
				RelativePath=".\tokentesthelper.h">
			</File>
			<File
				RelativePath=".\unittesting.h">
			</File>
		</Filter>
	</Files>
	<Globals>
	</Globals>
</VisualStudioProject>

--- NEW FILE: cpplexer.cpp ---
#include "cpplexer.h"
#include "stlhelper.h"
#include <boost/regex/v4/regex.hpp>
#include <algorithm>
#include <iterator>
#include <set>
#include <utility>
#include <vector>


using namespace Parser::CppLexerMatchType;

namespace {

template<bool isMacroLexer>
class GenericCppLexer : public Parser::CppLexer
{
public:
   GenericCppLexer()
   {
      initializePatterns();
   }

   // overridden from CppLexer
   void tokenize( const char *inputStart, 
                  const char *inputEnd, 
                  Parser::CppLexerMatchHandler &handler )
   {
      handler.setTokenizedText( inputStart, inputEnd );
      const char *current = inputStart;

      while ( current != inputEnd )
      {
         bool matchFound = false;
         Patterns::iterator it = patterns().begin();
         for ( ; it != patterns().end(); ++it )
         {
            boost::match_results<const char *> results;
            if ( boost::regex_search( current, inputEnd, results, it->second,
                                    boost::match_default ) )
            {
               MatchType type = it->first;
               if ( type == matchIdentifier  &&  
                  keywords().count( std::string( results[0].first,results[0].length() ) ) > 0 )
               {
                  type = matchSymbol;
               }
               else if ( type == matchInteger  &&  
                        results[0].length() == 1  &&
                        *(results[0].first) == '0' )
               {
                  type = matchSymbol;
               }
               handler.matched( results[0].first - inputStart, 
                                results[0].second - inputStart, 
                                type );
               current = results[0].second;
               matchFound = true;
               break;
            }
         }

         if ( !matchFound )
         {
            handler.matched( current  - inputStart, 
                             current+1  - inputStart, 
                             matchError );
            ++current;
         }
      }
   }


   static void addPattern( Parser::CppLexerMatchType::MatchType type, 
                           const std::string &pattern )
   {
      boost::regex expression( pattern );
      patterns().push_back( PatternInfo( type, expression ) );
   }


   static void setKeywords( const std::string &keywordsList )
   {
      StlHelper::splitStringList( keywordsList, 
                                  std::inserter( keywords(), keywords().begin() ) );
   }

private:
   static void initializePatterns()
   {
      // pattern compilation takes a large amount of time, so they are static
      if ( !patterns().empty() )
         return;

      addPattern( matchSpaces, "\\A[ \t\r\n]+" );
      addPattern( matchCppComment, "\\A//[^\\r\\n]*" );
      addPattern( matchCComment, "\\A(?:/\\*.*?\\*/)" );
      if ( !isMacroLexer )
      {
         addPattern( matchPreprocessorDirective, "\\A#[ \t]*[a-zA-Z_]"
                     "(?:" 
                        "(?:\\\\(?:\\r|\\n|\\r\\n))|" // '\n': macro continue
                        "(?:/\\*.*?\\*/)|"    // C comment
                        "[^\\r\\n]"
                     ")*" );
      }
      addPattern( matchIdentifier, "\\A[A-Za-z_][A-Za-z0-9_]*" );
      addPattern( matchString, "\\A\"(?:[^\\\\\"]|\\\\.)*\"" );
      addPattern( matchCharacter, "\\A'(?:[^\\\\']|\\\\.)*'" );
      addPattern( matchFloat, "\\A(?:[0-9]*\\.[0-9]+(?:e[-\\+]?[0-9]+)?[fFL]?|" // [1234].1234[e[-+]123][fFL]
                                 "[0-9]+(?:[eE][+-]?[0-9]+)[fFL]?)" );       // 1234e[-+]123[fFL]
      addPattern( matchInteger, "\\A(?:0x[0-9a-fA-F]+|[0-9]+)L?" );

      std::string keywords = "__asm asm else struct __assume enum __multiple_inheritance switch "
         "auto __except __single_inheritance template __based explicit __virtual_inheritance this "
         "bool extern mutable break false throw case __fastcall namespace true "
         "catch __finally new try __cdecl float __try char for operator typedef "
         "class friend private typeid const goto protected typename const_cast if public union "
         "continue inline register unsigned __declspec __inline reinterpret_cast using "
         "default int return delete __int8 short __uuidof __int16 signed virtual "
         "__int32 sizeof void do __int64 static volatile double __leave static_cast wmain "
         "dynamic_cast long __stdcall while ";
      setKeywords( keywords );

      std::string symbols = "\\.\\.\\. >>= <<= :: << >> <= >= == != \\|\\| && "
         "&= \\|= \\+= -= /= \\*= \\^= %= -> \\+\\+ -- "
         "\\. \\( \\) \\[ \\] \\{ \\} ~ / - \\+ / \\* = \\^ % < > & \\| ; , : \\? !";
      if ( isMacroLexer )
         symbols = "## " + symbols + " #";
      std::replace( symbols.begin(), symbols.end(), ' ', '|' );
      addPattern( matchSymbol, "\\A(?:" + symbols + ")" );
   }

private:
   typedef std::pair<Parser::CppLexerMatchType::MatchType,boost::regex> PatternInfo;
   typedef std::vector<PatternInfo> Patterns;

   static Patterns &patterns()
   {
      static Patterns patternsGlobal;
      return patternsGlobal;
   }

   typedef std::set<std::string> Keywords;
   static Keywords &keywords()
   {
      static Keywords keywordsGlobal;
      return keywordsGlobal;
   }
};

} // anonymous namespace


namespace Parser {


CppLexerPtr 
makeCppLexer()
{
   return CppLexerPtr( new GenericCppLexer<false>() );
}


CppLexerPtr 
makeCppMacroLexer()
{
   return CppLexerPtr( new GenericCppLexer<true>() );
}


} // namespace Parser
--- NEW FILE: grammar_tree.txt ---
/* C++ grammar Tree  */

translation_unit[
   declarations[...]
]


///////////////////////////////// declarations ///////////////////////////
declarations[
   [ n declaration_error 
   |  namespace_alias_def[
         id
         namespace_specifier[...]
            ~ n id]                 // +(namespace_name)
   |  using_declaration[
         using_id
            ~ n id]                 // @todo any_id
   |  using_directive[
      namespace_specifier[...] ]
   |  simple_declaration[
         [ typedef_decl_specifiers[
            type_specifier[...] ]
         | friend_specifier[
            declaration_specifier[...] ]
         | declaration_specifier[
            //content same as friend_specifier
            ? storage_class_specifier
            ? function_specifier
            type_specifier[...]
            ]
         ]
         ? init_declarators[
            n init_declarator[
               declarator[...]
               ? assign_initializer[
                  [ assignment_expression[...]  // = 1234
                  | n initializer_clause ]      // = {1,2,3,4}
                  | expression_list[...]        // (1,2,3,4)  ctor
                  ]
               ]
            ]
         ]
      ]
   | linkage_specification[
      'extern'
      $string
      [ '{' !declaration_seq '}' => same as translation_unit
      | declaration[...] ]
      ]
   | named_namespace_def[
      id
      namespace_body[ !declaration_seq ]
      ]
   | unnamed_namespace_def[
      namespace_body[ !declaration_seq ]
      ]
   | function_definition[ ... ]
   | template_declaration[ ... ]
   | explicit_template_instantiation[...]
   | explicit_template_specialization[...]
   ]
]
            
///////////////////////////////// type_specifier ///////////////////////////

type_specifier[
   cv_qualifiers
   [ fundamental_type_specifier
   | type_name
   | enum_specifier[
      id
      enumerator_definitions[
         n enumerator_definition[
            id
            constant_expression[...]
         ] ]
      ]
   | class_specifier[...]
   | forward_class_specifier[
      forwarded_type_id[
         ~n id ] ]
   | forward_enum_specifier[        // @todo better tree structure
      forwarded_type_id[
      ~n id ] ]
   | forward_typename_specifier[
      forwarded_type_id[
      ~n id ] ]
   ]
   ? cv_qualifiers
]

   

/////////////////////////////////
class_specifier[
   class_name
      ? dll_macro[
         id]
      ? nested_name_specifier[...]
      id[
         id]
      | template_id[...]
   ? class_bases[
      n base_specifier[
         base_class_access
            ? 'virtual'
            %access_specifier
            ? 'virtual']
         base_class_name[
            ? nested_name_specifier
            id]
      ]
   ? member_specification[
      n access_specification[
         %access_specifier ]
      | member_declaration[
         member_function_definition[
            function_definition[
               ? function_return_type
               declarator_id[~ n id]
               function_parameters[
                  n function_parameter[
                     declaration_specifier[
                        ]
                     declarator[
                        ? ptr_operator_declarator[ 
                              '*' ? cv_qualifier_seq[...]
                              | '&'
                              | pointer_to_member[
                                 nested_name_specifier[..] ?cv_qualifier_seq[...]
                              ]
                           ]
                        declarator_id[...]
                        | braced_declarator[ declarator[...] ]
                        function_parameters[...]
                        | abstract_array_declarator[
                           constant_expression[...] ]
                     ? assign_initializer[
                        assignment_expression[...]
                     ] ]
                  ? ellipsis_parameter[ '...' ] ? cv_qualifiers ? exception_specification

               ]
               ? cv_qualifiers[
                  n cv_qualifier]
               ? exception_specification[
                  type_id_list[
                     type_id[...] ] ]
         ]
      | member_error[...]
      ]
]


--- NEW FILE: cppparserfacade.cpp ---
#include <rfta/cppparser/cppparserfacade.h>
#include <rfta/cppparser/cppparser.h>
#include "grammarbuilder.h"
#include "parsecontext.h"
//#include "commandstream.h"
#include <fstream>


namespace Parser {

CppParserFacade::CppParserFacade( const CppParserSettings &settings )
   : builder_( new GrammarBuilder() )
   , settings_( settings )
{
   std::ifstream is( settings.grammarPath().c_str() );
   if ( is.fail() )
      throw CppParserFacadeError( "Failed to open c++ grammar file: " + settings.grammarPath() );
   std::string grammar;
   std::string line;
   while ( !is.eof() )
   {
      std::getline( is, line );
      grammar += line;
      grammar += '\n';
   }
   is.close();


   CppParser::setUpGrammarBuilder( *builder_ );
   builder_->build( grammar );
   if ( builder_->getErrorCount() > 0 )
      throw CppParserFacadeError( "C++ grammar contains error." );

   translationUnitMatcher_ = builder_->getMatcher( "translation_unit" );
   if ( !translationUnitMatcher_ )
      throw CppParserFacadeError( "C++ grammar does not contains a 'translation_unit' rule." );
}


NodePtr 
CppParserFacade::parse( const std::string &input )
{
   Parser::Tokens tokens;
//   Parser::CppParser::preprocess( input, tokens );
   Parser::CppParser::tokenize( input, tokens );
   Parser::CommandStream result;
   Parser::ParseContext context( tokens.begin(), tokens.end(), result );

   bool matched = translationUnitMatcher_->parse( context );
   if ( !matched )   // should never happen because of error recovery
      throw CppParserFacadeError( "Critical error: failed to parse input." );

   // note: at the current time, ending silent tokens are not added to the ast
   
   NodePtr rootNode = result.makeNodeTree();
   if ( rootNode->hasChildren() )
      return rootNode->childAt(0);
   return NodePtr();
}


} // namespace Parser

--- NEW FILE: parsertesthelper.cpp ---
#include "parsertesthelper.h"


namespace ParserTesting {


void addParseContextRemainingTokens( Parser::ParseContext &context,
                                     CppUT::Message &message )
{
   Parser::Tokens remainingTokens;
   context.getRelativeTokenRange( 0, 100, remainingTokens );
   message.add( "Remaining tokens: " 
                + CppUT::enumToString( CppUT::enumStl( remainingTokens ) ) );
}

static bool containsErrorNode( const Parser::NodePtr &tree )
{
   if ( tree->name().str().find( "error" ) != std::string::npos )
      return true;
   Parser::NodeEnumerator enumNode( tree->enumChildren() );
   while ( enumNode.hasNext() )
      if ( containsErrorNode( enumNode.nextPtr() ) )
         return true;
   return false;
}

void checkContainsNoErrorNode( const Parser::NodePtr &tree,
                               const CppUT::Message &message )
{
   if ( containsErrorNode( tree ) )
   {
      CppUT::Message errorMessage( message );
      errorMessage.add( "Produced tree:\n" + tree->treeStr(2) );
      CppUT::fail( errorMessage );
   }
}

void checkParseContextIsBalanced( Parser::ParseContext &context,
                                  Parser::Tokens tokens,
                                  const Parser::MatcherPtr &matcher,
                                  Parser::FullParsingTracker &tracker )
{
   if ( context.isStateBalanced() )
      return;

   CppUT::Message newMessage;
   newMessage.add( "Parser state is not balanced." );
   addParseContextRemainingTokens( context, newMessage );
   newMessage.add( "Tokens: " + CppUT::enumToString( CppUT::enumStl( tokens ) ) );
   newMessage.add( "Parser: " + matcher->str() );
   newMessage.add( "Parsing traces:\n" + tracker.collectedTraces() );
   CppUT::fail( newMessage );
}


void checkParserMatchPartial( Parser::Tokens tokens,
                              const Parser::MatcherPtr &matcher,
                              int expectedRemainingTokens,
                              TraceFlag flag,
                              Parser::NodePtr expectedTree )
{
//   tokens.push_back( Parser::Token( "", Parser::endStream ) );
   Parser::FullParsingTracker tracker;
   if ( (flag & progressiveTrace) != 0 )
      tracker.setProgressive( true );

   bool enableTrace = (flag & traceEnabled) != 0;

   Parser::CommandStream result;
   Parser::ParseContext context( tokens.begin(), tokens.end(), result, 
                                 enableTrace ? &tracker : 0 );
   bool matched = matcher->parse( context );
   checkParseContextIsBalanced( context, tokens, matcher, tracker );

   CppUT::Message newMessage;
   if ( matched )
   {
      context.pushState();
      int actualRemainingTokens = 0;
      while ( context.hasMoreToken() ) // context.nextToken() &&  .type() != Parser::endStream )
      {
         context.nextToken();
         ++actualRemainingTokens;
      }
      context.popState();

      if ( actualRemainingTokens == expectedRemainingTokens )
      {
         bool checkNoError = (flag & checkNoErrorNodeInTree) != 0;
         if ( expectedTree  ||  checkNoError )
         {
            Parser::NodePtr actualTree = result.makeNodeTree();
            if ( expectedTree )
            {
               if ( (flag & addEmptyRootToExpectedTree) != 0 )
                  expectedTree = testNode( "", expectedTree );
               ParserTesting::checkNodeTreeEquals( expectedTree, actualTree, 
                                                   "Parsing did not produce the expected tree." );
            }

            if ( checkNoError )
               ParserTesting::checkContainsNoErrorNode( actualTree, 
                                                      "Error recovery was triggered during parsing." );
         }
         return;
      }
      if ( expectedRemainingTokens == 0 )
         newMessage.add( "All tokens were not consumed by parsing." );
      else
         newMessage.add( CppUT::makeEqualityFailedMessage( expectedRemainingTokens,
                                                           actualRemainingTokens,
                                                           "Wrong remaining token count." ) );
   }
   else
      newMessage.add( "Parsing failed." );

   addParseContextRemainingTokens( context, newMessage );
   newMessage.add( "Tokens: " + CppUT::enumToString( CppUT::enumStl( tokens ) ) );
   if ( (flag & dumpParser) != 0 )
      newMessage.add( "Parser: " + matcher->str() );
   else if ( !matcher->name().empty() )
      newMessage.add( "Parser name: " + matcher->name() );
   if ( enableTrace )
      newMessage.add( "Parsing traces:\n" + tracker.collectedTraces() );
   CppUT::fail( newMessage );
}


void checkParserMatch( Parser::Tokens tokens,
                       const Parser::MatcherPtr &matcher,
                       TraceFlag flag )
{
   checkParserMatchPartial( tokens, matcher, 0, flag, Parser::NodePtr() );
}


void checkParserFail( Parser::Tokens tokens,
                      const Parser::MatcherPtr &matcher,
                      TraceFlag flag )
{
//   tokens.push_back( Parser::Token( "", Parser::endStream ) );
   Parser::FullParsingTracker tracker;
   if ( (flag & progressiveTrace) != 0 )
      tracker.setProgressive( true );

   Parser::CommandStream result;
   Parser::ParseContext context( tokens.begin(), tokens.end(), result, &tracker );
   bool matched = matcher->parse( context );
   checkParseContextIsBalanced( context, tokens, matcher, tracker );
   if ( !matched )
      return;

   CppUT::Message newMessage;
   newMessage.add( "Parsing did not failed." );
   newMessage.add( "Tokens: " + CppUT::enumToString( CppUT::enumStl( tokens ) ) );
   newMessage.add( "Parser: " + matcher->str() );
   newMessage.add( "Parsing traces:\n" + tracker.collectedTraces() );
   CppUT::fail( newMessage );
}


void checkProducedTreeEqual( Parser::Tokens tokens,
                             const Parser::MatcherPtr &matcher,
                             const Parser::NodePtr &expected,
                             TraceFlag flag )
{
   checkParserMatchPartial( tokens, matcher, 0, flag, expected );
}


void checkPartialProducedTreeEqual( Parser::Tokens tokens,
                                    const Parser::MatcherPtr &matcher,
                                    const Parser::NodePtr &expected,
                                    int expectedRemaining,
                                    TraceFlag flag )
{
   checkParserMatchPartial( tokens, matcher, expectedRemaining, flag, expected );
}


} // namespace ParserTesting

--- NEW FILE: token.h ---
#ifndef PARSER_TOKEN_H_INCLUDED
# define PARSER_TOKEN_H_INCLUDED

# include <rfta/cppparser/conststring.h>
# include <map>
# include <sstream>
# include <vector>


namespace Parser
{
   typedef unsigned int BufferId;

   class TokenLocation
   {
   public:
      TokenLocation()
         : startPos_(-1)
         , endPos_(-1)
         , bufferId_(-1)
      {
      }

      TokenLocation( unsigned int startPos, 
                     unsigned int endPos, 
                     BufferId bufferId )
         : startPos_( startPos )
         , endPos_( endPos )
         , bufferId_( bufferId )
      {
      }

      bool isValid() const
      {
         return startPos_ != -1  &&  endPos_ != -1;
      }

      bool operator ==( const TokenLocation &other ) const
      {
         return startPos_ == other.startPos_  &&
                endPos_ == other.endPos_  &&
                bufferId_ == other.bufferId_;
      }

      bool operator !=( const TokenLocation &other ) const
      {
         return !( *this == other );
      }

      unsigned int length() const
      {
         return endPos_ - startPos_;
      }

      unsigned int startPos_;
      unsigned int endPos_;
      BufferId bufferId_;
   };


   class TokenType
   {
   public:
      enum { idShift = 16 };

      TokenType()
      {
      }

      TokenType( unsigned int type )
         : type_( type )
      {
      }

      TokenType( TokenType category, unsigned int id )
         : type_( category.type_ | id )
      {
      }

      bool operator ==( TokenType other ) const;
      bool operator !=( TokenType other ) const;
      bool operator <( TokenType other ) const;

      unsigned int id() const;

      bool isSilent() const;

      TokenType category() const;

      TokenType nextCategory() const;

      unsigned int hash() const;

      void setSilent();

   private:
      enum {
         tokenIdMask = 0xffff0000,
         tokenCategoryMask = 0xfff,
         tokenCategoryIncrement = 0x1,
         tokenSilentMask = 0x8000000
      };
      unsigned int type_;
   };


   class TokenIdManager
   {
   public:
      static unsigned int get( const ConstString &text );
      static std::string str( TokenType type );

   private:
      static TokenIdManager &instance();

      typedef std::map<ConstString,unsigned int> TokenIds;
      TokenIds ids_;
      typedef std::map<unsigned int,std::string> IdTexts;
      IdTexts texts_;
   };


   enum DefaultTokenType {
      invalidTokenType = -1,
      endStream = 0,
      symbol,
      lastUsedTokenType
   };


   class TokenTypeManager
   {
   public:
      TokenTypeManager();

      static TokenTypeManager &instance();

      TokenType add( const ConstString &name, bool isSilent = false );

      ConstString typeName( TokenType type ) const;

   private:
      void set( TokenType id, const ConstString &name );

      typedef std::map<ConstString,TokenType> Types;
      Types types_;
      typedef std::map<TokenType,ConstString> Names;
      Names names_;
      ConstString unknownName_;
      TokenType last_;
   };

   inline ConstString tokenTypeName( TokenType type )
   {
      return TokenTypeManager::instance().typeName( type );
   }

   class Token
   {
   public:
      Token( const ConstString &text, 
             TokenType tokenType,
             TokenLocation location = TokenLocation() );

      const ConstString &text() const;

      const TokenLocation &location() const;

      TokenType type() const;

      TokenType category() const;

      bool isSilent() const;

      ConstString typeName() const;

      unsigned int hash() const;

      bool operator ==( const Token &other ) const;

      bool operator !=( const Token &other ) const;

      std::string str() const;

   private:
      ConstString text_;
      TokenLocation location_;
      TokenType tokenType_;
   };

   typedef std::vector<Token> Tokens;


   // Inline functions
   // //////////////////////////////////////////

   inline bool 
   TokenType::operator ==( TokenType other ) const
   {
      return type_ == other.type_;
   }
   
   inline bool 
   TokenType::operator !=( TokenType other ) const
   {
      return type_ != other.type_;
   }

   inline bool 
   TokenType::operator <( TokenType other ) const
   {
      return type_ < other.type_;
   }

   inline unsigned int 
   TokenType::id() const
   {
      return type_ & tokenIdMask;
   }

   inline bool 
   TokenType::isSilent() const
   {
      return (type_ & tokenSilentMask) != 0;
   }

   inline void 
   TokenType::setSilent()
   {
      type_ |= tokenSilentMask;
   }


   inline TokenType 
   TokenType::category() const
   {
      return TokenType( type_ & ~tokenIdMask );
   }

   inline TokenType 
   TokenType::nextCategory() const
   {
      return category().type_ + tokenCategoryIncrement;
   }

   inline unsigned int 
   TokenType::hash() const
   {
      return type_;
   }



   inline unsigned int 
   TokenIdManager::get( const ConstString &text )
   {
      TokenIds::iterator it = instance().ids_.find( text );
      if ( it == instance().ids_.end() )
      {
         unsigned int id = (instance().ids_.size()+1) << TokenType::idShift;
         instance().ids_.insert( TokenIds::value_type( text, id ) );
         instance().texts_.insert( IdTexts::value_type( id, text.str() ) );
         return id;
      }
      return it->second;
   }

   inline std::string 
   TokenIdManager::str( TokenType type )
   {
      if ( instance().texts_.count(type.id()) == 0)
         return "<unknown>";
      return instance().texts_.find(type.id())->second;
   }

   inline TokenIdManager &
   TokenIdManager::instance()
   {
      static TokenIdManager manager;
      return manager;
   }




   inline 
   TokenTypeManager::TokenTypeManager()
      : last_( lastUsedTokenType )
      , unknownName_( "?" )
   {
      set( endStream, "end_stream" );
      set( symbol, "symbol" );
   }

   inline TokenTypeManager &
   TokenTypeManager::instance()
   {
      static TokenTypeManager manager;
      return manager;
   }

   inline TokenType 
   TokenTypeManager::add( const ConstString &name, bool isSilent )
   {
      Types::iterator it = types_.find( name );
      if ( it == types_.end() )
      {
         TokenType type = last_;
         if ( isSilent )
         {
            type.setSilent();
            set( last_, "__silent__." + name.str() );
         }
         else
            set( last_, name );
         last_ = last_.nextCategory();
         return type;
      }
      return it->second;
   }

   inline ConstString 
   TokenTypeManager::typeName( TokenType type ) const
   {
      Names::const_iterator it = names_.find( type.category() );
      if ( it == names_.end() )
         return unknownName_;
      if ( type.isSilent() )
         return it->second.str().substr( 11 );  // "__silent__."
      return it->second;
   }

   inline void
   TokenTypeManager::set( TokenType id, const ConstString &name )
   {
      types_[name] = id;
      names_[id] = name;
   }




   inline 
   Token::Token( const ConstString &text, 
            TokenType tokenType,
            TokenLocation location )
      : text_( text )
      , tokenType_( tokenType )
      , location_( location )
   {
      if ( tokenType.category() == TokenType(symbol)  &&  tokenType.id() == 0 )
         tokenType_ = TokenType( tokenType.category(), TokenIdManager::get( text ) ); 
   }

   inline const ConstString &
   Token::text() const
   {
      return text_;
   }

   inline const TokenLocation &
   Token::location() const
   {
      return location_;
   }

   inline TokenType 
   Token::type() const
   {
      return tokenType_;
   }

   inline TokenType 
   Token::category() const
   {
      return tokenType_.category();
   }

   inline bool 
   Token::isSilent() const
   {
      return tokenType_.isSilent();
   }

   inline ConstString
   Token::typeName() const
   {
      return tokenTypeName( tokenType_ );
   }

   inline unsigned int 
   Token::hash() const
   {
      return (tokenType_.hash() << 17) + text_.hash();
   }

   inline bool 
   Token::operator ==( const Token &other ) const
   {
      return text_ == other.text_  &&
               tokenType_ == other.tokenType_  &&
               location_ == other.location_;
   }

   inline bool 
   Token::operator !=( const Token &other ) const
   {
      return !( *this == other );
   }

   inline std::string 
   Token::str() const
   {
      std::ostringstream os;
      os  << "Token(" << tokenTypeName(tokenType_).str() << ", " 
            << location_.startPos_ << "-"  <<  location_.endPos_ 
            << " = '"  << text_.str()  << "')";
      return os.str();
   }


} // namespace Parser



#endif // PARSER_TOKEN_H_INCLUDED

--- NEW FILE: cppparsertest.h ---
#ifndef PARSER_PARSERTEST_H_INCLUDED
# define PARSER_PARSERTEST_H_INCLUDED

# include "cppnodetestbase.h"
# include "grammarbuilder.h"
# include "parsertesthelper.h"


class CppParserTest : public CppNodeTestBase
{
public:
   CPPUT_TESTSUITE_BEGIN( CppParserTest );
   CPPUT_TEST( testCheckAssertFailIfContainsErrorNode );
   CPPUT_TEST( testLiteralExpression );
   CPPUT_TEST( testUnqualifiedId );
   CPPUT_TEST( testTokenize );
   CPPUT_TEST( testQualifiedId );
   CPPUT_TEST( testPrimaryExpression );
   CPPUT_TEST( testPostFixExpressionSuffix );
   CPPUT_TEST( testPostFixExpression );
   CPPUT_TEST( testDeleteExpressionParser );
   CPPUT_TEST( testNewTypeIdParser );
   CPPUT_TEST( testNewExpressionParser );
   CPPUT_TEST( testUnaryExpressionParser );
   CPPUT_TEST( testPMExpression );
   CPPUT_TEST( testLogicalOrExpressionParser );
   CPPUT_TEST( testConditionalExpressionParser );
   CPPUT_TEST( testAssignmentExpressionParser );
   CPPUT_TEST( testExpressionParser );
   CPPUT_TEST( testExpressionListParser );
   CPPUT_TEST( testConstantExpressionParser );
   CPPUT_TEST( testJumpStatementParser );
   CPPUT_TEST( testCompoundStatementParser );
   CPPUT_TEST( testSelectionStatementParser );
   CPPUT_TEST( testExpressionStatementParser );
   CPPUT_TEST( testLabelStatementParser );
   CPPUT_TEST( testIterationStatementParser );
   CPPUT_TEST( testDeclarationStatement );
   CPPUT_TEST( testTryBlockParser );
   CPPUT_TEST( testStatementParser );
   CPPUT_TEST( testExceptionSpecifierParser );
   CPPUT_TEST( testPtrOperatorParser );
   CPPUT_TEST( testParameterDeclarationParser );
   CPPUT_TEST( testParameterDeclarationClauseParser );
   CPPUT_TEST( testAbstractDeclaratorParser );
   CPPUT_TEST( testDirectDeclaratorParser );
   CPPUT_TEST( testDeclaratorParser );
   CPPUT_TEST( testInitializerParser );
   CPPUT_TEST( testInitDeclaratorParser );
   CPPUT_TEST( testDirectAbstractDeclaratorFnSuffixParser );
   CPPUT_TEST( testDirectAbstractDeclaratorArraySuffixParser );
   CPPUT_TEST( testFunctionReturnTypeParser );
   CPPUT_TEST( testFunctionDefinitionStartParser );
   CPPUT_TEST( testFunctionDefinitionParser );
   CPPUT_TEST( testTypeIdParser );
   CPPUT_TEST( testASMDefinitionParser );
   CPPUT_TEST( testNamespaceAliasDefinitionParser );
   CPPUT_TEST( testUsingDeclarationParser );
   CPPUT_TEST( testUsingDirectiveParser );
   CPPUT_TEST( testNamespaceDefinitionParser );
   CPPUT_TEST( testSimpleTypeSpecifierParser );
   CPPUT_TEST( testEnumSpecifierParser );
   CPPUT_TEST( testLinkageSpecificationParser );
   CPPUT_TEST( testElaboratedTypeSpecifierParser );
   CPPUT_TEST( testTypeSpecifierSeqParser );
   CPPUT_TEST( testWorkAroundTypeSpecifierParser );
   CPPUT_TEST( testDeclSpecifiersParser );
   CPPUT_TEST( testSimpleDeclarationParser );
   CPPUT_TEST( testDeclarationParser );
   CPPUT_TEST( testPureSpecifier );
   CPPUT_TEST( testBaseClauseParser );
   CPPUT_TEST( testClassHeadParser );
   CPPUT_TEST( testMemInitializer );
   CPPUT_TEST( testCtorInitializer );
   CPPUT_TEST( testMemberDeclarationParser );
   CPPUT_TEST( testMemberSpecificationParser );
   CPPUT_TEST( testClassSpecifierParser );
   CPPUT_TEST( testOperatorFunctionIdParser );
   CPPUT_TEST( testConversionFunctionIdParser );
   CPPUT_TEST( testTemplateParameter );
   CPPUT_TEST( testTemplateDeclaration );
   CPPUT_TEST( testTranslationUnit );
   CPPUT_TESTSUITE_END();
   
   void setUp();

   void tearDown();


   void testTokenize();
   void testCheckAssertFailIfContainsErrorNode();

   // expressions
   void testLiteralExpression();
   void testUnqualifiedId();
   void testQualifiedId();
   void testPrimaryExpression();
   void testPostFixExpressionSuffix();
   void testPostFixExpression();
   void testDeleteExpressionParser();
   void testNewTypeIdParser();
   void testNewExpressionParser();
   void testUnaryExpressionParser();
   void testPMExpression();
   void testLogicalOrExpressionParser();
   void testConditionalExpressionParser();
   void testAssignmentExpressionParser();
   void testExpressionParser();
   void testExpressionListParser();
   void testConstantExpressionParser();

   // statements
   void testJumpStatementParser();
   void testCompoundStatementParser();
   void testSelectionStatementParser();
   void testExpressionStatementParser();
   void testLabelStatementParser();
   void testIterationStatementParser();
   void testDeclarationStatement();
   void testTryBlockParser();
   void testStatementParser();

   // declarators
   void testExceptionSpecifierParser();
   void testPtrOperatorParser();
   void testParameterDeclarationParser();
   void testParameterDeclarationClauseParser();
   void testAbstractDeclaratorParser();
   void testDirectDeclaratorParser();
   void testDeclaratorParser();
   void testInitializerParser();
   void testInitDeclaratorParser();
   void testDirectAbstractDeclaratorFnSuffixParser();
   void testDirectAbstractDeclaratorArraySuffixParser();
   void testFunctionReturnTypeParser();
   void testFunctionDefinitionStartParser();
   void testFunctionDefinitionParser();
   void testTypeIdParser();

   // Declarations
   void testASMDefinitionParser();
   void testNamespaceAliasDefinitionParser();
   void testUsingDeclarationParser();
   void testUsingDirectiveParser();
   void testNamespaceDefinitionParser();
   void testSimpleTypeSpecifierParser();
   void testEnumSpecifierParser();
   void testLinkageSpecificationParser();
   void testElaboratedTypeSpecifierParser();
   void testTypeSpecifierSeqParser();
   void testWorkAroundTypeSpecifierParser();
   void testDeclSpecifiersParser();
   void testSimpleDeclarationParser();
   void testDeclarationParser();

   // Classes
   void testPureSpecifier();
   void testBaseClauseParser();
   void testClassHeadParser();
   void testMemInitializer();
   void testCtorInitializer();
   void testMemberDeclarationParser();
   void testMemberSpecificationParser();
   void testClassSpecifierParser();
   void testOperatorFunctionIdParser();
   void testConversionFunctionIdParser();

   // Templates
   void testTemplateParameter();
   void testTemplateDeclaration();

   // Program 
   void testTranslationUnit();

private:
   Parser::NodePtr fundamentalTypeSpecifierNode( const std::string &type1, int spaces = 0 );
   Parser::NodePtr fundamentalTypeNode( const std::string &type1, int spaces = 0 );
   Parser::NodePtr unqualifiedIdNode( const std::string &name, int spaces = 0 );
   Parser::NodePtr unqualifiedIdExpressionNode( const std::string &name, int spaces = 0 );
   Parser::NodePtr declaratorIdNode( const std::string &id, int spaces = 0 );
   Parser::NodePtr intLiteralNode( const std::string &value );
   Parser::NodePtr floatLiteralNode( const std::string &value );

   void checkGrammarParserMatch( const std::string &input,
                                 const std::string &parserName,
                                 ParserTesting::TraceFlag flag = ParserTesting::noParseTrace );
   void checkGrammarParserMatchPartial( const std::string &input,
                                        const std::string &parserName,
                                        int expectedRemainingToken,
                                        ParserTesting::TraceFlag flag = ParserTesting::noParseTrace );

   void checkGrammarParserMatchFail( const std::string &input,
                                     const std::string &parserName,
                                     ParserTesting::TraceFlag flag = ParserTesting::noParseTrace );

   void checkGrammarParserProducedTree( const std::string &input,
                                        const std::string &parserName,
                                        const Parser::NodePtr &expected,
                                        ParserTesting::TraceFlag flag = ParserTesting::noParseTrace );

   void checkGrammarParserProducedPartialTree( const std::string &input,
                                               const std::string &parserName,
                                               const Parser::NodePtr &expected,
                                               int expectedRemaining,
                                               ParserTesting::TraceFlag flag = ParserTesting::noParseTrace );

   void checkGrammarParserProducedTreeInPlace( const std::string &input,
                                               const std::string &parserName,
                                               const Parser::NodePtr &expected,
                                               ParserTesting::TraceFlag flag = 
                                               ParserTesting::matchedTrace );

   static boost::shared_ptr<Parser::GrammarBuilder> builder_;
   static bool initializationFailed_;
};


#endif // PARSER_PARSERTEST_H_INCLUDED

--- NEW FILE: cppparsertest.cpp ---
#include "cppparsertest.h"
#include <rfta/cppparser/cppparser.h>
#include "testtools.h"
#include <algorithm>
#include <fstream>


// Notes: when debugging a unit test, set the trace flag
// to Parser::matchedTrace:
// CPPPARSER_ASSERT_MATCH( "(this)", "postfix_expression", Parser::matchedTrace );
// Trace are disabled by default, there just way to big at high-level...


CPPUT_REGISTER_SUITE_TO_DEFAULT( CppParserTest );

using namespace ParserTesting;


#define CPPPARSER_ASSERT_MATCH \
[...1609 lines suppressed...]
   //   "  void f() {}"
   //   "};", 
   //   "translation_unit",
   //   testNode( "translation_unit" ) );


   // test error recovery
   CPPPARSER_ASSERT_TREE( "struct S { };"
                          "WEIRD_MACRO(12)"
                          "int x;"
                          "struct X {};"
                          , "translation_unit",
      testNode( "translation_unit",
         testNode( "declarations",
            jokerNode("simple_declaration"),
            testNode("declaration_error",
                     id_t("WEIRD_MACRO"), "(",int_t("12"),")", 
                     "int", spaces_t(1), id_t("x"), ";" ),
            jokerNode("simple_declaration") ) ) );
}

--- NEW FILE: commandstreamtest.h ---
#ifndef PARSER_COMMANDSTREAMTEST_H_INCLUDED
# define PARSER_COMMANDSTREAMTEST_H_INCLUDED

# include "nodetreetestbase.h"
# include "commandstream.h"
# include <boost/shared_ptr.hpp>


class CommandStreamTest : public NodeTreeTestBase
{
public:
   CPPUT_TESTSUITE_BEGIN( CommandStreamTest );
   CPPUT_TEST( testMakeEmptyNodeTree );
   CPPUT_TEST( testLinearTokenStream );
   CPPUT_TEST( testNewNodeCommand );
   CPPUT_TEST( testRenameNodeCommand );
   CPPUT_TEST( testRenameLastChildNodeCommand );
   CPPUT_TEST( testSaveRestoreCurrentNodeCommand );
   CPPUT_TEST( testMakePenultimateChildOfLast );
   CPPUT_TEST( testRemoveLastChildCommand );
   CPPUT_TEST( testPushPopState );
   CPPUT_TESTSUITE_END();
   
   void setUp();

   void tearDown();

   void testMakeEmptyNodeTree();
   void testLinearTokenStream();
   void testNewNodeCommand();
   void testRenameNodeCommand();
   void testRenameLastChildNodeCommand();
   void testSaveRestoreCurrentNodeCommand();
   void testMakePenultimateChildOfLast();
   void testRemoveLastChildCommand();
   void testPushPopState();

private:
   boost::shared_ptr<Parser::CommandStream> stream_;
};


#endif // PARSER_COMMANDSTREAMTEST_H_INCLUDED

--- NEW FILE: cpplexer.h ---
#ifndef PARSER_CPPLEXER_H_INCLUDED
# define PARSER_CPPLEXER_H_INCLUDED

# include <boost/shared_ptr.hpp>
# include <string>

namespace Parser {

   namespace CppLexerMatchType {
      enum MatchType 
      {
         matchSpaces = 0,
         matchIdentifier,
         matchString,
         matchCharacter,
         matchInteger,
         matchFloat,
         matchSymbol,
         matchCppComment,
         matchCComment,
         matchPreprocessorDirective,
         matchError
      };
   };

   class CppLexerMatchHandler
   {
   public:
      virtual ~CppLexerMatchHandler()
      {
      }

      virtual void setTokenizedText( const char *begin, const char *end ) = 0;

      virtual void matched( unsigned int startPos, 
                            unsigned int endPos, 
                            CppLexerMatchType::MatchType type ) = 0;
   };

   class CppLexerMatchHandlerBase : public CppLexerMatchHandler
   {
   public:
      CppLexerMatchHandlerBase()
         : begin_( 0 )
         , end_( 0 )
      {
      }

      const char *tokenizedTextAt( unsigned int pos ) const
      {
         return begin_ + pos;
      }

      const std::string tokenizedStringAt( unsigned int startPos, unsigned int endPos ) const
      {
         return std::string( tokenizedTextAt( startPos ), tokenizedTextAt( endPos ) );
      }

   public: // overridden from CppLexerMatchHandler
      void setTokenizedText( const char *begin, const char *end )
      {
         begin_ = begin;
         end_ = end;
      }

      virtual void matched( unsigned int startPos, 
                            unsigned int endPos, 
                            CppLexerMatchType::MatchType type ) = 0;

   private:
      const char *begin_;
      const char *end_;
   };

   class CppLexer
   {
   public:
      virtual ~CppLexer()
      {
      }

      virtual void tokenize( const char *inputStart, 
                             const char *inputEnd, 
                             CppLexerMatchHandler &handler ) = 0;
   };

   typedef boost::shared_ptr<CppLexer> CppLexerPtr;
   
   CppLexerPtr makeCppLexer();
   CppLexerPtr makeCppMacroLexer();

} // namespace Parser


#endif // PARSER_CPPLEXER_H_INCLUDED

--- NEW FILE: preprocessortest.h ---
#ifndef PARSER_PREPROCESSORTEST_H_INCLUDED
# define PARSER_PREPROCESSORTEST_H_INCLUDED

# include "cppnodetestbase.h"


class PreprocessorTest : public CppNodeTestBase
{
public:
   CPPUT_TESTSUITE_BEGIN( PreprocessorTest );
   CPPUT_TEST( testNoMacro );
   CPPUT_TEST( testMacroDefinition );
   CPPUT_TEST( testMacroNoParameterExpansion );
   CPPUT_TEST( testMultiLineNoParameterMacroExpansion );
   CPPUT_TEST( testBracedMacroNoParameterExpansion );
   CPPUT_TEST( testOneParameterMacroExpansion );
   CPPUT_TEST( testThreeParametersMacroExpansion );
   CPPUT_TEST( testNestedMacrosExpansion );
   CPPUT_TEST( testMacroAsMacroParameter );
   CPPUT_TESTSUITE_END();
   
   void setUp();

   void tearDown();

   void testNoMacro();
   void testMacroDefinition();
   void testMacroNoParameterExpansion();
   void testMultiLineNoParameterMacroExpansion();
   void testBracedMacroNoParameterExpansion();
   void testOneParameterMacroExpansion();
   void testThreeParametersMacroExpansion();
   void testNestedMacrosExpansion();
   void testMacroAsMacroParameter();

private:
   void checkPreprocess( const std::string &input,
                         const Parser::Tokens &expected );
};


#endif // PARSER_PREPROCESSORTEST_H_INCLUDED

--- NEW FILE: cppnodetestbase.cpp ---
#include "cppnodetestbase.h"
#include <rfta/cppparser/cppparser.h>



const Parser::Token &
CppNodeTestBase::id_t( const std::string &text )
{
   return addToken( Parser::Token( text, Parser::CppParser::tkIdentifier ) );
}


const Parser::Token &
CppNodeTestBase::spaces_t( int count, char separator )
{
   return addToken( Parser::Token( std::string(count, separator), 
                                   Parser::CppParser::tkSpaces ) );
}


const Parser::Token &
CppNodeTestBase::symb_t( const std::string &text )
{
   return addToken( Parser::Token( text, Parser::CppParser::tkSymbol ) );
}


const Parser::Token &
CppNodeTestBase::int_t( const std::string &text )
{
   return addToken( Parser::Token( text, Parser::CppParser::tkInteger ) );
}


const Parser::Token &
CppNodeTestBase::string_t( const std::string &text )
{
   return addToken( Parser::Token( "\"" + text + "\"", Parser::CppParser::tkString ) );
}

const Parser::Token &
CppNodeTestBase::char_t( const std::string &text )
{
   return addToken( Parser::Token( "'" + text + "'", Parser::CppParser::tkCharacter ) );
}

const Parser::Token &
CppNodeTestBase::float_t( const std::string &text )
{
   return addToken( Parser::Token( text, Parser::CppParser::tkFloat ) );
}

const Parser::Token &
CppNodeTestBase::error_t( const std::string &text )
{
   return addToken( Parser::Token( text, Parser::CppParser::tkError ) );
}

--- NEW FILE: cppnodetestbase.h ---
#ifndef PARSER_CPPNODETESTBASE_H_INCLUDED
# define PARSER_CPPNODETESTBASE_H_INCLUDED

# include "nodetreetestbase.h"
# include "tokentesthelper.h"


class CppNodeTestBase : public NodeTreeTestBase
{
public:
   const Parser::Token &id_t( const std::string &text );
   const Parser::Token &spaces_t( int count, char separator = ' ' );
   const Parser::Token &symb_t( const std::string &text );
   const Parser::Token &string_t( const std::string &text );
   const Parser::Token &char_t( const std::string &text );
   const Parser::Token &int_t( const std::string &text );
   const Parser::Token &float_t( const std::string &text );
   const Parser::Token &error_t( const std::string &text );
};


#endif // PARSER_CPPNODETESTBASE_H_INCLUDED

--- NEW FILE: cpp_grammar.txt ---
# Syntax:
# '#' any on the line for one line comment
# $token_type matches 1 occurence of the specified token type
#             ( @string matches a string, @float a float...)
# '...' matches 1 occurrence of the 'symbol' token type with the specified text
# %'... ...' matches 1 occurrence of one of the symbol specified in the text.
#            Symbols must be separated by exactly one space ( %'true false' will
#            match either the token symbol 'true' or 'false'.
# @node_name( ... ) braced expression tokens become children of a node which
#                   name is specified by 'node_name'.
# $token_type( '...' ) matches 1 occurrence of the specified token with
#                      the specified type.
# list( expression, separator_expression ) predefined parametrized parser
# :rename_last_node( '...' )
# parser_name = ... ';' declare a parser with the specified name matches the expression
#               on the right side of '='. A parser name may be declared only once
# parser_name(.p1,.p2,...) = ... declare a parametrized parser with the specified name 
#               matches the expression on the right side of '='. A parser name may 
#               be declared only once. '.p1' are the expression passed as parameter
#               of the parser.
# parser_name matches the expression specified in the declaration of 'parser_name'.
#             A parser may be referenced before being declared, but all references
#             must be resolved.
# parser_name(p1,p2,...) matched the expression specified in the declaration of
#             the parametrized parser 'parser_name'. p1, p2, ... are passed as parameters.
# ... ... matches the specified sequence.
# ... || ... matches either the left side expression or the right side expression.
#            All expressions are tested for matching and the longuest is kept as
#            a matched (the one that matched the most important number of tokens).
# ... | ... matches either the left side expression or the right side expression.
#           If the left side expression is matched, then the right side expression
#           is not tested for matching.
# '[' ... ']' matches the braced expression.
# ?( ... ) matches 0 or 1 occurence of the braced expression
# +( ... ) matches 1 or more occurrences of the braced expression
# *( ... ) matches 0 or more occurrences of the braced expression
# %('... ...' '... ...') matches 1 occurrence of one of the symbol specified in the text.
#            Symbols must be separated by exactly one space ( %'true false' will
#            match either the token symbol 'true' or 'false'.
# !( ... ) fail if the inner expression is matched
# Operators priority:
# sequence, longuest alternative, first alternative
#
# Commands:
# :node( name, matcher )	create a new node which children are the specified matcher production
# :rename( name )			rename the 'current' node
# :rename_last_child( name )	rename the last child node
# :new_node(name)			create a new node with the specified name. Next matched token will be first child.
# :save_node()				push the current node on the memory stack
# :restore_node()			pop the last node from the memory stack
# :make_penultimate_node_child_of_last()	make the penultimate node child of the last node
# :remove_last_child		remove the last node

::TokenType
  $symbol
  $spaces
  $identifier
  $string
  $character
  $integer
  $float
  $cpp_comment
  $c_comment
  $error

::Grammar

# Additional commands available:
# :cpp_block_error_recovery(): skip all tokens until the end of the statement is found.
#    The end of the statement is:
#    - closing of the current declaration block (non nested '}' found)
#    - a non nested ';' found
#    - no more token available (end of input)

list(.item,.sep) = .item *(.sep .item);

list_min2(.item,.sep) = .item .sep .item *(.sep .item);

binary_operator(.left_operand,.right_operand_node) = 
    .left_operand *( .right_operand_node :make_penultimate_node_child_of_last() )
    ;
    
make_right_operand(.name,.operand) = :node(.name, .operand );

# Used parse sequence where the first element is optional and can not be distinguished
# from the mandatory part.
optional_alternative(.optional, .mandatory) = .optional .mandatory | .mandatory;


# ############################## A.4: Program ############################
# A.4, C++PL3 p.798

translation_unit = :node( 'translation_unit', declaration_seq );


# ############################## A.5: Expressions  ############################
# A.5, C++PL3 p.798-802

literal_expression = :node('literal',
                           $integer
                           | '0'
                           | $character
                           | $string
                           | $float
                           | %'true false' 
                           );

id = $identifier;

namespace_name = id;

class_or_namespace_name = class_name; # | namespace_name;

template_name_specifier = 'template' nested_name_specifier;

nested_name_specifier = +( class_or_namespace_name '::' ?(template_name_specifier) );


# Notes: qualified_id can not include 'template' => A::template B::zozo is not parsed (IT SHOULD BE)
qualified_id = :node( 'qualified_id', 
    ?('::') nested_name_specifier ?('template') unqualified_id
    | [ '::' [ id
             | operator_function_id
             | conversion_function_id ] 
             ] );


# Notes: template id not included
# destructor_ref may be parsed as such while it is not a reference to a destructor (overloaded ~ operator & ())
unqualified_id = id
               | operator_function_id
               | conversion_function_id
               | :node( 'destructor_ref', '~' id ??( '(' ')' ) )
			   ;

id_expression = qualified_id
              || :node( 'unqualified_id', unqualified_id )
              ;

# C++ Standard p65
primary_expression = literal_expression
                   | :node('this_ref', 'this' )
                   | :node('braced_expr', '(' expression ')' )
                   | :node('id_expression', id_expression )
                   ;

# Notes: not included in postfix_expression_p:
# typename with template in scope
#
# postfix_expression_p as been modified to avoid the infinite left recursion. This recursion has been
# moved to multiple 'optional' suffixes
#

array_operator_suffix = :node('array_index', '[' expression ']');

call_operator_suffix = :node('call_parameters', '(' ?( expression_list ) ')');

# Notes: missing template in scope case
pseudo_destructor_name = :node( 'pseudo_destructor_name',
    ?('::') ?(nested_name_specifier) ?(class_name '::') '~' id );    # type_name

accessed_member = pseudo_destructor_name 
                | [ ?( 'template' ) ?('::') id_expression ]
                ;

instance_accessor = %'. ->';
                
member_access_suffix = :node('accessed_member', instance_accessor accessed_member );

post_incdec_suffix = :node( 'post_operator', %'++ --' );


cast_keywords = %'dynamic_cast static_cast reinterpret_cast const_cast';

cppcast_expression = :node('cppcast_expression',
                           cast_keywords '<' type_id '>' '(' expression ')' );

type_id_expression = :node('typeid_expression',
                           'typeid' '(' [type_id | expression] ')' );


# Notes: because of the typeless parsing a function call can not be distinguished from a
# functional type conversion. In the case of a function call, the name of the function will be found in the
# type id.
call_or_conversion_expression = :node( 'call_or_conversion_expression',
    [ ['typename' ?('::') nested_name_specifier id]
      | simple_type_specifier
    ] '(' ?( expression_list ) ')' );

#Notes: template_id construction is missing
postfix_expression = binary_operator(
    cppcast_expression 
        | type_id_expression 
        | call_or_conversion_expression
        | primary_expression,
    make_right_operand( 'array_access', array_operator_suffix )
        | make_right_operand( 'call', call_operator_suffix )
        | make_right_operand( 'member_access', member_access_suffix )
        | make_right_operand( 'post_incdec', post_incdec_suffix ) );


cast_expression = unary_expression
                | :node('cast_expression',
                        '(' type_id ')' :node('casted_expression',
                                              cast_expression ) )
                ;

delete_expression = :node( 'delete_operator',
                           ?( '::') 'delete' ?( '[' ']' ) cast_expression )
                  ;

new...
 
[truncated message content]