I have been playing with the new -K option and like it, thanks!
I have added some extra output with a quick/hacky patch to allow tracking the pre-processors usage of macros and where they are undefined.
This allows post processing to debug macros lifetimes and impact etc.
Specifically I have a quick script that now shows the dependences between the files from the pre-processors point of view.
I added to the -K option the following additional output:
/*u[MACRO_NAME] file:line*/ - indicating macro used in if style statement
/*r[MACRO_NAME] file:line*/ - indicating macro undefined
The file:line pair is not really needed, the chosen letters are not so great, the patch may well not cover all the cases, I am quite certain it break a few things and the code is not that nice... But hopefully the feature is useful and a patch is always better nothing I think.
Let me know what you think
Samuel
ps here is the patch.
Index: src/directive.c
--- src/directive.c (revision 91)
+++ src/directive.c (working copy)
@@ -434,6 +434,13 @@
return FALSE; /* Next token is not an identifier */
}
found = (look_id( identifier) != NULL); /* Look in table */
+
+ if (found && (mcpp_debug & MACRO_CALL)){
+ DEFBUF *defp =look_id( identifier); /* Look in table */
+ if(defp){
+ mcpp_fprintf( DBG, "/*u%s %s:%i*/",defp->name, defp->fname, defp->mline);
+ }
+ }
}
if (found == (hash == L_ifdef)) {
compiling = TRUE;
@@ -1428,6 +1435,11 @@
if (standard && dp->push)
return FALSE; /* 'Pushed' macro */
*prevp = dp->link; /* Link the previous and the next */
+
+ if (mcpp_debug & MACRO_CALL){
+ mcpp_fprintf( DBG, "/*r%s %s:%i*/",dp->name, dp->fname, dp->mline);
+ }
+
free( dp); /* Delete the definition */
if (standard)
num_of_macro--;
Index: src/expand.c
===================================================================
--- src/expand.c (revision 91)
+++ src/expand.c (working copy)
@@ -670,6 +670,10 @@
dump_unget( "replace entry");
}
+ if(in_if && (mcpp_debug & MACRO_CALL)){
+ mcpp_fprintf( DBG, "/*u%s %s:%i*/",defp->name, defp->fname, defp->mline);
+ }
+
enable_trace_macro = trace_macro && defp->nargs != DEF_PRAGMA;
if (enable_trace_macro) {
int num;
Index: src/internal.H
===================================================================
--- src/internal.H (revision 91)
+++ src/internal.H (working copy)
@@ -386,6 +386,7 @@
extern IFINFO ifstack[]; /* Information of #if nesting */
extern char work_buf[];
/* Temporary buffer for directive line and macro expansion */
+extern int in_if; /* flag to say if we need extera debug in -K mode */
expr_t eval_if( void)
/*
@@ -291,17 +292,21 @@
VAL_SIGN * valp = value; /* -> Value and signedness */
OPTAB * opp = opstack; /* -> Operator stack */
int op; /* Current operator */
+ expr_t ret = 0L;
opp->op = OP_END; /* Mark bottom of stack */
opp->prec = opdope[ OP_END]; /* And its precedence */
skip = skip_cur = opp->skip = 0; /* Not skipping now */
+ in_if=1;
+
while (1) {
if (mcpp_debug & EXPRESSION)
mcpp_fprintf( DBG
, "In eval loop skip = %d, binop = %d, line is: %s¥n"
, opp->skip, binop, infile->bptr);
skip = opp->skip;
+
op = eval_lex();
skip = 0; /* Reset to be ready to return */
switch (op) {
@@ -314,7 +319,7 @@
op = OP_PLU; /* Unary plus */
break;
case OP_FAIL:
- return 0L; /* Token error */
+ goto out;
}
if (mcpp_debug & EXPRESSION)
mcpp_fprintf( DBG
@@ -324,11 +329,11 @@
if (binop != 0) { /* Binop is needed */
cerror( "Misplaced constant ¥"%s¥"" /* _E_ */
, work_buf, 0L, NULL);
- return 0L;
+ goto out;
} else if (& value[ NEXP * 2] <= valp) {
cerror( "More than %.0s%ld constants stacked at %s" /* _E_ */
, NULL, (long) (NEXP * 2 - 1), work_buf);
- return 0L;
+ goto out;
} else {
if (mcpp_debug & EXPRESSION) {
dump_val( "pushing ", &ev);
@@ -349,7 +354,7 @@
else
cerror( "Operator ¥"%s¥" in incorrect context" /* _E_ */
, opname[ op], 0L, NULL);
- return 0L;
+ goto out;
}
binop = (prec & 2) >> 1; /* Binop should follow? */
@@ -382,7 +387,7 @@
cerror(
"More than %.0s%ld operators and parens stacked at %s" /* _E_ */
, NULL, (long) (NEXP * 3 - 1), opname[ op]);
- return 0L;
+ goto out;
}
opp->op = op;
opp->prec = prec;
@@ -427,15 +432,17 @@
case OP_END: /* Stack end marker */
if (op == OP_RPA) { /* No corresponding ( */
cerror( "Excessive ¥")¥"", NULL, 0L, NULL); /* _E_ */
- return 0L;
+ goto out;
}
- if (op == OP_EOE)
- return valp[-1].val; /* Finished ok. */
+ if (op == OP_EOE){
+ ret = valp[-1].val; /* Finished ok. */
+ goto out;
+ }
break;
case OP_LPA: /* ( on stack */
if (op != OP_RPA) { /* Matches ) on input? */
cerror( "Missing ¥")¥"", NULL, 0L, NULL); /* _E_ */
- return 0L;
+ goto out;
}
opp--; /* Unstack it */
parens--; /* Count down nest level*/
@@ -448,7 +455,7 @@
cerror(
"Misplaced ¥":¥", previous operator is ¥"%s¥"" /* _E_ */
, opname[opp->op], 0L, NULL);
- return 0L;
+ goto out;
}
/* Evaluate op1. Fall through */
default: /* Others: */
@@ -463,8 +470,9 @@
else
skip = skip_cur;
valp = eval_eval( valp, op1);
- if (valp->sign == VAL_ERROR)
- return 0L; /* Out of range or divide by 0 */
+ if (valp->sign == VAL_ERROR){
+ goto out;
+ }
valp++;
skip = 0;
if (mcpp_debug & EXPRESSION) {
@@ -479,7 +487,10 @@
}
- return 0L; /* Never reach here */
+out:
+ mcpp_fprintf( DBG, "¥n"); /* so the next #line is at the start */
+ in_if=0;
+ return ret;
}
About the proposal from Samuel on additional output of "macro
notification mode":
1. notification on #undef:
This may be of some meaning, I think.
Samuel's implementation reports the file and line of #define. They have
more meaning, if any, than those of #undef, because the file and line of
#undef are self-explanatory. But, the file and line pair may not be
necessary at all.
2. notification on a macro in #if, #elif, #ifdef and #ifndef:
I don't know what use it has. In addition, what should be done on
macros in other directives such as #include or #line, if #if would be
annotated?
One idea is to implement this notification, on the directives other than
#define and #undef, only for -Kv option, that is "verbose" output of
macro notification mode.
Anyway, I would like to hear other users' opinions as well as Samuel.
If you would like to refer to this comment somewhere else in this project, copy and paste the following link:
There is no need to provide file:line info in this case. I specced position info for other nodes because they also provide character positions.
A minor issue: I also think that /*u[MACRO NAME]*/ should be used for undef. I'm not completely sure on what you mean by "if style statement"
if you mean if defined(FOO), similar stuff why not /*i[MACRO NAME]*/.
Other than that I think this is a useful and natural extension to the existing spec.
Samuel, I'm curious if your post-processing tool and/or sample results are available to take a peek at.
If you would like to refer to this comment somewhere else in this project, copy and paste the following link:
Kiyoshi,
Sorry, I didn't see notice your message right away.
#1
I think my previous message addresses most of it. I think it should be triggered by -K.
#2. You raise a good point here. I think annotating #if is useful because that information disappears during preprocessing. I'm guessing it shows which macros triggered the conditional statement.
Your comment about regarding what else should be annotated made me realize that when #if is annotated, #endif should be annotated too(/*#endif*/ ?). This was it will be possible to identify exactly what parts of the code got turned on by preprocessing.
As to #line, #include, etc I think we should leave them alone for now until a user comes up with a good use-case for annotating them like Samuel did here. Both of those options are seen by the parser (unlike macro expansions and conditionals) so annotating them doesn't really make sense to me.
Cheers,
Taras
If you would like to refer to this comment somewhere else in this project, copy and paste the following link:
What I first wanted to be-able to tell was just which header files where not needed for the build. In a c++ tree the size of your headers makes a big difference to the build time.
For that I needed to know not just when a macro is defined and used to produce code but also the #if usage and undef.
I currently have no usage for the endif marker, the conditional code is hard to track. For example adding this marking to the AST produced by the c/c++ parser is too big a job for me to consider, especially when I can think of no good use case...
I think the #line and #include marking is not needed, we already have that in the output stream and in an easy to parse form.
As for the markup, I like the /*u[MACRO_NAME]*/ for undef and /*i[MACRO_NAME]*/ for conditional usage.
Finally as for my "app" its just a quick perl hack, too poor to show but the output looks like this:-
main
PC /storage/masuel/example/main.cpp
C - /storage/masuel/example/XYZMenu.h
C - - /storage/masuel/example/IdMenu.h
PC - /usr/include/stdio.h
P - - /usr/include/features.h
P - - - /usr/include/sys/cdefs.h
- - - - /usr/include/bits/wordsize.h
- - - /usr/include/gnu/stubs.h
- - - - /usr/include/bits/wordsize.h
- - - - /usr/include/gnu/stubs-32.h
PC - - /usr/lib/gcc/i386-redhat-linux/4.1.1/include/stddef.h
C - - /usr/include/bits/types.h
P - - - /usr/include/features.h
- - - /usr/include/bits/wordsize.h
- - - /usr/include/bits/typesizes.h
PC - - /usr/include/libio.h
PC - - - /usr/include/_G_config.h
C - - - - /usr/include/bits/types.h
PC - - - - /usr/lib/gcc/i386-redhat-linux/4.1.1/include/stddef.h
PC - - - - /usr/include/wchar.h
PC - - - - - /usr/lib/gcc/i386-redhat-linux/4.1.1/include/stddef.h
PC - - - /usr/lib/gcc/i386-redhat-linux/4.1.1/include/stdarg.h
- - /usr/include/bits/stdio_lim.h
C - - /usr/include/bits/sys_errlist.h
P - /storage/masuel/example/defines.h
- /storage/masuel/example/indirect.h
C - - /storage/masuel/example/needed.h
C - /storage/masuel/example/static.h
C - /storage/masuel/example/externalfunc.h
Where P indicates that the Pre-processor required the file and C indicates that the compiler did (gcc -fdump-translation-unit, not 100% reliable, will look at Pork!).
It would be nice to output the file inter dependence as a dot graph etc... perhaps another day :)
Samuel
If you would like to refer to this comment somewhere else in this project, copy and paste the following link:
I implemented the additional annotations of "macro notification mode" on
#if, #endif and #undef, revising Samuel's patch, changing its behavior
specifications. Also I updated mcpp-manual.html#3.5.8. And committed
SVN revision 93.
As for the specifications, please refer to mcpp-manual.html#3.5.8. How
about these specs, Samuel, Taras and all?
If you would like to refer to this comment somewhere else in this project, copy and paste the following link:
I revised annotations on #if (#elif, #ifdef, #ifndef), #endif and #undef,
enabled annotation on #else, updated mcpp-manual.html#3.5.8 accordingly,
and committed SVN revision 94. Now all these directives are annotated,
even if no macro is found on the line, unless the line is in a skipped
block. I removed file:line pair in verbose mode of #if annotation for
the sake of symmetry with other directives. Please see mcpp-manual.html
#3.5.8.
At the same time, I fixed a bug of macro notification mode (in
get_an_arg()), which had caused an error on some rare and irregular case
of macro.
I will release mcpp V.2.7 in a few days.
If you would like to refer to this comment somewhere else in this project, copy and paste the following link:
Hi Matsui-san, all,
I have been playing with the new -K option and like it, thanks!
I have added some extra output with a quick/hacky patch to allow tracking the pre-processors usage of macros and where they are undefined.
This allows post processing to debug macros lifetimes and impact etc.
Specifically I have a quick script that now shows the dependences between the files from the pre-processors point of view.
I added to the -K option the following additional output:
/*u[MACRO_NAME] file:line*/ - indicating macro used in if style statement
/*r[MACRO_NAME] file:line*/ - indicating macro undefined
The file:line pair is not really needed, the chosen letters are not so great, the patch may well not cover all the cases, I am quite certain it break a few things and the code is not that nice... But hopefully the feature is useful and a patch is always better nothing I think.
Let me know what you think
Samuel
ps here is the patch.
Index: src/directive.c
--- src/directive.c (revision 91)
+++ src/directive.c (working copy)
@@ -434,6 +434,13 @@
return FALSE; /* Next token is not an identifier */
}
found = (look_id( identifier) != NULL); /* Look in table */
+
+ if (found && (mcpp_debug & MACRO_CALL)){
+ DEFBUF *defp =look_id( identifier); /* Look in table */
+ if(defp){
+ mcpp_fprintf( DBG, "/*u%s %s:%i*/",defp->name, defp->fname, defp->mline);
+ }
+ }
}
if (found == (hash == L_ifdef)) {
compiling = TRUE;
@@ -1428,6 +1435,11 @@
if (standard && dp->push)
return FALSE; /* 'Pushed' macro */
*prevp = dp->link; /* Link the previous and the next */
+
+ if (mcpp_debug & MACRO_CALL){
+ mcpp_fprintf( DBG, "/*r%s %s:%i*/",dp->name, dp->fname, dp->mline);
+ }
+
free( dp); /* Delete the definition */
if (standard)
num_of_macro--;
Index: src/expand.c
===================================================================
--- src/expand.c (revision 91)
+++ src/expand.c (working copy)
@@ -670,6 +670,10 @@
dump_unget( "replace entry");
}
+ if(in_if && (mcpp_debug & MACRO_CALL)){
+ mcpp_fprintf( DBG, "/*u%s %s:%i*/",defp->name, defp->fname, defp->mline);
+ }
+
enable_trace_macro = trace_macro && defp->nargs != DEF_PRAGMA;
if (enable_trace_macro) {
int num;
Index: src/internal.H
===================================================================
--- src/internal.H (revision 91)
+++ src/internal.H (working copy)
@@ -386,6 +386,7 @@
extern IFINFO ifstack[]; /* Information of #if nesting */
extern char work_buf[];
/* Temporary buffer for directive line and macro expansion */
+extern int in_if; /* flag to say if we need extera debug in -K mode */
/* main.c */
extern void un_predefine( int clearall);
Index: src/eval.c
===================================================================
--- src/eval.c (revision 91)
+++ src/eval.c (working copy)
@@ -269,6 +269,7 @@
skip = 0;
}
#endif
+int in_if=0;
expr_t eval_if( void)
/*
@@ -291,17 +292,21 @@
VAL_SIGN * valp = value; /* -> Value and signedness */
OPTAB * opp = opstack; /* -> Operator stack */
int op; /* Current operator */
+ expr_t ret = 0L;
opp->op = OP_END; /* Mark bottom of stack */
opp->prec = opdope[ OP_END]; /* And its precedence */
skip = skip_cur = opp->skip = 0; /* Not skipping now */
+ in_if=1;
+
while (1) {
if (mcpp_debug & EXPRESSION)
mcpp_fprintf( DBG
, "In eval loop skip = %d, binop = %d, line is: %s¥n"
, opp->skip, binop, infile->bptr);
skip = opp->skip;
+
op = eval_lex();
skip = 0; /* Reset to be ready to return */
switch (op) {
@@ -314,7 +319,7 @@
op = OP_PLU; /* Unary plus */
break;
case OP_FAIL:
- return 0L; /* Token error */
+ goto out;
}
if (mcpp_debug & EXPRESSION)
mcpp_fprintf( DBG
@@ -324,11 +329,11 @@
if (binop != 0) { /* Binop is needed */
cerror( "Misplaced constant ¥"%s¥"" /* _E_ */
, work_buf, 0L, NULL);
- return 0L;
+ goto out;
} else if (& value[ NEXP * 2] <= valp) {
cerror( "More than %.0s%ld constants stacked at %s" /* _E_ */
, NULL, (long) (NEXP * 2 - 1), work_buf);
- return 0L;
+ goto out;
} else {
if (mcpp_debug & EXPRESSION) {
dump_val( "pushing ", &ev);
@@ -349,7 +354,7 @@
else
cerror( "Operator ¥"%s¥" in incorrect context" /* _E_ */
, opname[ op], 0L, NULL);
- return 0L;
+ goto out;
}
binop = (prec & 2) >> 1; /* Binop should follow? */
@@ -382,7 +387,7 @@
cerror(
"More than %.0s%ld operators and parens stacked at %s" /* _E_ */
, NULL, (long) (NEXP * 3 - 1), opname[ op]);
- return 0L;
+ goto out;
}
opp->op = op;
opp->prec = prec;
@@ -427,15 +432,17 @@
case OP_END: /* Stack end marker */
if (op == OP_RPA) { /* No corresponding ( */
cerror( "Excessive ¥")¥"", NULL, 0L, NULL); /* _E_ */
- return 0L;
+ goto out;
}
- if (op == OP_EOE)
- return valp[-1].val; /* Finished ok. */
+ if (op == OP_EOE){
+ ret = valp[-1].val; /* Finished ok. */
+ goto out;
+ }
break;
case OP_LPA: /* ( on stack */
if (op != OP_RPA) { /* Matches ) on input? */
cerror( "Missing ¥")¥"", NULL, 0L, NULL); /* _E_ */
- return 0L;
+ goto out;
}
opp--; /* Unstack it */
parens--; /* Count down nest level*/
@@ -448,7 +455,7 @@
cerror(
"Misplaced ¥":¥", previous operator is ¥"%s¥"" /* _E_ */
, opname[opp->op], 0L, NULL);
- return 0L;
+ goto out;
}
/* Evaluate op1. Fall through */
default: /* Others: */
@@ -463,8 +470,9 @@
else
skip = skip_cur;
valp = eval_eval( valp, op1);
- if (valp->sign == VAL_ERROR)
- return 0L; /* Out of range or divide by 0 */
+ if (valp->sign == VAL_ERROR){
+ goto out;
+ }
valp++;
skip = 0;
if (mcpp_debug & EXPRESSION) {
@@ -479,7 +487,10 @@
}
- return 0L; /* Never reach here */
+out:
+ mcpp_fprintf( DBG, "¥n"); /* so the next #line is at the start */
+ in_if=0;
+ return ret;
}
static int eval_lex( void)
@@ -520,6 +531,13 @@
if (scan_token( c, (workp = work_buf, &workp), work_end) == NAM) {
if (warn)
ev.val = (look_id( identifier) != NULL);
+
+ if (mcpp_debug & MACRO_CALL){
+ DEFBUF * defp =look_id( identifier); /* Look in table */
+ if(defp){
+ mcpp_fprintf( DBG, "/*u%s %s:%i*/",defp->name, defp->fname, defp->mline);
+ }
+ }
if (c1 != '(' || skip_ws() == ')') /* Balanced ? */
return VAL; /* Parsed ok */
}
About the proposal from Samuel on additional output of "macro
notification mode":
1. notification on #undef:
This may be of some meaning, I think.
Samuel's implementation reports the file and line of #define. They have
more meaning, if any, than those of #undef, because the file and line of
#undef are self-explanatory. But, the file and line pair may not be
necessary at all.
2. notification on a macro in #if, #elif, #ifdef and #ifndef:
I don't know what use it has. In addition, what should be done on
macros in other directives such as #include or #line, if #if would be
annotated?
One idea is to implement this notification, on the directives other than
#define and #undef, only for -Kv option, that is "verbose" output of
macro notification mode.
Anyway, I would like to hear other users' opinions as well as Samuel.
There is no need to provide file:line info in this case. I specced position info for other nodes because they also provide character positions.
A minor issue: I also think that /*u[MACRO NAME]*/ should be used for undef. I'm not completely sure on what you mean by "if style statement"
if you mean if defined(FOO), similar stuff why not /*i[MACRO NAME]*/.
Other than that I think this is a useful and natural extension to the existing spec.
Samuel, I'm curious if your post-processing tool and/or sample results are available to take a peek at.
Kiyoshi,
Sorry, I didn't see notice your message right away.
#1
I think my previous message addresses most of it. I think it should be triggered by -K.
#2. You raise a good point here. I think annotating #if is useful because that information disappears during preprocessing. I'm guessing it shows which macros triggered the conditional statement.
Your comment about regarding what else should be annotated made me realize that when #if is annotated, #endif should be annotated too(/*#endif*/ ?). This was it will be possible to identify exactly what parts of the code got turned on by preprocessing.
As to #line, #include, etc I think we should leave them alone for now until a user comes up with a good use-case for annotating them like Samuel did here. Both of those options are seen by the parser (unlike macro expansions and conditionals) so annotating them doesn't really make sense to me.
Cheers,
Taras
Hi Kiyoshi-san, Taras,
Nice to get both of your feedback so quickly.
What I first wanted to be-able to tell was just which header files where not needed for the build. In a c++ tree the size of your headers makes a big difference to the build time.
For that I needed to know not just when a macro is defined and used to produce code but also the #if usage and undef.
I currently have no usage for the endif marker, the conditional code is hard to track. For example adding this marking to the AST produced by the c/c++ parser is too big a job for me to consider, especially when I can think of no good use case...
I think the #line and #include marking is not needed, we already have that in the output stream and in an easy to parse form.
As for the markup, I like the /*u[MACRO_NAME]*/ for undef and /*i[MACRO_NAME]*/ for conditional usage.
Finally as for my "app" its just a quick perl hack, too poor to show but the output looks like this:-
main
PC /storage/masuel/example/main.cpp
C - /storage/masuel/example/XYZMenu.h
C - - /storage/masuel/example/IdMenu.h
PC - /usr/include/stdio.h
P - - /usr/include/features.h
P - - - /usr/include/sys/cdefs.h
- - - - /usr/include/bits/wordsize.h
- - - /usr/include/gnu/stubs.h
- - - - /usr/include/bits/wordsize.h
- - - - /usr/include/gnu/stubs-32.h
PC - - /usr/lib/gcc/i386-redhat-linux/4.1.1/include/stddef.h
C - - /usr/include/bits/types.h
P - - - /usr/include/features.h
- - - /usr/include/bits/wordsize.h
- - - /usr/include/bits/typesizes.h
PC - - /usr/include/libio.h
PC - - - /usr/include/_G_config.h
C - - - - /usr/include/bits/types.h
PC - - - - /usr/lib/gcc/i386-redhat-linux/4.1.1/include/stddef.h
PC - - - - /usr/include/wchar.h
PC - - - - - /usr/lib/gcc/i386-redhat-linux/4.1.1/include/stddef.h
PC - - - /usr/lib/gcc/i386-redhat-linux/4.1.1/include/stdarg.h
- - /usr/include/bits/stdio_lim.h
C - - /usr/include/bits/sys_errlist.h
P - /storage/masuel/example/defines.h
- /storage/masuel/example/indirect.h
C - - /storage/masuel/example/needed.h
C - /storage/masuel/example/static.h
C - /storage/masuel/example/externalfunc.h
Where P indicates that the Pre-processor required the file and C indicates that the compiler did (gcc -fdump-translation-unit, not 100% reliable, will look at Pork!).
It would be nice to output the file inter dependence as a dot graph etc... perhaps another day :)
Samuel
I implemented the additional annotations of "macro notification mode" on
#if, #endif and #undef, revising Samuel's patch, changing its behavior
specifications. Also I updated mcpp-manual.html#3.5.8. And committed
SVN revision 93.
As for the specifications, please refer to mcpp-manual.html#3.5.8. How
about these specs, Samuel, Taras and all?
Mainly looks good to me, it works fine for my purpose.
I noticed however that if you do
#if 0
...
#endif
the #endif gets marked up but the #if statement does not (as it has no associated macro).
If you are unconditionally marking #endif we should do the same with #if (and #else) so...
#if 1 should be marked up as /*i lineno*/ ie same as before but with no marco name?
Samuel
ps I will be travailing for the next couple of weeks so will not be able to comment/help out much.
I revised annotations on #if (#elif, #ifdef, #ifndef), #endif and #undef,
enabled annotation on #else, updated mcpp-manual.html#3.5.8 accordingly,
and committed SVN revision 94. Now all these directives are annotated,
even if no macro is found on the line, unless the line is in a skipped
block. I removed file:line pair in verbose mode of #if annotation for
the sake of symmetry with other directives. Please see mcpp-manual.html
#3.5.8.
At the same time, I fixed a bug of macro notification mode (in
get_an_arg()), which had caused an error on some rare and irregular case
of macro.
I will release mcpp V.2.7 in a few days.