Menu
▾
▴
[ST-devel] Patch: printing stack of all threads
|
From: Michael Abd-El-M. <mab...@cm...> - 2005-05-07 23:32:50
|
Hello, This is a patch that enables programmatically dumping the stack of every thread. This has been useful in debugging deadlocks, etc... Our usage model is that the SIGUSR2 handler calls the new _st_print_thread_stacks function, which dumps the stack for all threads. A convenient feature is that for thread stacks that are the same (which is common for application with a lot of worker threads waiting for work), only one stack trace is printed, along with a count of how many threads have that same stack. I use the glibc backtrace function to get the backtrace, and then use popen to execute addr2line and convert memory addresses to file names, function names, and line numbers. If glibc isn't available, _st_print_thread_stacks just prints a warning. And this feature is only available if DEBUG is turned on. We've found this feature extremely helpful when debugging. To see a demo, try running the program: http://abdelmalek.org/test-backtrace.c using StateThreads with the patch below. I've also put the patch and new file at: http://abdelmalek.org/st-patch-1 http://abdelmalek.org/backtrace.c The patch can be a bit more robust (it assumes addr2line exists). But I didn't want to go through the hassle of doing this, if the StateThreads community doesn't want to use this patch. (In our environment, addr2line will always be there.) Cheers, Mike diff -ubr st-1.5.1.clean/Makefile st/Makefile --- st-1.5.1.clean/Makefile 2004-05-14 01:43:40.000000000 -0400 +++ st/Makefile 2005-05-07 16:24:08.000000000 -0400 @@ -247,7 +247,8 @@ $(TARGETDIR)/stk.o \ $(TARGETDIR)/sync.o \ $(TARGETDIR)/key.o \ - $(TARGETDIR)/io.o + $(TARGETDIR)/io.o \ + $(TARGETDIR)/backtrace.o OBJS += $(EXTRA_OBJS) HEADER = $(TARGETDIR)/st.h SLIBRARY = $(TARGETDIR)/libst.a diff -ubr st-1.5.1.clean/common.h st/common.h --- st-1.5.1.clean/common.h 2004-01-11 17:18:15.000000000 -0500 +++ st/common.h 2005-05-07 15:34:17.000000000 -0400 @@ -360,8 +360,18 @@ */ #ifdef DEBUG -void _st_iterate_threads(void); -#define ST_DEBUG_ITERATE_THREADS() _st_iterate_threads() +typedef void(*_st_func_ptr_t)(_st_thread_t *thread, + int start_flag, + int end_flag); +/* Pointer to function that will be called on thread switch */ +extern _st_func_ptr_t _st_iterate_func_ptr; +extern int _st_iterate_threads_flag; +/* Thread iteration function that will call an arbitrary function */ +extern void _st_iterate_threads_helper(_st_func_ptr_t func); +#define ST_DEBUG_ITERATE_THREADS() \ + if (_st_iterate_func_ptr) { \ + _st_iterate_threads_helper(_st_iterate_func_ptr); \ + } #else #define ST_DEBUG_ITERATE_THREADS() #endif @@ -370,6 +380,7 @@ * Switch away from the current thread context by saving its state and * calling the thread scheduler */ + #define _ST_SWITCH_CONTEXT(_thread) \ ST_BEGIN_MACRO \ if (!MD_SETJMP((_thread)->context)) { \ diff -ubr st-1.5.1.clean/public.h st/public.h --- st-1.5.1.clean/public.h 2004-05-14 01:40:30.000000000 -0400 +++ st/public.h 2005-05-07 15:25:53.000000000 -0400 @@ -157,8 +157,10 @@ extern st_netfd_t st_open(const char *path, int oflags, mode_t mode); #ifdef DEBUG -extern void _st_show_thread_stack(st_thread_t thread, const char *messg); +extern void _st_show_thread_stack(st_thread_t thread, int start_flag, + int end_flag); extern void _st_iterate_threads(void); +extern void _st_print_thread_stacks(int ignore); #endif #ifdef __cplusplus diff -ubr st-1.5.1.clean/sched.c st/sched.c --- st-1.5.1.clean/sched.c 2004-05-14 01:40:02.000000000 -0400 +++ st/sched.c 2005-05-07 15:40:22.000000000 -0400 @@ -904,16 +904,13 @@ #ifdef DEBUG -/* ARGSUSED */ -void _st_show_thread_stack(_st_thread_t *thread, const char *messg) -{ - -} - /* To be set from debugger */ int _st_iterate_threads_flag = 0; +/* Thread iteration function that will call an arbitrary function */ +_st_func_ptr_t _st_iterate_func_ptr = NULL; -void _st_iterate_threads(void) +/* This function iterates over all threads, calling "func" for each thread. */ +void _st_iterate_threads_helper(_st_func_ptr_t func) { static _st_thread_t *thread = NULL; static jmp_buf orig_jb, save_jb; @@ -929,16 +926,20 @@ if (thread) { memcpy(thread->context, save_jb, sizeof(jmp_buf)); - _st_show_thread_stack(thread, NULL); + func(thread, 0, 0); } else { if (MD_SETJMP(orig_jb)) { _st_iterate_threads_flag = 0; + _st_iterate_func_ptr = NULL; thread = NULL; - _st_show_thread_stack(thread, "Iteration completed"); + /* Last thread to iterate through */ + func(thread, 0, 1); return; } + /* First thread to iterate through */ thread = _ST_CURRENT_THREAD(); - _st_show_thread_stack(thread, "Iteration started"); + _st_iterate_func_ptr = func; + func(thread, 1, 0); } q = thread->tlink.next; @@ -951,5 +952,18 @@ memcpy(save_jb, thread->context, sizeof(jmp_buf)); MD_LONGJMP(thread->context, 1); } + +/* ARGSUSED */ +void _st_show_thread_stack(_st_thread_t *thread, int start_flag, int end_flag) +{ +} + + +/* Iterate over threads inside debugger; see st/README */ +void _st_iterate_threads(void) +{ + _st_iterate_threads_helper(_st_show_thread_stack); +} + #endif /* DEBUG */ New file: backtrace.c: /* * The contents of this file are subject to the Mozilla Public * License Version 1.1 (the "License"); you may not use this file * except in compliance with the License. You may obtain a copy of * the License at http://www.mozilla.org/MPL/ * * Software distributed under the License is distributed on an "AS * IS" basis, WITHOUT WARRANTY OF ANY KIND, either express or * implied. See the License for the specific language governing * rights and limitations under the License. * * Contributor(s): Michael Abd-El-Malek (mab...@cm...) * Carnegie Mellon University * * Alternatively, the contents of this file may be used under the * terms of the GNU General Public License Version 2 or later (the * "GPL"), in which case the provisions of the GPL are applicable * instead of those above. If you wish to allow use of your * version of this file only under the terms of the GPL and not to * allow others to use your version of this file under the MPL, * indicate your decision by deleting the provisions above and * replace them with the notice and other provisions required by * the GPL. If you do not delete the provisions above, a recipient * may use your version of this file under either the MPL or the * GPL. */ /* * This file contains routines for printing a stack trace of all threads. * Only works when DEBUG is defined and where glibc is available, since it * provides the backtrace() function. */ #define _GNU_SOURCE /* to get program_invocation_name */ #include <stdio.h> #include <stdlib.h> #if defined(DEBUG) && defined(__GLIBC__) #include <errno.h> #include "common.h" #include <execinfo.h> #include <inttypes.h> #include <string.h> /* The maximum number of frames to get a stack trace for. If a thread has more * frames than this, then we only show the latest X frames. */ #define MAX_NUM_FRAMES 64 typedef struct thread_stack_s { uint32_t num_frames; void* addresses[MAX_NUM_FRAMES]; /* frame pointers */ char* locations[MAX_NUM_FRAMES]; /* file/function/line numbers */ uint32_t num_matches; struct thread_stack_s* next; } thread_stack_t; static thread_stack_t* stacks = NULL; /* Converts the function's memory addresses to function names, file names, and * line numbers. Calls binutil's addr2line program. */ static void get_symbol_names(thread_stack_t *stack) { char program_to_run[1024], function[256], filename_lineno[256], temp[19]; FILE* output; int num_bytes_left; uint32_t i; /* Construct the arguments to addr2line */ num_bytes_left = sizeof(program_to_run); num_bytes_left -= snprintf(program_to_run, sizeof(program_to_run), "addr2line -fCe %s", program_invocation_name); for (i = 0; i < stack->num_frames && num_bytes_left > 0; ++i) { num_bytes_left -= snprintf(temp, sizeof(temp), " %p", stack- >addresses[i]); strncat(program_to_run, temp, num_bytes_left); } /* Use popen to execute addr2line and read its ouput */ output = popen(program_to_run, "r"); for (i = 0; i < stack->num_frames; ++i) { char* function_listing = (char*) malloc(512); fscanf(output, "%255s\n", function); fscanf(output, "%255s\n", filename_lineno); snprintf(function_listing, 512, "%s at %s", function, filename_lineno); stack->locations[i] = function_listing; } pclose(output); } static void print_stack(thread_stack_t* stack) { int skip_offset = 0, cmp_len; uint32_t i; /* Get the function names/filenames/line numbers */ get_symbol_names(stack); cmp_len = strlen("_st_iterate_threads_helper"); /* Print the backtrace */ for (i = 0; i < stack->num_frames; ++i) { /* Skip frames we don't have location info for */ if (!strncmp(stack->locations[i], "??", 2)) { continue; } /* Skip the frames that are used for printing the stack trace */ if (skip_offset) { printf("\t#%2d %s %p\n", i - skip_offset, stack- >locations[i], stack->addresses[i]); } else if (!strncmp(stack->locations[i], "_st_iterate_threads_helper", cmp_len)) { skip_offset = i + 1; } } } static void add_current_thread_stack(void) { thread_stack_t *new_stack = malloc(sizeof(thread_stack_t)); thread_stack_t *search; /* Call glibc function to get the backtrace */ new_stack->num_frames = backtrace(new_stack->addresses, MAX_NUM_FRAMES); /* Check if we have another stacks that is equivalent. If so, then coaelsce * two stacks into one, to minimize output to user. */ search = stacks; while (search) { if (search->num_frames == new_stack->num_frames && !memcmp(search->addresses, new_stack->addresses, search->num_frames * sizeof(void*))) { /* Found an existing stack that is the same as this thread's stack */ ++search->num_matches; free(new_stack); return; } else { search = search->next; } } /* This is a new stack. Add it to the list of stacks. */ new_stack->num_matches = 1; new_stack->next = stacks; stacks = new_stack; } static void print_stack_frames(void) { while (stacks) { printf("\n%u thread(s) with this backtrace:\n", stacks- >num_matches); print_stack(stacks); stacks = stacks->next; } printf("\n"); } static void free_stacks(void) { uint32_t i; while (stacks) { thread_stack_t *next = stacks->next; for (i = 0; i < stacks->num_frames; ++i) { free(stacks->locations[i]); } free(stacks); stacks = next; } stacks = NULL; } static void st_print_thread_stack(_st_thread_t *thread, int start_flag, int end_flag) { if (end_flag == 0) { add_current_thread_stack(); } else { print_stack_frames(); } } void _st_print_thread_stacks(int ignore) { _st_iterate_threads_flag = 1; _st_iterate_threads_helper(st_print_thread_stack); _st_iterate_threads_flag = 0; /* Deallocate memory */ free_stacks(); } #else /* defined(DEBUG) && defined(__GLIBC__) */ void _st_print_thread_stacks(int ignore) { printf("%s: need DEBUG mode and glibc-specific functions to read stack.\n", __FUNCTION__); } #endif /* defined(DEBUG) && defined(__GLIBC__) */ |
