How ro resolve cpp symbols from backtrace_symbols() in the offset during runtime for addr2line

Question

To catch fatal errors like Segmentation Fault during runtime I write a custom SignalHandler that will print a stack trace to console and into a log file.

To achieve this I use (as hundreds before me) the backtrace() and backtrace_symbols() functions in combination with addr2line.

A call to backtrace_symbols() produces following output:

Obtained 8 stack frames.
./Mainboard_Software(+0xb1af5) [0x56184991baf5]
./Mainboard_Software(+0xb1a79) [0x56184991ba79]
/lib/x86_64-linux-gnu/libpthread.so.0(+0x12dd0) [0x7fe72948bdd0]
./Mainboard_Software(causeSIGFPE+0x16) [0x561849918a10]
./Mainboard_Software(_Z13MainboardInit7QString+0xf3) [0x56184990e0df]
./Mainboard_Software(main+0x386) [0x5618499182a3]
/lib/x86_64-linux-gnu/libc.so.6(__libc_start_main+0xeb) [0x7fe727fd909b]
./Mainboard_Software(_start+0x2a) [0x5618498ff0aa]

I need to pass the offset to addr2line to get my module name and line number.

$ addr2line -C -a -s -f -p -e ./D098_Mainboard_Software 0xb1a79
0x00000000000b1a79: HandleBacktraceSignals at SignalModule.c:492

However, in some modules (especially cpp ones) I get the offset as a combination off sybols and hex, like _Z13MainboardInit7QString+0xf3

I can resolve the symbol to hex with a call to nm:

$ nm Mainboard_Software | grep _Z13MainboardInit7QString
00000000000a3fec T _Z13MainboardInit7QString

Now I can add these two hex numbers, pass them to addr2line and get my module name and line number, even demangled if I want to:

$ addr2line -C -a -s -f -p -e ./D098_Mainboard_Software 0xa40df
0x00000000000a40df: MainboardInit(QString) at MainboardInit.cpp:219

But I want to do the last two steps during runtime. Is there a way to resolve these symbols (e.g. _Z13MainboardInit7QString+0xf3) during runtime so that I can pass them directly to addr2line? My program consists of both .c and.cpp modules.

Anett

This is not really an answer but why don't you try with gdb? gdb can do all these kinds of stuff at run-time. – Masera

This shall also run on an embedded system, where gbd might or might not be included in the final release – Anett

I see, then, maybe this answer might be helpful – Masera

I also stumbled across this answer, here the address itself is used, not the offset. I, however, need to use the offset since my addresses do not have to correspond to the calls. – Anett

Ok, then my final suggestion is to produce a coredump file when segfault happens. So you can use that file with gdb. If this doesn't work then sorry for not able to help you. – Masera

Answer 1

Took me a while but with Linux, one can use the dlfcn.h GNU library. Just be sure to define _GNU_SOURCE above all header file includes. Beware this include will make your program POSIX nonconform.

For the linker flags add -ldl for both architectures and -g3 for x86 and -g3, -funwind-tables,-mapcs-frame for ARM.

#define _GNU_SOURCE

#include <execinfo.h>
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <string.h>

#include <dlfcn.h>
#include <gnu/lib-names.h>

#define STACK_FRAMES_BUFFERSIZE (int)128

static void * STACK_FRAMES_BUFFER[128];
static void * OFFSET_FRAMES_BUFFER[128];
static char   EXECUTION_FILENAME[32] = "Mainboard_Software";


/*-----------------------------------------------------------------------------------*/
/*
 * Function will attempt to backtrace the signal cause by collecting the last called addresses.
 * The addresses will then be translated into readable stings by addr2line
 */

static void PrintBacktrace(void)
{
   const char errorString[] = "Offset cannot be resolved: No offset present?\n\0?";
   char       printArray[100] = {0};
   size_t     bufferEntries;
   char **    stackFrameStrings;
   size_t     frameIterator;

   //backtrace the last calls
   bufferEntries = backtrace(STACK_FRAMES_BUFFER, STACK_FRAMES_BUFFERSIZE);
   stackFrameStrings = backtrace_symbols(STACK_FRAMES_BUFFER, (int)bufferEntries);

   //print the number of obtained frames
  sprintf(printArray,"\nObtained %zd stack frames.\n\r", bufferEntries);
  (void)write(STDERR_FILENO, printArray, strlen(printArray));

   //iterate over addresses and print the stings
   for (frameIterator = 0; frameIterator < bufferEntries; frameIterator++)
   {
#if __x86_64__
      //calculate the offset on x86_64 and print the file and line number with addr2line
      OFFSET_FRAMES_BUFFER[frameIterator] = CalculateOffset(stackFrameStrings[frameIterator]);
      if(OFFSET_FRAMES_BUFFER[frameIterator] == NULL)
      {
         (void)write(STDERR_FILENO, errorString, strlen(errorString));
      }
      else
      {
         Addr2LinePrint(OFFSET_FRAMES_BUFFER[frameIterator]);
      }
#endif
#if __arm__
      //the address itself can be used on ARM for a call to addr2line
      Addr2LinePrint(STACK_FRAMES_BUFFER[frameIterator]);
#endif
   }
   free (stackFrameStrings);
 }

/*-----------------------------------------------------------------------------------*/
/*
 * Use add2line on the obtained addresses to get a readable sting
 */
static void Addr2LinePrint(void const * const addr)
{
  char addr2lineCmd[512] = {0};

  //have addr2line map the address to the relent line in the code
  (void)sprintf(addr2lineCmd,"addr2line -C -i -f -p -s -a -e ./%s %p ", EXECUTION_FILENAME, addr);

  //This will print a nicely formatted string specifying the function and source line of the address
  (void)system(addr2lineCmd);
}
/*-----------------------------------------------------------------------------------*/
/*
 * Pass a string which was returned by a call to backtrace_symbols() to get the total offset
 * which might be decoded as (symbol + offset). This function will return the calculated offset
 * as void pointer, this pointer can be passed to addr2line in a following call.
 */
void *  CalculateOffset(char * stackFrameString)
{
   void *     objectFile;
   void *     address;
   void *     offset = NULL;
   char       symbolString[75] = {'\0'};
   char       offsetString[25] = {'\0'};
   char *      dlErrorSting;
   int        checkSscanf = EOF;
   int        checkDladdr = 0;
   Dl_info    symbolInformation;

   //parse the string obtained by backtrace_symbols() to get the symbol and offset
   parseStrings(stackFrameString, symbolString, offsetString);

   //convert the offset from a string to a pointer
   checkSscanf = sscanf(offsetString, "%p",&offset);

   //check if a symbol string was created,yes, convert symbol string to offset
   if(symbolString[0] != '\0')
   {
      //open the object (if NULL the executable itself)
      objectFile = dlopen(NULL, RTLD_LAZY);
      //check for error
      if(!objectFile)
      {
         dlErrorSting = dlerror();
         (void)write(STDERR_FILENO, dlErrorSting, strlen(dlErrorSting));
      }
      //convert sting to a address
      address = dlsym(objectFile, symbolString);
      //check for error
      if(address == NULL)
      {
         dlErrorSting = dlerror();
         (void)write(STDERR_FILENO, dlErrorSting, strlen(dlErrorSting));
      }
      //extract the symbolic information pointed by address
      checkDladdr = dladdr(address, &symbolInformation);

      if(checkDladdr != 0)
      {
         //calculate total offset of the symbol
         offset = (symbolInformation.dli_saddr - symbolInformation.dli_fbase) + offset;
         //close the object
         dlclose(objectFile);
      }
      else
      {
         dlErrorSting = dlerror();
         (void)write(STDERR_FILENO, dlErrorSting, strlen(dlErrorSting));
      }
   }

   return checkSscanf != EOF ? offset : NULL;
}
/*-----------------------------------------------------------------------------------*/
/*
 * Parse a string which was returned from backtrace_symbols() to get the symbol name
 * and the offset. 
 */

void parseStrings(char * stackFrameString, char * symbolString, char * offsetString)
{
   char *        symbolStart = NULL;
   char *        offsetStart = NULL;
   char *        offsetEnd = NULL;
   unsigned char stringIterator = 0;

   //iterate over the string and search for special characters
   for(char * iteratorPointer = stackFrameString; *iteratorPointer; iteratorPointer++)
   {
      //The '(' char indicates the beginning of the symbol
      if(*iteratorPointer == '(')
      {
         symbolStart = iteratorPointer;
      }
      //The '+' char indicates the beginning of the offset
      else if(*iteratorPointer == '+')
      {
         offsetStart = iteratorPointer;
      }
      //The ')' char indicates the end of the offset
      else if(*iteratorPointer == ')')
      {
         offsetEnd = iteratorPointer;
      }

   }
   //Copy the symbol string into an array pointed by symbolString
   for(char * symbolPointer = symbolStart+1; symbolPointer != offsetStart; symbolPointer++)
   {
      symbolString[stringIterator] = *symbolPointer;
      ++stringIterator;
   }
   //Reset string iterator for the new array which will be filled
   stringIterator = 0;
   //Copy the offset string into an array pointed by offsetString
   for(char * offsetPointer = offsetStart+1; offsetPointer != offsetEnd; offsetPointer++)
   {
      offsetString[stringIterator] = *offsetPointer;
      ++stringIterator;
   }
}

Calls to this function will produce output like this on console:

Obtained 11 stack frames.
0x00000000000b1ba5: PrintBacktrace at SignalModule.c:524
0x00000000000b1aeb: HandleBacktraceSignals at SignalModule.c:494
0x0000000000012dd0: ?? ??:0
0x00000000000aea85: baz at testFunctions.c:75
0x00000000000aea6b: bar at testFunctions.c:70
0x00000000000aea5f: foo at testFunctions.c:65
0x00000000000aea53: causeSIGSEGV at testFunctions.c:53
0x00000000000a412f: MainboardInit(QString) at MainboardInit.cpp:218
0x00000000000ae2f3: main at Main.cpp:142 (discriminator 2)
0x000000000002409b: ?? ??:0
0x00000000000950fa: _start at ??:?

Anett

Answer 2

You can demangle the symbol run-time by using the library cxxabi:

#include <cxxabi.h>
//...
char *symbolName = "_Z13MainboardInit7QString";
int st;
char* cxx_sname = abi::__cxa_demangle
(
      symbolName,
      nullptr,
      0,
      &st
);

The returned cxx_name array contains the demangled symbol.

The address (base and offset) can be recovered from the initial string by a simple parsing using the brackets as start and end delimiters.

Arcboutant

Hopefully, I understood your answer correctly because I do not need the demangled symbols, but the value hidden beneath the symbols. The offset inside the brackets is, at this time, only useful for me if it contains a hex value and no symbols at all. (My handler can obtain the values inside the brackets) – Anett

Answer 3

Another simple answer:

/*
cc -g -O0 example.c -rdynamic -ldl -o example
*/
#define _GNU_SOURCE
#include <dlfcn.h>
#include <stdio.h>
#include <string.h>
#include <execinfo.h>
#include <stdlib.h>

extern void print_stack();
extern void* parse_symbol_offset(char* frame);
extern char* addr2line_format(void* addr, char* symbol, char* buffer, int nn_buffer);

char* program = NULL;
int main(int argc, char** argv)
{
    program = argv[0];
    print_stack();
    return 0;
}

void print_stack()
{
    void* addresses[64];
    int nn_addresses = backtrace(addresses, sizeof(addresses) / sizeof(void*));
    printf("%s addresses:\n", program);
    for (int i = 0; i < nn_addresses; i++) {
        printf("%p\n", addresses[i]);
    }

    char** symbols = backtrace_symbols(addresses, nn_addresses);
    printf("\nsymbols:\n");
    for (int i = 0; i < nn_addresses; i++) {
        printf("%s\n", symbols[i]);
    }

    char buffer[128];
    printf("\nframes:\n");
    for (int i = 0; i < nn_addresses; i++) {
        void* frame = parse_symbol_offset(symbols[i]);
        char* fmt = addr2line_format(frame, symbols[i], buffer, sizeof(buffer));
        int parsed = (fmt == buffer);
        printf("%p %d %s\n", frame, parsed, fmt);
    }

    free(symbols);
}

void* parse_symbol_offset(char* frame)
{
    char* p = NULL;
    char* p_symbol = NULL;
    int nn_symbol = 0;
    char* p_offset = NULL;
    int nn_offset = 0;

    // Read symbol and offset, for example:
    //      /tools/backtrace(foo+0x1820) [0x555555555820]
    for (p = frame; *p; p++) {
        if (*p == '(') {
            p_symbol = p + 1;
        } else if (*p == '+') {
            if (p_symbol) nn_symbol = p - p_symbol;
            p_offset = p + 1;
        } else if (*p == ')') {
            if (p_offset) nn_offset = p - p_offset;
        }
    }
    if (!nn_symbol && !nn_offset) {
        return NULL;
    }

    // Convert offset(0x1820) to pointer, such as 0x1820.
    char tmp[128];
    if (!nn_offset || nn_offset >= sizeof(tmp)) {
        return NULL;
    }

    int r0 = EOF;
    void* offset = NULL;
    tmp[nn_offset] = 0;
    if ((r0 = sscanf(strncpy(tmp, p_offset, nn_offset), "%p", &offset)) == EOF) {
        return NULL;
    }

    // Covert symbol(foo) to offset, such as 0x2fba.
    if (!nn_symbol || nn_symbol >= sizeof(tmp)) {
        return offset;
    }

    void* object_file;
    if ((object_file = dlopen(NULL, RTLD_LAZY)) == NULL) {
        return offset;
    }

    void* address;
    tmp[nn_symbol] = 0;
    if ((address = dlsym(object_file, strncpy(tmp, p_symbol, nn_symbol))) == NULL) {
        dlclose(object_file);
        return offset;
    }

    Dl_info symbol_info;
    if ((r0 = dladdr(address, &symbol_info)) == 0) {
        dlclose(object_file);
        return offset;
    }

    dlclose(object_file);
    return symbol_info.dli_saddr - symbol_info.dli_fbase + offset;
}

char* addr2line_format(void* addr, char* symbol, char* buffer, int nn_buffer)
{
    char cmd[512] = {0};
    int r0 = snprintf(cmd, sizeof(cmd), "addr2line -C -p -s -f -a -e %s %p", program, addr);
    if (r0 < 0 || r0 >= sizeof(cmd)) return symbol;

    FILE* fp = popen(cmd, "r");
    if (!fp) return symbol;

    char* p = fgets(buffer, nn_buffer, fp);
    pclose(fp);

    if (p == NULL) return symbol;
    if ((r0 = strlen(p)) == 0) return symbol;

    // Trait the last newline if exists.
    if (p[r0 - 1] == '\n') p[r0 - 1] = '\0';

    // Find symbol not match by addr2line, like
    //      0x0000000000021c87: ?? ??:0
    //      0x0000000000002ffa: _start at ??:?
    for (p = buffer; p < buffer + r0 - 1; p++) {
        if (p[0] == '?' && p[1] == '?') return symbol;
    }

    return buffer;
}

Build and run:

cc -g -O0 example.c -rdynamic -ldl -o example
./example

The result is:

./example addresses:
0x559be0516e06
0x559be0516dd1
0x7f6374edec87
0x559be0516cca

symbols:
./example(print_stack+0x2e) [0x559be0516e06]
./example(main+0x27) [0x559be0516dd1]
/lib/x86_64-linux-gnu/libc.so.6(__libc_start_main+0xe7) [0x7f6374edec87]
./example(_start+0x2a) [0x559be0516cca]

frames:
0xe06 1 0x0000000000000e06: print_stack at example.c:26
0xdd1 1 0x0000000000000dd1: main at example.c:20
0x21c87 0 /lib/x86_64-linux-gnu/libc.so.6(__libc_start_main+0xe7) [0x7f6374edec87]
0xcca 0 ./example(_start+0x2a) [0x559be0516cca]

Innes