quickstack.cc

#include "quickstack.h"

#include <linux/version.h>
#include <asm/unistd.h>
#include <getopt.h>
#include <sys/mman.h>

#include <sstream>

#if defined(__i386__)
#define STACK_IP eip
#define STACK_SP esp
#elif defined(__x86_64__)
#define STACK_IP rip
#define STACK_SP rsp
#elif defined(__aarch64__)
#define STACK_IP pc
#define STACK_SP sp
#else
#error "unsupported cpu arch"
#endif

#if !defined(__linux__)
#error "unsupported os"
#endif

const char* version = "0.10";
int target_pid = 0;
int debug_level = 2;
int debug_print_time_level = 10;
int frame_check = 0;
int flush_log = 3;
int timeout_seconds = 600;
bool lock_all = false;

const char* debug_dir = "/usr/lib/debug";
int* _attach_started = nullptr;
stopper_symbol stopper[3] = {
    {"main", 0, 0}, {"start_thread", 0, 0}, {"do_sigwait", 0, 0}};
int num_stopper_symbol = 3;
string basic_libs[10] = {"ld-",
                         "libaio.",
                         "libc-",
                         "libm-",
                         "libdl-",
                         "libpthread-",
                         "librt-",
                         "libgcc_",
                         "libcrypt-",
                         "libnss_"
                         "libnsl_"
                         "libstdc++"};
int num_basic_libs = 10;
volatile sig_atomic_t shutdown_program = 0;
int print_arg = 0;
int single_line = 0;
int thread_names = 0;
int trace_multiple_procs = 0;
int basename_only = 0;
int max_ptrace_calls = 1000;
int max_frame_size = 16384 * sizeof(long);
const char* stack_out;
FILE* stack_out_fp;

static void set_shutdown(int) { shutdown_program = 1; }

static void init_signals() {
  int signals[] = {SIGHUP, SIGINT, SIGTERM};
  for (uint i = 0; i < sizeof(signals) / sizeof(int); i++)
    signal(signals[i], set_shutdown);
  return;
}

static void ignore_signals() {
  int signals[] = {SIGHUP, SIGINT, SIGTERM};
  for (uint i = 0; i < sizeof(signals) / sizeof(int); i++)
    signal(signals[i], SIG_IGN);
  return;
}

void print_log(int level, const char* format, ...) {
  if (level < debug_print_time_level) {
    struct timeval tv;
    time_t tt;
    struct tm *tm;
    gettimeofday(&tv, 0);
    tt = tv.tv_sec;
    tm = localtime(&tt);
    fprintf(stdout,
            "%04d-%02d-%02d %02d:%02d:%02d %06ld: ",
            tm->tm_year + 1900,
            tm->tm_mon + 1,
            tm->tm_mday,
            tm->tm_hour,
            tm->tm_min,
            tm->tm_sec,
            tv.tv_usec);
  } else {
    fprintf(stdout, "                            ");
  }

  va_list args;
  va_start(args, format);
  vfprintf(stdout, format, args);
  va_end(args);
  fprintf(stdout, "\n");
  if (flush_log >= level)
    fflush(stdout);
}

void print_stack(const char* format, ...) {
  va_list args;
  va_start(args, format);
  if (stack_out_fp) {
    vfprintf(stack_out_fp, format, args);
    fflush(stack_out_fp);
  } else {
    vfprintf(stdout, format, args);
    fflush(stdout);
  }
  va_end(args);
}

static string dirname(const string& path) {
  return path.substr(0, path.find_last_of('/'));
}

static string basename(const string& path) {
  return path.substr(path.find_last_of('/') + 1);
}

static bool startwith(const string& fullstring, const string& starting) {
  if (fullstring.length() >= starting.length()) {
    if (!fullstring.compare(0, starting.length(), starting))
      return true;
  } else {
    return false;
  }
  return false;
}

static bool endwith(const string& fullstring, const string& ending) {
  if (fullstring.length() >= ending.length()) {
    return (0 ==
            fullstring.compare(fullstring.length() - ending.length(),
                               ending.length(),
                               ending));
  } else {
    return false;
  }
  return false;
}

static int is_pid_stopped(int pid) {
  FILE* status_file;
  char buf[100];
  int retval = 0;

  snprintf(buf, sizeof(buf), "/proc/%d/status", (int)pid);
  status_file = fopen(buf, "r");
  if (status_file != NULL) {
    int have_state = 0;
    while (fgets(buf, sizeof(buf), status_file)) {
      buf[strlen(buf) - 1] = '\0';
      if (strncmp(buf, "State:", 6) == 0) {
        have_state = 1;
        break;
      }
    }
    if (have_state && strstr(buf, "T") != NULL) {
      DBG(9, "Process %d %s", pid, buf);
      retval = 1;
    }
    fclose(status_file);
  }
  return retval;
}

static bool match_debug_file(const string& name, const char* file) {
  string ptr = file;
  string ending = ".debug";

  if (name.empty() || ptr.empty())
    return false;
  if (!startwith(ptr, name))
    return false;
  if (name.length() == ptr.length())
    return true;

  ptr = ptr.substr(name.length());
  if (!endwith(ptr, ending))
    return false;
  ptr = ptr.substr(0, ptr.length() - ending.length());
  char* work = (char*)ptr.c_str();
  while (*work) {
    if ((*work >= 0 && *work <= 9) || *work == '.')
      work++;
    else
      return false;
  }
  return true;
}

static int get_user_regs(int pid, struct iovec& iovec) {
  int count = 100;
  while (1) {
    int e = ptrace(PTRACE_GETREGSET, pid, NT_PRSTATUS, &iovec);
    if (e != 0) {
      if (errno == ESRCH && count-- > 0) {
        sched_yield();
        continue;
      }
      perror("ptrace(PTRACE_GETREGSET)");
      return -1;
    }
    break;
  }
  return 0;
}

static char* find_debug_file(const char* stripped_file) {
  string path;
  path = debug_dir;
  path += stripped_file;
  string dirname1 = dirname(path);
  string name = basename(path);

  string real_debug_file;
  DIR* dp = opendir(dirname1.c_str());
  if (dp) {
    struct dirent* dent;
    do {
      dent = readdir(dp);
      if (dent) {
        if (dent->d_name[0] == '.') {
          continue;
        }
        if (match_debug_file(name, dent->d_name)) {
          real_debug_file = dirname1;
          real_debug_file += "/";
          real_debug_file += dent->d_name;
          break;
        }
      }
    } while (dent);
    closedir(dp);
  }
  return strdup(real_debug_file.c_str());
}

void bfd_handle::free_syms_if() {
  if (syms != NULL) {
    free(syms);
    syms = NULL;
  }
}

void bfd_handle::free_st_if() {
  if (st != NULL) {
    delete st;
    st = NULL;
  }
}

bool bfd_handle::has_debug() const { return has_debug_symbols; }

int bfd_handle::get_file_line(ulong addr,
                              const char** file,
                              const char** function,
                              uint* lineno) const {
  return bfd_find_nearest_line(
      abfd, dbgsec, syms, addr - 1, file, function, lineno);
}

void bfd_handle::close_all() {
  free_st_if();
  free_syms_if();
  close();
  if (debug_file)
    free(debug_file);
}

void bfd_handle::init(const char* file = nullptr) {
  abfd = bfd_openr(file ? file : filename, NULL);
  if (!abfd) {
    fprintf(stderr, "Failed at bfd_openr! %s\n", file ? file : filename);
    exit(1);
  }
  bfd_check_format(abfd, bfd_object);
  assert((bfd_get_file_flags(abfd) & HAS_SYMS) != 0);
}

void bfd_handle::debug_init() {
  close();
  init(debug_file);
}

bfd* bfd_handle::get() { return abfd; }

void bfd_handle::close() {
  if (abfd)
    bfd_close(abfd);
  abfd = NULL;
}

void load_stopper_symbols(symbol_table* sorted_st,
                          bool relative,
                          ulong addr_begin) {
  int matched_stopper = -1;
  for (symbol_table::symbols_type::iterator i = sorted_st->symbols.begin();
       i != sorted_st->symbols.end();
       i++) {
    int j = 0;
    if (matched_stopper >= 0) {
      stopper[matched_stopper].addr_end = i->addr;
      if (relative) {
        stopper[matched_stopper].addr_end += addr_begin;
      }
      matched_stopper = -1;
    }
    for (j = 0; j < num_stopper_symbol; j++) {
      if (stopper[j].name == i->name) {
        stopper[j].addr_begin = i->addr;
        if (relative) {
          stopper[j].addr_begin += addr_begin;
        }
        matched_stopper = j;
      }
    }
  }
}

void bfd_handle::load_symbols(bool relative, ulong addr_begin) {
  st = new symbol_table();
  load_debug_section_if();
  asection* const text_sec = bfd_get_section_by_name(abfd, ".text");
  if (text_sec) {
    st->text_vma = text_sec->vma;
    st->text_size = text_sec->size;
  }

  symcnt = bfd_read_minisymbols(abfd, 0, (void**)&syms, &size);
  if (!symcnt) {
    free_syms_if();
    symcnt = bfd_read_minisymbols(abfd, 1, (void**)&syms, &size);
    dynamic = 1;
  }

  asymbol* store = bfd_make_empty_symbol(abfd);
  assert(store);

  bfd_byte* p = (bfd_byte*)syms;
  bfd_byte* pend = p + symcnt * size;
  for (; p < pend; p += size) {
    asymbol* sym = bfd_minisymbol_to_symbol(abfd, dynamic, p, store);
    if ((sym->flags & BSF_FUNCTION) == 0) {
      continue;
    }
    symbol_info sinfo;
    bfd_get_symbol_info(abfd, sym, &sinfo);
    if (sinfo.type != 'T' && sinfo.type != 't' && sinfo.type != 'W' &&
        sinfo.type != 'w') {
      continue;
    }
    DBG(40, "%s %lx f=%x", sinfo.name, (long)sinfo.value, (int)sym->flags);

    if (startwith(sinfo.name, "__tcf"))
      continue;
    if (startwith(sinfo.name, "__tz"))
      continue;

    symbol_ent e;
    e.addr = sinfo.value;
    e.name = string(sinfo.name);
    st->symbols.push_back(e);
  }
  std::sort(st->symbols.begin(), st->symbols.end(), std::less<symbol_ent>());
  load_stopper_symbols(st, relative, addr_begin);
}

void bfd_handle::load_debug_section_if() {
  dbgsec = bfd_get_section_by_name(abfd, ".debug_info");
  if (!dbgsec) {
    debug_file = find_debug_file(filename);
    if (debug_file && strlen(debug_file) > 0) {
      DBG(3, "Reading debug file %s, original(%s) ..", debug_file, filename);
      debug_init();
      dbgsec = bfd_get_section_by_name(abfd, ".debug_info");
    }
  }
  if (dbgsec) {
    DBG(3, "has debug symbols on %s", debug_file ? debug_file : filename);
    has_debug_symbols = true;
  } else {
    DBG(1, "No debug symbols found on %s", filename);
    has_debug_symbols = false;
  }
}

static bool check_shlib(const std::string& fn) {
  auto_fp fp(fopen(fn.c_str(), "r"));
  if (fp == 0) {
    return false;
  }
  elf_version(EV_CURRENT);
  Elf* elf = elf_begin(fileno(fp), ELF_C_READ, nullptr);
  if (elf == 0) {
    return false;
  }
  ulong vaddr = 0;
#if defined(__i386__)
  Elf32_Ehdr* const ehdr = elf32_getehdr(elf);
  Elf32_Phdr* const phdr = elf32_getphdr(elf);
#else
  Elf64_Ehdr* const ehdr = elf64_getehdr(elf);
  Elf64_Phdr* const phdr = elf64_getphdr(elf);
#endif
  const int num_phdr = ehdr->e_phnum;
  for (int i = 0; i < num_phdr; ++i) {
#if defined(__i386__)
    Elf32_Phdr* const p = phdr + i;
#else
    Elf64_Phdr* const p = phdr + i;
#endif
    if (p->p_type == PT_LOAD && (p->p_flags & 1) != 0) {
      vaddr = p->p_vaddr;
      break;
    }
  }
  elf_end(elf);
  return vaddr == 0;
}

static bool file_exists(const char* filename) {
  if (FILE* file = fopen(filename, "r")) {
    fclose(file);
    return true;
  }
  return false;
}

static bool has_exec_permission(const char* filename) {
  struct stat results;
  stat(filename, &results);
  if (results.st_mode & S_IXUSR)
    return true;
  return false;
}

static void read_proc_map_ent(const string& line,
                              proc_info& pinfo,
                              symbol_table_map* stmap) {
  bool delete_marked = false;
  std::istringstream line_sin(line);
  string tok;
  vector<string> tokens;
  while (line_sin >> tok) {
    tokens.push_back(tok);
  }
  if (tokens.size() < 6) {
    return;
  } else if (tokens.size() > 6 && startwith(tokens[6], "(deleted")) {
    delete_marked = true;
  }
  const string& perms = tokens[1];
  if (perms.size() == 5 && perms[3] != 'x') {
    return;
  }
  ulong a0 = 0, a1 = 0;
  sscanf(line.c_str(), "%lx-%lx", &a0, &a1);

  proc_map_ent e;

  e.addr_begin = a0;
  e.addr_size = a1 - a0;
  e.offset = std::stol(tokens[2], nullptr, 16);
  e.path = tokens[5];
  if (e.path.empty()) {
    return;
  } else if (e.path == "[vdso]") {
    e.is_vdso = true;
  } else if (startwith(e.path, "[")) {
    return;
  } else {
    if (delete_marked && !file_exists(e.path.c_str()))
      return;
    if (!has_exec_permission(e.path.c_str()))
      return;
    e.stbl = stmap->get(e.path.c_str());
    e.relative = check_shlib(e.path);
    if (!e.stbl) {
      bfd_handle* bh = new bfd_handle(e.path.c_str());
      bh->init();
      bh->load_symbols(e.relative, e.addr_begin);
      stmap->set(e.path.c_str(), bh->st);
      e.stbl = bh->st;
      e.stbl->bh = bh;
    }
  }
  DBG(10,
      "%s: relative=%d addr_begin=%016lx",
      e.path.c_str(),
      (int)e.relative,
      e.addr_begin);
  pinfo.maps.push_back(e);
}

static void read_proc_maps(int pid, proc_info& pinfo, symbol_table_map* stmap) {
  const string maps_file_path = "/proc/" + std::to_string(pid) + "/maps";
  std::ifstream maps_file(maps_file_path);
  if (!maps_file.is_open()) {
    return;
  }
  string line;
  while (std::getline(maps_file, line)) {
    read_proc_map_ent(line, pinfo, stmap);
  }
  std::sort(pinfo.maps.begin(), pinfo.maps.end(), std::less<proc_map_ent>());
}

static const symbol_ent* find_symbol(const symbol_table* st,
                                     ulong addr,
                                     bool& is_text_r,
                                     ulong& pos_r,
                                     ulong& offset_r) {
  is_text_r = false;
  pos_r = 0;
  offset_r = 0;
  const symbol_table::symbols_type& ss = st->symbols;
  symbol_table::symbols_type::const_iterator j =
      std::upper_bound(ss.begin(), ss.end(), symbol_ent(addr));
  if (j != ss.begin()) {
    --j;
  } else {
    return 0;
  }
  if (j == ss.end()) {
    return 0;
  }
  is_text_r =
      (j->addr >= st->text_vma && j->addr < st->text_vma + st->text_size);
  pos_r = j - ss.begin();
  offset_r = addr - j->addr;
  return &*j;
}

static bool match_basic_lib(const string& path) {
  for (int i = 0; i < num_basic_libs; i++) {
    if (startwith(basename(path), basic_libs[i])) {
      return true;
    }
  }
  return false;
}

static int pinfo_symbol_exists(const proc_info& pinfo, ulong addr) {
  /* 0: not exists, 1: core library, 2: others */
  int symbol_type = 0;
  proc_info::maps_type::const_iterator i = std::upper_bound(
      pinfo.maps.begin(), pinfo.maps.end(), proc_map_ent(addr));
  if (i != pinfo.maps.begin()) {
    --i;
  } else {
    i = pinfo.maps.end();
  }
  if (i == pinfo.maps.end()) {
    DBG(30, "%lx not found", addr);
  } else if (addr >= i->addr_begin + i->addr_size) {
    DBG(30,
        "%lx out of range [%lx %lx]",
        addr,
        i->addr_begin,
        i->addr_begin + i->addr_size);
  } else if (!i->stbl || !i->stbl->bh) {
    DBG(30, "%lx no symbol found", addr);
  } else {
    symbol_type = 2;
    if (match_basic_lib(i->path)) {
      symbol_type = 1;
      DBG(10, "Matched basic libs %s", i->path.c_str());
    }
  }
  return symbol_type;
}

static const symbol_ent* pinfo_find_symbol(
    const proc_info& pinfo,
    const ulong addr,
    ulong& offset_r,
    ulong& maps_offset,
    ulong& relative_addr,
    const bool ignore_basic_libs = false) {
  offset_r = 0;
  proc_info::maps_type::const_iterator i = std::upper_bound(
      pinfo.maps.begin(), pinfo.maps.end(), proc_map_ent(addr));
  if (i != pinfo.maps.begin()) {
    --i;
  } else {
    i = pinfo.maps.end();
  }
  if (i == pinfo.maps.end()) {
    DBG(30, "%lx not found", addr);
  } else if (addr >= i->addr_begin + i->addr_size) {
    DBG(30,
        "%lx out of range [%lx %lx]",
        addr,
        i->addr_begin,
        i->addr_begin + i->addr_size);
  } else if (!i->stbl || !i->stbl->bh) {
    DBG(30, "%lx no symbol found", addr);
    if (i->is_vdso) {
      offset_r = addr - i->addr_begin;
    }
  } else {
    if (ignore_basic_libs) {
      if (match_basic_lib(i->path)) {
        DBG(10, "Matched basic libs2 %s", i->path.c_str());
        return nullptr;
      }
    }
    ulong a = addr;
    if (i->relative) {
      a -= i->addr_begin;
      relative_addr = a;
    }
    ulong pos = 0;
    ulong offset = 0;
    bool is_text = false;
    const symbol_ent* const e = find_symbol(i->stbl, a, is_text, pos, offset);
    if (e != 0 && is_text) {
      offset_r = offset;
      maps_offset = i - pinfo.maps.begin();
      return e;
    }
  }
  return nullptr;
}

static bool is_stopper_addr(ulong addr) {
  bool is_stopper = false;
  int i;
  for (i = 0; i < num_stopper_symbol; i++) {
    if (stopper[i].addr_begin < addr && stopper[i].addr_end > addr) {
      is_stopper = true;
      break;
    }
  }
  return is_stopper;
}

static int get_stack_trace(int pid,
                           proc_info& pinfo,
                           uint maxlen,
                           const user_regs_struct& regs,
                           std::vector<ulong>& vals_r) {
  uint i = 0;
  // candidate for the next stack address
  ulong next_likely_sp = 0;
  // address to start re-scanning if getting invalid addresses
  ulong rollback_addr = 0;
  bool matched_top_sp = false;
  uint error_count = 0;
  uint n_frames = 0;
  uint n_scanned = 0;
  bool top_addr_is_user_func = 0;
  uint n_scanned_from_last_frame = 0;
  bool sp_jumped = false;

  ulong sp = regs.STACK_SP;
  ulong top_addr = regs.STACK_IP;
  ulong top_sp = sp;
  // needs special care for second addr
  ulong second_addr = 0;

  DBG(10, "Top addr %016lx", top_addr);
  DBG(10, "Top sp: %016lx", sp);

  vals_r.push_back(top_addr);
  n_frames++;
  if (is_stopper_addr(top_addr)) {
    DBG(3, "Matched stopper func.", 0);
    return 0;
  }

  if (pinfo_symbol_exists(pinfo, top_addr) == 2) {
    DBG(10, "Top addr is user func.", "");
    top_addr_is_user_func = true;
  }

  for (i = 0; i < maxlen; ++i) {
    ulong retaddr = 0;
    retaddr = ptrace(PTRACE_PEEKDATA, pid, sp, 0);
    if (errno != 0) {
      DBG(3, "Got error %d on PTRACE_PEEKDATA", errno);
      break;
    }
    n_scanned++;
    n_scanned_from_last_frame++;
    DBG(10, "SP %016lx", sp);
    DBG(10, "Got addr %016lx", retaddr);
    DBG(10, "Frame count %d", n_frames);
    sp += sizeof(long);
    if (next_likely_sp < sp) {
      next_likely_sp = 0;
    }
    if (retaddr == 0) {
      continue;
    }

    if (retaddr == top_sp) {
      DBG(10,
          "Addr matched top sp %016lx. "
          "Dropping previously scanned blocks.",
          retaddr);
      matched_top_sp = true;
      if (n_frames >= 2) {
        vals_r.clear();
        vals_r.push_back(top_addr);
        n_scanned_from_last_frame = 0;
        n_frames = 1;
        sp_jumped = false;
      }
    }
    if (pinfo.maps.empty()) {
      continue;
    }
    int symbol_type = pinfo_symbol_exists(pinfo, retaddr);
    DBG(10, "Got symbol type %d", symbol_type);

    if (symbol_type) {
      error_count = 0;
      if (frame_check && symbol_type == 2 && n_scanned_from_last_frame >= 3 &&
          ((!matched_top_sp && n_scanned_from_last_frame >= 10) ||
           n_frames >= 2 || top_addr_is_user_func) &&
          !next_likely_sp) {
        DBG(10, "Non target addr %016lx", retaddr);
      } else {
        if (next_likely_sp) {
          DBG(10, "Jumping to next likely sp %016lx", next_likely_sp);
          rollback_addr = sp - sizeof(long);
          sp_jumped = true;
          sp = next_likely_sp;
        }
        n_frames++;
        n_scanned_from_last_frame = 0;
        DBG(10, "Pushed addr %016lx", retaddr);
        vals_r.push_back(retaddr);
        next_likely_sp = 0;
        if (n_frames == 2 && !second_addr) {
          DBG(10, " This is second frame addr.");
          second_addr = retaddr;
        }
      }
    } else {
      error_count++;
      if (retaddr >= sp + sizeof(long) && retaddr < sp + max_frame_size) {
        DBG(10, "retaddr is in sp range", "");
        error_count = 0;
      } else if (frame_check && error_count >= 2) {
        next_likely_sp = 0;
        if (rollback_addr && sp_jumped) {
          DBG(10,
              "Previous next likely sp was invalid."
              "Scanning from %016lx",
              rollback_addr);
          sp = rollback_addr;
          rollback_addr = 0;
          // n_scanned= n_scanned - error_count;
          error_count = 0;
          sp_jumped = false;
          if (!matched_top_sp || n_frames >= 2) {
            vals_r.clear();
            vals_r.push_back(top_addr);
            n_scanned_from_last_frame = 0;
          }
          n_frames = 1;
          continue;
        }
      }
    }
    if (is_stopper_addr(retaddr)) {
      DBG(3, "Matched stopper func.", 0);
      if (vals_r.size() == 2 && second_addr && second_addr != retaddr) {
        DBG(10, "Putting second_addr.");
        vals_r.pop_back();
        vals_r.push_back(second_addr);
        vals_r.push_back(retaddr);
      }
      break;
    }
    bool do_rollback = false;
    if (retaddr >= sp + sizeof(long) && retaddr < sp + max_frame_size) {
      if (next_likely_sp && n_frames <= 2) {
        DBG(10,
            "Finding Next Likely sp consecutive times. "
            "Previously pushed addr was invalid");
        do_rollback = true;
      }
      next_likely_sp = retaddr;
      if (retaddr > sp + sizeof(long)) {
        sp_jumped = true;
      } else {
        sp_jumped = false;
      }
      DBG(10, "Next Likely SP: %016lx", next_likely_sp);
    }
    if (sp_jumped && frame_check) {
      if (n_scanned_from_last_frame >= 5 && n_frames >= 3) {
        DBG(10,
            "Scanned %d blocks from the last frame. This is invalid.",
            n_scanned_from_last_frame);
        do_rollback = true;
      } else if (symbol_type == 1 && n_frames >= 3 &&
                 !is_stopper_addr(retaddr)) {
        DBG(10,
            "Matched core lib symbol type in the middle of stack frames. "
            "This is invalid.");
        do_rollback = true;
      }
    }
    if (do_rollback) {
      if (rollback_addr) {
        DBG(10, "Scanning from %016lx", rollback_addr);
        sp = rollback_addr;
        rollback_addr = 0;
      }
      vals_r.clear();
      vals_r.push_back(top_addr);
      n_scanned_from_last_frame = 0;
      n_frames = 1;
      sp_jumped = false;
    }
  }
  DBG(3, "Scanned %d blocks.", n_scanned);
  return 0;
}

char* get_demangled_symbol(const char* symbol_name) {
  uint arg = DMGL_ANSI;
  if (print_arg) {
    arg |= DMGL_PARAMS;
    arg |= DMGL_TYPES;
  }
  char* demangled = cplus_demangle(symbol_name, arg);
  if (!demangled) {
    demangled = strdup(symbol_name);
  }
  return demangled;
}

void parse_stack_trace(const proc_info& pinfo,
                       const std::vector<ulong>& vals_sps,
                       size_t maxlen) {
  char buf[128];
  uint rank = 1;
  std::string rstr;

  for (size_t i = 0; i < std::min(vals_sps.size(), maxlen); ++i) {
    std::stringstream out;
    const char *file, *name;
    uint lineno;
    ulong addr = vals_sps[i];
    ulong rel_addr = 0;
    ulong offset = 0;
    ulong maps_offset = 0;
    char* demangled;
    DBG(11, "addr: %016lx %ld %ld", addr, vals_sps.size(), maxlen);
    bool ignore_basic_libs = true;
    if (i == 0 || i == std::min(vals_sps.size(), maxlen) - 1) {
      ignore_basic_libs = false;
    }
    const symbol_ent* e = pinfo_find_symbol(
        pinfo, addr, offset, maps_offset, rel_addr, ignore_basic_libs);
    if (e != 0) {
      if (offset != 0) {
        if (!single_line) {
          snprintf(buf, sizeof(buf), "#%02d  ", rank++);
          out << buf;
          snprintf(buf, sizeof(buf), "0x%016lx", addr);
          out << buf;
          out << " in ";
          demangled = get_demangled_symbol(e->name.c_str());
          out << demangled;
          free(demangled);
          if (!print_arg) {
            out << " ()";
          }
          const proc_map_ent& proc_maps_ent = pinfo.maps[maps_offset];
          const bfd_handle* bh = proc_maps_ent.stbl->bh;
          if (bh->has_debug()) {
            int ret;
            ret = bh->get_file_line(
                rel_addr ? rel_addr : addr, &file, &name, &lineno);
            if (ret && file) {
              if (basename_only) {
                file = basename(file);
              }
              out << " from ";
              out << file;
              out << ":";
              out << lineno;
              out << "";
            }
          } else {
            out << " from " << (basename_only ? basename(proc_maps_ent.path)
                                              : proc_maps_ent.path);
          }
          print_stack("%s\n", out.str().c_str());
        } else {
          if (!rstr.empty()) {
            rstr += ":";
          }
          demangled = get_demangled_symbol(e->name.c_str());
          rstr += demangled;
          free(demangled);
        }
      }
    }
  }
  if (!rstr.empty()) {
    print_stack("%s\n", rstr.c_str());
  }
}

static int ptrace_attach_proc(int pid) {
  if (ptrace(PTRACE_ATTACH, pid, 0, 0) != 0) {
    if (errno == ESRCH) {
      DBG(11, "No such process: %d", pid);
      return ESRCH;
    } else {
      perror("ptrace(PTRACE_ATTACH)");
      return -1;
    }
  }
  int st = 0;
  const pid_t v = waitpid(pid, &st, __WALL | WUNTRACED);
  if (v < 0) {
    perror("waitpid");
    return -1;
  }
  return 0;
}

static int ptrace_detach_proc(int pid) {
  if (ptrace(PTRACE_DETACH, pid, 0, 0) != 0) {
    perror("ptrace(PTRACE_DETACH)");
    return -1;
  }
  return 0;
}

int get_tgid(int target_pid) {
  int tgid = -1;
  const string status_file_path =
      "/proc/" + std::to_string(target_pid) + "/status";
  std::ifstream status_file(status_file_path);
  if (!status_file.is_open()) {
    return tgid;
  }
  string line;
  while (std::getline(status_file, line)) {
    if (startwith(line, "Tgid")) {
      sscanf(line.c_str(), "Tgid:%d", &tgid);
    }
  }
  return tgid;
}

void get_tids(const int target_pid, thread_list& threads) {
  char fn[PATH_MAX];
  bool target_pid_exists = false;
  snprintf(fn, sizeof(fn), "/proc/%d/task", target_pid);
  DIR* dp = opendir(fn);
  int tgid = get_tgid(target_pid);
  if (tgid <= 0) {
    fprintf(stderr, "Failed to get parent's process id!\n");
    exit(1);
  }
  if (dp) {
    struct dirent* dent;
    do {
      dent = readdir(dp);
      if (dent) {
        if (dent->d_name[0] == '.') {
          continue;
        }
        int tid = atoi(dent->d_name);
        if (tid == target_pid) {
          target_pid_exists = true;
          continue;
        } else if (tgid != target_pid) {
          continue;
        } else {
          threads.push_back(thread_info(tid));
        }
      }
    } while (dent);
    closedir(dp);
  } else {
    fprintf(stderr, "Failed to access directory %s\n", fn);
    exit(1);
  }
  if (!target_pid_exists) {
    fprintf(stderr, "Process id %d does not exist on %s\n", target_pid, fn);
    exit(1);
  }
  std::sort(threads.begin(), threads.end());
  threads.push_back(thread_info(target_pid));
  return;
}

static double timediff(struct timeval tv0, struct timeval tv1) {
  return (tv1.tv_sec - tv0.tv_sec) * 1e+3 +
         (double)((double)tv1.tv_usec * 1e-3 - (double)tv0.tv_usec * 1e-3);
}

void print_trace_report(proc_info* pinfos, const thread_list& threads) {
  int slowest_tid = 0;
  double slowest_time = -1;
  double sum_time = 0;

  for (uint i = 0; i < threads.size(); i++) {
    if (pinfos[i].stall_time > slowest_time) {
      slowest_tid = threads[i].tid;
      slowest_time = pinfos[i].stall_time;
    }
    sum_time += pinfos[i].stall_time;
  }
  // Traced time == hangup time
  DBG(1, "Total Traced Time: %6.3f milliseconds", sum_time);
  DBG(1,
      "Average Traced Time Per LWP: %6.3f milliseconds",
      (double)sum_time / threads.size());
  DBG(1,
      "Longest Traced LWP: LWP %d, %6.3f milliseconds",
      slowest_tid,
      slowest_time);
}

void attach_and_dump_all(const thread_list& threads,
                         proc_info* pinfos,
                         vector<ulong>* vals_sps,
                         struct iovec* iovecs,
                         bool* fails,
                         bool lock_main) {
  for (size_t i = 0; i < threads.size(); ++i) {

    if (lock_main && threads[i].tid == target_pid) {
      if (get_user_regs(threads[i].tid, iovecs[i]) != 0) {
        fails[i] = true;
      } else {
        if (max_ptrace_calls &&
            get_stack_trace(threads[i].tid,
                            pinfos[i],
                            max_ptrace_calls,
                            *static_cast<user_regs_struct*>(iovecs[i].iov_base),
                            vals_sps[i]) != 0) {
          fails[i] = true;
        }
      }
      continue;
    }

    fails[i] = false;
    // do not attach if pid does not exist
    if (kill(threads[i].tid, 0) != 0) {
      DBG(11, "kill(0) failed (pid not exist): %d", threads[i].tid);
      fails[i] = true;
      continue;
    }
    if (is_pid_stopped(threads[i].tid)) {
      DBG(3, "PID %d stops. Skipping tracing the pid.", threads[i].tid);
    }
    DBG(3, "Attaching process %d.", threads[i].tid);
    gettimeofday(&pinfos[i].tv_start, 0);
    int rc = ptrace_attach_proc(threads[i].tid);
    if (rc != 0) {
      fails[i] = true;
    } else {
      if (get_user_regs(threads[i].tid, iovecs[i]) != 0) {
        fails[i] = true;
      } else {
        if (max_ptrace_calls &&
            get_stack_trace(threads[i].tid,
                            pinfos[i],
                            max_ptrace_calls,
                            *static_cast<user_regs_struct*>(iovecs[i].iov_base),
                            vals_sps[i]) != 0) {
          fails[i] = true;
        }
      }
    }
    if (rc != ESRCH) {
      ptrace_detach_proc(threads[i].tid);
      gettimeofday(&pinfos[i].tv_end, 0);
      pinfos[i].stall_time = timediff(pinfos[i].tv_start, pinfos[i].tv_end);
      DBG(3, "Detached process %d", threads[i].tid);
      DBG(3,
          "Tracing duration at LWP %d was "
          "%7.3f milliseconds\n",
          threads[i].tid,
          pinfos[i].stall_time);
    }
    if (shutdown_program) {
      DBG(1, "Got termination signal");
      break;
    }
  }
}

void attach_and_dump_lock_all(const thread_list& threads,
                              proc_info* pinfos,
                              vector<ulong>* vals_sps,
                              struct iovec* iovecs,
                              bool* fails) {
  struct timeval tv_start, tv_end;
  DBG(1, "Attaching main process %d, locking all.", target_pid);
  gettimeofday(&tv_start, 0);
  int rc = ptrace_attach_proc(target_pid);
  if (rc) {
    DBG(1, "Failed to attach main process.");
    if (rc != ESRCH)
      ptrace_detach_proc(target_pid);
    exit(1);
  }
  attach_and_dump_all(threads, pinfos, vals_sps, iovecs, fails, true);
  ptrace_detach_proc(target_pid);
  gettimeofday(&tv_end, 0);
  DBG(1, "Detached main process %d", target_pid);
  DBG(3,
      "Tracing duration at main process %d was "
      "%7.3f milliseconds\n",
      target_pid,
      timediff(tv_start, tv_end));
}

void dump_stack(const thread_list& threads) {
  uint trace_length = 1000;
  symbol_table_map* stmap = new symbol_table_map();
  proc_info* pinfos = new proc_info[threads.size()];
  vector<ulong>* vals_sps = new vector<ulong>[threads.size()];
  struct iovec* iovecs = new iovec[threads.size()];
  for (size_t i = 0; i < threads.size(); i++) {
    iovecs[i].iov_base = new user_regs_struct{};
    memset(iovecs[i].iov_base, 0, sizeof(user_regs_struct));
    iovecs[i].iov_len = sizeof(user_regs_struct);
  }
  bool* fails = new bool[threads.size()];

  DBG(1, "Reading process symbols..");
  for (size_t i = 0; i < threads.size(); ++i) {
    if (!trace_multiple_procs && i >= 1) {
      pinfos[i] = pinfos[0];
    } else {
      read_proc_maps(threads[i].tid, pinfos[i], stmap);
    }
  }

  if (is_pid_stopped(target_pid)) {
    DBG(1,
        "Target PID %d stops. Consider starting "
        "the process with kill -CONT. Exiting without tracing.",
        target_pid);
    exit(1);
  }
  DBG(1, "Gathering stack traces..");
  *_attach_started = 1;

  if (lock_all) {
    attach_and_dump_lock_all(threads, pinfos, vals_sps, iovecs, fails);
  } else {
    attach_and_dump_all(threads, pinfos, vals_sps, iovecs, fails, false);
  }

  DBG(1, "Printing stack traces..");
  print_trace_report(pinfos, threads);

  if (stack_out) {
    stack_out_fp = fopen(stack_out, "w");
  }

  for (size_t i = 0; i < threads.size(); ++i) {
    if (fails[i] == false) {
      if (single_line) {
        const size_t name_width =
            thread_names ? thread_info::max_name_len() + 2 : 0;
        const string name_info = thread_names ? threads[i].name : "";
        print_stack("%d  %-*s", threads[i].tid, name_width, name_info.c_str());
      } else {
        const string name_info = thread_names ? ", " + threads[i].name : "";
        print_stack("\nThread %ld (LWP %d)%s:\n",
                    threads.size() - i,
                    threads[i].tid,
                    name_info.c_str());
      }
      parse_stack_trace(pinfos[i], vals_sps[i], trace_length);
    }
  }
  if (stack_out_fp) {
    fclose(stack_out_fp);
    stack_out_fp = NULL;
  }
  delete[] fails;
  delete[] pinfos;
  delete[] vals_sps;
  for (size_t i = 0; i < threads.size(); i++) {
    delete static_cast<user_regs_struct*>(iovecs[i].iov_base);
  }
  delete[] iovecs;
  delete stmap;
}

struct option long_options[] = {
    {"?", no_argument, nullptr, '?'},
    {"help", no_argument, nullptr, 'h'},
    {"version", no_argument, nullptr, 'v'},
    {"arg_print", no_argument, nullptr, 'a'},
    {"basename_only", no_argument, nullptr, 'b'},
    {"debug", required_argument, nullptr, 'd'},
    {"debug_print_time_level", required_argument, nullptr, 't'},
    {"pid", required_argument, nullptr, 'p'},
    {"single_line", no_argument, nullptr, 's'},
    {"thread_names", no_argument, nullptr, 'n'},
    {"calls", required_argument, nullptr, 'c'},
    {"frame_check", no_argument, nullptr, 'f'},
    {"stack_out", required_argument, nullptr, 'o'},
    {"multiple_targets", required_argument, nullptr, 'm'},
    {"flush_log", required_argument, nullptr, 'w'},
    {"timeout_seconds", required_argument, nullptr, 'k'},
    {"lock_all", no_argument, nullptr, 'l'},
    {nullptr, 0, nullptr, 0}};

static void show_version() { printf("quickstack version %s\n", version); }

static void version_exit() {
  show_version();
  exit(1);
}

static void usage_exit() {
  show_version();
  printf("Usage: \n");
  printf(" quickstack [OPTIONS]\n\n");
  printf("Example: \n");
  printf(" quickstack -p `pidof mysqld`\n\n");
  printf("Options (short name):\n");
  printf(" -p, --pid=N                    :Target process id\n");
  printf(" -d, --debug=N                  :Debug level\n");
  printf(
      " -s, --single_line              :Printing call stack info into one line "
      "per process, instead of gdb-like output\n");
  printf(" -n, --thread_names             :Print thread names\n");
  printf(
      " -c, --calls=N                  :Maximum ptrace call counts per "
      "process. Default is 1000\n");
  printf(
      " -b, --basename_only            :Suppressing printing directory name of "
      "the target source files, but printing basename only. This makes easier "
      "for reading.\n");
  printf(
      " -f, --frame_check              :Checking frame pointers on "
      "non-standard libraries.\n");
  printf(
      " -o, --stack_out=f              :Writing stack traces to this file. "
      "Default is STDOUT.\n");
  printf(
      " -t, --debug_print_time_level=N :Suppressing printing timestamp if "
      "debug level is higher than N. This is for performance reason and "
      "default level (10) should be fine in most of cases.\n");
  printf(
      " -f, --multipe_targets=[0|1]    :Set 1 if tracing multiple different "
      "processes at one time\n");
  printf(
      " -w, --flush_log=N              :Flushing every log output if log level "
      "is equal or under N\n");
  printf(
      " -k, --timeout_seconds=N        :Terminates quickstack if exceeding N "
      "seconds. Default is 600 seconds\n");
  printf(
      " -l, --lock_all                 :Locking main process (given by --pid) "
      "during parsing all other processes. This will lock the whole process "
      "during taking all stack traces, so stall time is slightly increased, "
      "but will give more accurate results.\n");
  exit(1);
}

static void get_options(int argc, char** argv) {
  int c, opt_ind = 0;
  while ((c = getopt_long(
              argc, argv, "?absnflvw:k:d:c:t:p:o:", long_options, &opt_ind)) !=
         EOF) {
    switch (c) {
    case '?':
      usage_exit();
      break;
    case 'h':
      usage_exit();
      break;
    case 'a':
      print_arg = 1;
      break;
    case 'b':
      basename_only = 1;
      break;
    case 'c':
      max_ptrace_calls = atoi(optarg);
      break;
    case 'd':
      debug_level = atoi(optarg);
      break;
    case 't':
      debug_print_time_level = atoi(optarg);
      break;
    case 'p':
      target_pid = atoi(optarg);
      break;
    case 's':
      single_line = 1;
      break;
    case 'n':
      thread_names = 1;
      break;
    case 'o':
      stack_out = optarg;
      break;
    case 'f':
      frame_check = 1;
      break;
    case 'm':
      trace_multiple_procs = atoi(optarg);
      break;
    case 'v':
      version_exit();
      break;
    case 'w':
      flush_log = atoi(optarg);
      break;
    case 'k':
      timeout_seconds = atoi(optarg);
      break;
    case 'l':
      lock_all = true;
      break;
    default:
      usage_exit();
      break;
    }
  }
  if (!target_pid)
    usage_exit();
}

/* Sending SIGCONT if target pid is stopped. Since signal is sent in async,
 * we may retry a few times to verify if the target is really started. */
int cont_process_if(int pid) {
  int retry = 10;
  bool is_stopped = false;
  while ((is_stopped = is_pid_stopped(pid)) && retry-- > 0) {
    DBG(1,
        "pid %d is stopped. Sending SIGCONT.. (remaining %d times)",
        target_pid,
        retry);
    int rc = kill(pid, SIGCONT);
    if (rc == ESRCH)
      return 0;
    sleep(2);
  }
  if (is_stopped) {
    DBG(1, "Failed to start pid %d", pid);
    return 1;
  }
  return 0;
}

/* Checking target process status and sending SIGCONT if needed.
 * If child process (quickstack core logic) is
 * not aborted and target_pid (main pid) is running, we don't check
 * other processes. Otherwise we check all pids (including all LWPs of
 * the target process). */
int cont_all_process(int main_pid, const thread_list& threads, bool aborted) {
  int status = 0;
  status = cont_process_if(main_pid);
  if (status <= 0 && !aborted)
    return status;
  DBG(1, "Needs cleanup. Sending SIGCONT to all processes if needed..")
  for (size_t i = 0; i < threads.size(); ++i) {
    int rc = cont_process_if(threads[i].tid);
    if (rc)
      status = rc;
  }
  if (!status)
    DBG(1, "done.");
  return status;
}

int main(int argc, char** argv) {
  thread_list threads;
  if (argc <= 1) {
    usage_exit();
  }
  struct timeval t_begin;
  struct timeval t_current;
  gettimeofday(&t_begin, 0);
  get_options(argc, argv);
  get_tids(target_pid, threads);
  _attach_started = (int*)mmap(
      0, PTRACE_SEIZE, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0);

  pid_t quickstack_core_pid = fork();
  if (quickstack_core_pid < 0) {
    DBG(1, "Failed to fork quickstack process.. Exit");
    exit(1);
  } else if (quickstack_core_pid == 0) {
    /* quickstack core process */
    init_signals();
    dump_stack(threads);
    exit(0);
  }

  ignore_signals();
  int exit_code = 0;
  char *args = argv[0];
  sprintf(args, "quickstack_watchdog");
  int status;
  bool cleanup_needed = false;
  pid_t exited_pid;
  do {
    exited_pid = waitpid(quickstack_core_pid, &status, WNOHANG);
    if (exited_pid == -1) {
      if (errno == EINTR) {
        continue;
      }
      DBG(1, "Got error on waitpid: %d", exited_pid);
    } else if (exited_pid == 0) {
      /* quickstack is running */
      sleep(1);
      gettimeofday(&t_current, 0);
      if (t_current.tv_sec >= t_begin.tv_sec + timeout_seconds) {
        DBG(1,
            "Timeout %d seconds reached. Killing quickstack..",
            timeout_seconds);
        kill(quickstack_core_pid, SIGKILL);
        sleep(1);
      }
    } else {
      /* quickstack ended */
      break;
    }
  } while (exited_pid == 0);

  exit_code = WEXITSTATUS(status);
  if (exit_code) {
    DBG(1, "Got rc %d", WEXITSTATUS(status));
    cleanup_needed = true;
  }

  if (WIFSIGNALED(status)) {
    DBG(1, "Killed by signal %d", WTERMSIG(status));
    cleanup_needed = true;
    sleep(5);
  }
  if (*_attach_started) {
    int stop_status = cont_all_process(target_pid, threads, cleanup_needed);
    if (stop_status) {
      DBG(1,
          "FATAL: SIGCONT target process failed."
          " Target process remains state T.");
      exit(1);
    }
  }
  exit(exit_code);
}