Skip to content

Fix PyTorch fatbinary dumping #177

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 3 commits into
base: master
Choose a base branch
from
Open

Fix PyTorch fatbinary dumping #177

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
00c59f8
Use ".nv.info." to match cuobjdump function name instead of .nv.info.…
elementgreen May 19, 2021
88e889a
Don't dump CUDA binaries which have no .nv.info CUDA_INFO argument in…
elementgreen May 19, 2021
bd28a81
Use accomulated ht_name2idx function name hash for pytorch opt worker…
elementgreen May 19, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
140 changes: 74 additions & 66 deletions cava/samples/pytorch/pt_dump.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -178,39 +178,6 @@ ava_utility void __helper_dump_fatbin(void *fatCubin,
struct fatBinaryHeader *fbh = reinterpret_cast<struct fatBinaryHeader *>(wp->ptr);
int fd, ret;

/* Increase fatbin counter */
static int fatbin_num = 0;
fatbin_num++;
if (ava_is_worker) {
char* file_name = "/tmp/fatbin-info.ava";
fd = open(file_name, O_RDWR | O_CREAT, 0666);
if (fd == -1) {
fprintf(stderr, "open %s [errno=%d, errstr=%s] at %s:%d",
file_name, errno, strerror(errno), __FILE__, __LINE__);
exit(EXIT_FAILURE);
}
AVA_DEBUG << "Fatbinary counter = " << fatbin_num;
ret = write(fd, (const void *)&fatbin_num, sizeof(int));
if (ret == -1) {
fprintf(stderr, "write [errno=%d, errstr=%s] at %s:%d",
errno, strerror(errno), __FILE__, __LINE__);
exit(EXIT_FAILURE);
}
ret = lseek(fd, 0, SEEK_END);
if (ret == -1) {
fprintf(stderr, "lseek [errno=%d, errstr=%s] at %s:%d",
errno, strerror(errno), __FILE__, __LINE__);
exit(EXIT_FAILURE);
}
ret = write(fd, (const void *)wp, sizeof(struct fatbin_wrapper));
if (ret == -1) {
fprintf(stderr, "write [errno=%d, errstr=%s] at %s:%d",
errno, strerror(errno), __FILE__, __LINE__);
exit(EXIT_FAILURE);
}
close(fd);
}

/* Dump fat binary to a file */
char fatbin_filename[32];
if (ava_is_worker) {
Expand Down Expand Up @@ -245,19 +212,6 @@ ava_utility void __helper_dump_fatbin(void *fatCubin,
*num_funcs = 0;
}

/* Add separator to the functions of different fatbinaries */
if (ava_is_worker) {
if (ava_metadata(NULL)->fd_functions != 0) {
size = 0;
ret = write(ava_metadata(NULL)->fd_functions, (const void *)&size, sizeof(size_t));
if (ret == -1) {
fprintf(stderr, "write [errno=%d, errstr=%s] at %s:%d",
errno, strerror(errno), __FILE__, __LINE__);
exit(EXIT_FAILURE);
}
}
}

/* Open the command pipe for reading */
char pip_command[80];
sprintf(pip_command, "/usr/local/cuda-10.1/bin/cuobjdump -elf /tmp/fatbin-%d.ava",
Expand All @@ -267,25 +221,20 @@ ava_utility void __helper_dump_fatbin(void *fatCubin,

/* Open function argument dump file */
int function_arg_fd;
char function_arg_filename[32];
if (ava_is_worker) {
sprintf(function_arg_filename, "/tmp/function_arg-%d.ava", ava_metadata(NULL)->num_fatbins);
function_arg_fd = open(function_arg_filename, O_WRONLY | O_TRUNC | O_CREAT, 0666);
if (function_arg_fd == -1) {
fprintf(stderr, "open %s [errno=%d, errstr=%s] at %s:%d",
function_arg_filename, errno, strerror(errno), __FILE__, __LINE__);
exit(EXIT_FAILURE);
}
AVA_DEBUG << "Dump function argument info to " << function_arg_filename;
}
bool has_funcs = FALSE;

while (fgets(line, sizeof(line), fp_pipe) != NULL) {
/* Search functions */
if (strncmp(line, ".nv.info._Z", 11) == 0) {
if (strncmp(line, ".nv.info.", 9) == 0) {
sprintf(name, line + 9, strlen(line) - 10);
assert(strlen(line) - 10 < MAX_KERNEL_NAME_LEN);
name[strlen(line) - 10] = '\0';
ava_debug("[%d] %s@", *num_funcs, name);

// Skip duplicate function entries for multiple compute versions
if (g_hash_table_lookup(*fatbin_funcs, name) != NULL)
continue;

DEBUG_PRINT("[%d] %s@\n", *num_funcs, name);

/* Create a new hash table entry */
func = (struct fatbin_function *)g_malloc(sizeof(struct fatbin_function));
Expand Down Expand Up @@ -349,6 +298,20 @@ ava_utility void __helper_dump_fatbin(void *fatCubin,

/* Dump the function argument sizes to file */
if (ava_is_worker) {
/* Open function argument dump file on first valid function */
if (!has_funcs) {
char function_arg_filename[32];

sprintf(function_arg_filename, "/tmp/function_arg-%d.ava", ava_metadata(NULL)->num_fatbins);
function_arg_fd = open(function_arg_filename, O_WRONLY | O_TRUNC | O_CREAT, 0666);
if (function_arg_fd == -1) {
fprintf(stderr, "open %s [errno=%d, errstr=%s] at %s:%d",
function_arg_filename, errno, strerror(errno), __FILE__, __LINE__);
exit(EXIT_FAILURE);
}
AVA_DEBUG << "Dump function argument info to " << function_arg_filename;
}

size = strlen(name) + 1;
ret = write(function_arg_fd, (void *)&size, sizeof(size_t));
if (ret == -1) {
Expand All @@ -370,20 +333,65 @@ ava_utility void __helper_dump_fatbin(void *fatCubin,
}
}

has_funcs = TRUE;

/* Insert the function into hash table */
if (g_hash_table_lookup(*fatbin_funcs, name) != NULL)
g_free(func);
else
g_hash_table_insert((*fatbin_funcs), g_strdup(name), (gpointer)func);
//func = (struct fatbin_function *)g_hash_table_lookup(*fatbin_funcs, name);
g_hash_table_insert((*fatbin_funcs), g_strdup(name), (gpointer)func);
}
}

if (has_funcs)
++(ava_metadata(NULL)->num_fatbins);

// Were there any valid functions in the binary?
if (ava_is_worker)
close(function_arg_fd);
{
if (has_funcs)
{
close(function_arg_fd);

char* file_name = "/tmp/fatbin-info.ava";
fd = open(file_name, O_RDWR | O_CREAT, 0666);
if (fd == -1) {
fprintf(stderr, "open %s [errno=%d, errstr=%s] at %s:%d",
file_name, errno, strerror(errno), __FILE__, __LINE__);
exit(EXIT_FAILURE);
}
DEBUG_PRINT("Fatbinary counter = %d\n", ava_metadata(NULL)->num_fatbins);
ret = write(fd, (const void *)&(ava_metadata(NULL)->num_fatbins), sizeof(int));
if (ret == -1) {
fprintf(stderr, "write [errno=%d, errstr=%s] at %s:%d",
errno, strerror(errno), __FILE__, __LINE__);
exit(EXIT_FAILURE);
}
ret = lseek(fd, 0, SEEK_END);
if (ret == -1) {
fprintf(stderr, "lseek [errno=%d, errstr=%s] at %s:%d",
errno, strerror(errno), __FILE__, __LINE__);
exit(EXIT_FAILURE);
}
ret = write(fd, (const void *)wp, sizeof(struct fatbin_wrapper));
if (ret == -1) {
fprintf(stderr, "write [errno=%d, errstr=%s] at %s:%d",
errno, strerror(errno), __FILE__, __LINE__);
exit(EXIT_FAILURE);
}
close(fd);

/* Add separator to the functions of different fatbinaries */
if (ava_metadata(NULL)->fd_functions != 0) {
size = 0;
ret = write(ava_metadata(NULL)->fd_functions, (const void *)&size, sizeof(size_t));
if (ret == -1) {
fprintf(stderr, "write [errno=%d, errstr=%s] at %s:%d",
errno, strerror(errno), __FILE__, __LINE__);
exit(EXIT_FAILURE);
}
}
}
}

pclose(fp_pipe);
++(ava_metadata(NULL)->num_fatbins);
}

ava_utility void __helper_init_module(struct fatbin_wrapper *fatCubin, void **handle) {
Expand Down
79 changes: 5 additions & 74 deletions cava/samples/pytorch/pt_opt.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -348,7 +348,7 @@ ava_utility CUmodule __helper_init_module(struct fatbin_wrapper *fatCubin, void
return mod;
}

ava_utility void __helper_load_function_arg_info_guest(void)
ava_utility void __helper_load_function_arg_info(void)
{
GPtrArray *fatbin_funcs;
GHashTable *ht;
Expand Down Expand Up @@ -378,74 +378,6 @@ ava_utility void __helper_load_function_arg_info_guest(void)
size_t name_size;
char func_name[MAX_KERNEL_NAME_LEN];

while (1) {
read_ret = read(fd, (void *)&name_size, sizeof(size_t));
if (read_ret == 0)
break;
if (read_ret == -1) {
fprintf(stderr, "read [errno=%d, errstr=%s] at %s:%d",
errno, strerror(errno), __FILE__, __LINE__);
exit(EXIT_FAILURE);
}
assert(name_size < MAX_KERNEL_NAME_LEN && "name_size >= MAX_KERNEL_NAME_LEN");
read_ret = read(fd, (void *)func_name, name_size);
if (read_ret == -1) {
fprintf(stderr, "read [errno=%d, errstr=%s] at %s:%d",
errno, strerror(errno), __FILE__, __LINE__);
exit(EXIT_FAILURE);
}

func = g_new(struct fatbin_function, 1);
read_ret = read(fd, (void *)func, sizeof(struct fatbin_function));
if (read_ret == -1) {
fprintf(stderr, "read [errno=%d, errstr=%s] at %s:%d",
errno, strerror(errno), __FILE__, __LINE__);
exit(EXIT_FAILURE);
}
ava_debug("function %d (%s) has argc = %d",
fatbin_funcs->len - 1, func_name, func->argc);
/* Insert into the function table */
g_ptr_array_add(fatbin_funcs, (gpointer)func);

/* Add name->index mapping */
if (g_hash_table_lookup(ht, func_name) == NULL) {
assert(fatbin_funcs->len > 1 && "fatbin_funcs->len <= 1");
g_hash_table_insert(ht, g_strdup(func_name), (gpointer)((uintptr_t)fatbin_funcs->len - 1));
}
}
close(fd);

++(ava_metadata(NULL)->num_fatbins);
}

/**
* Loads the function argument information from dump.
*/
ava_utility GHashTable *__helper_load_function_arg_info(void) {
GPtrArray *fatbin_funcs;
if (ava_metadata(NULL)->fatbin_funcs == NULL) {
ava_metadata(NULL)->fatbin_funcs = g_ptr_array_new_with_free_func(g_free);
g_ptr_array_add(ava_metadata(NULL)->fatbin_funcs, (gpointer)NULL); // func_id starts from 1
}
fatbin_funcs = ava_metadata(NULL)->fatbin_funcs;

GHashTable *ht = g_hash_table_new_full(g_str_hash, g_str_equal, g_free, NULL);

int fd, read_ret;
char filename[50];
sprintf(filename, "/cuda_dumps/function_arg-%d.ava", ava_metadata(NULL)->num_fatbins);
AVA_DEBUG << "Loading " << filename;
fd = open(filename, O_RDONLY, 0666);
if (fd == -1) {
fprintf(stderr, "open [errno=%d, errstr=%s] at %s:%d",
errno, strerror(errno), __FILE__, __LINE__);
exit(EXIT_FAILURE);
}

struct fatbin_function *func;
size_t name_size;
char func_name[MAX_KERNEL_NAME_LEN];

while (1) {
read_ret = read(fd, (void *)&name_size, sizeof(size_t));
if (read_ret == 0)
Expand All @@ -471,7 +403,7 @@ ava_utility GHashTable *__helper_load_function_arg_info(void) {
exit(EXIT_FAILURE);
}

ava_debug("function %d (%s) has argc = %d",
DEBUG_PRINT("function %d (%s) has argc = %d\n",
fatbin_funcs->len - 1, func_name, func->argc);
/* Insert into the function table */
g_ptr_array_add(fatbin_funcs, (gpointer)func);
Expand All @@ -485,7 +417,6 @@ ava_utility GHashTable *__helper_load_function_arg_info(void) {
close(fd);

++(ava_metadata(NULL)->num_fatbins);
return ht;
}

/**
Expand Down Expand Up @@ -545,7 +476,8 @@ ava_utility void **__helper_load_and_register_fatbin(void *fatCubin) {
CUmodule mod = __helper_init_module(wrapper, fatbin_handle);

/* Load function argument information */
GHashTable *ht = __helper_load_function_arg_info();
__helper_load_function_arg_info ();
GHashTable *ht = ava_metadata(NULL)->ht_name2idx;

/* Register CUDA functions */
GPtrArray *fatbin_funcs = ava_metadata(NULL)->fatbin_funcs;
Expand Down Expand Up @@ -746,7 +678,6 @@ ava_utility void **__helper_load_and_register_fatbin(void *fatCubin) {
if (wSize) free(wSize);
}

g_hash_table_destroy(ht);
return fatbin_handle;
}

Expand Down Expand Up @@ -22889,7 +22820,7 @@ ava_utility void __helper_guestlib_init_prologue() {
int i;
ava_metadata(NULL)->num_fatbins = 0;
for (i = 0; i < fatbin_num; i++) {
__helper_load_function_arg_info_guest();
__helper_load_function_arg_info();
}
#endif
guestlib_tf_opt_init();
Expand Down