Madvise Transparent Huge Pages for large allocations

base/options.jl

@@ -70,6 +70,7 @@ struct JLOptions
     gc_sweep_always_full::Int8
     compress_sysimage::Int8
     alert_on_critical_error::Int8
+    hugepage_threshold::Int64
 end
 
 # This runs early in the sysimage != is not defined yet

src/gc-common.h

@@ -7,6 +7,9 @@
 #include "julia_internal.h"
 #ifndef _OS_WINDOWS_
 #include <sys/mman.h>
+#ifndef MADV_HUGEPAGE
+#define MADV_HUGEPAGE 14 //  Compatibility for kernels < 2.6.38 (as in numpy implementation)
+#endif
 #if defined(_OS_DARWIN_) && !defined(MAP_ANONYMOUS)
 #define MAP_ANONYMOUS MAP_ANON
 #endif
@@ -134,6 +137,7 @@ STATIC_INLINE void jl_free_aligned(void *p) JL_NOTSAFEPOINT
 #endif
 #define malloc_cache_align(sz) jl_malloc_aligned(sz, JL_CACHE_BYTE_ALIGNMENT)
 #define realloc_cache_align(p, sz, oldsz) jl_realloc_aligned(p, sz, oldsz, JL_CACHE_BYTE_ALIGNMENT)
+#define malloc_page_align(sz) jl_malloc_aligned(sz, jl_getpagesize())
 
 // =========================================================================== //
 // Pointer tagging

src/gc-stock.c

@@ -3850,7 +3850,18 @@ JL_DLLEXPORT void *jl_gc_managed_malloc(size_t sz)
 #ifdef _OS_WINDOWS_
     DWORD last_error = GetLastError();
 #endif
-    void *b = malloc_cache_align(allocsz);
+    #ifdef MADV_HUGEPAGE
+        void *b = NULL;
+        if ((jl_options.hugepage_threshold >= 0) && (allocsz >= jl_options.hugepage_threshold)) {
+            b = malloc_page_align(allocsz);
+            madvise(b, allocsz, MADV_HUGEPAGE);
+        }
+        else {
+            b = malloc_cache_align(allocsz);
+        }
+    #else
+        void *b = malloc_cache_align(allocsz);
+    #endif
     if (b == NULL)
         jl_throw(jl_memory_exception);
 

src/jl_exported_funcs.inc

@@ -182,6 +182,7 @@
     XX(jl_gensym) \
     XX(jl_getaffinity) \
     XX(jl_getallocationgranularity) \
+    XX(jl_gethugepagesize) \
     XX(jl_getnameinfo) \
     XX(jl_getpagesize) \
     XX(jl_get_ARCH) \

src/jloptions.c

@@ -84,6 +84,14 @@ uint64_t parse_heap_size_option(const char *optarg, const char *option_name, int
     return sz < limit ? (uint64_t)sz : UINT64_MAX;
 }
 
+static int64_t default_hugepage_threshold(void)
+{
+    long hps = jl_gethugepagesize();
+    if (hps > 0)
+        return (int64_t)(3 * hps / 4);
+    return -1;
+}
+
 static int jl_options_initialized = 0;
 
 JL_DLLEXPORT void jl_init_options(void)
@@ -163,6 +171,7 @@ JL_DLLEXPORT void jl_init_options(void)
                         0, // gc_sweep_always_full
                         0, // compress_sysimage
                         0, // alert_on_critical_error
+                        default_hugepage_threshold(), // hugepage_threshold
     };
     jl_options_initialized = 1;
 }
@@ -298,7 +307,14 @@ static const char opts[]  =
     "                                               number of bytes, optionally in units of: B,\n"
     "                                               K (kibibytes), M (mebibytes), G (gibibytes),\n"
     "                                               T (tebibytes), or % (percentage of physical memory).\n\n"
-;
+    " --hugepage-threshold={auto|no|<size>[<unit>]}   Minimum heap allocation size for which Julia\n"
+    "                                               requests MADV_HUGEPAGE (transparent huge pages).\n"
+    "                                               Linux-only; applies to the stock GC only. The\n"
+    "                                               default is 3/4 of the system huge page size; 'no'\n"
+    "                                               disables. The underlying allocator may still choose\n"
+    "                                               huge pages independently of this setting.\n"
+
+    ;
 
 static const char opts_hidden[]  =
     "Switches (a '*' marks the default value, if applicable):\n\n"
@@ -416,6 +432,7 @@ JL_DLLEXPORT void jl_parse_opts(int *argcp, char ***argvp)
            opt_trace_eval,
            opt_experimental_features,
            opt_compress_sysimage,
+           opt_hugepage_threshold,
     };
     static const char* const shortopts = "+vhqH:e:E:L:J:C:it:p:O:g:m:";
     static const struct option longopts[] = {
@@ -489,6 +506,7 @@ JL_DLLEXPORT void jl_parse_opts(int *argcp, char ***argvp)
         { "trim",  optional_argument, 0, opt_trim },
         { "compress-sysimage", required_argument, 0, opt_compress_sysimage },
         { "trace-eval",       optional_argument, 0, opt_trace_eval },
+        { "hugepage-threshold",required_argument, 0, opt_hugepage_threshold },
         { 0, 0, 0, 0 }
     };
 
@@ -1079,6 +1097,48 @@ JL_DLLEXPORT void jl_parse_opts(int *argcp, char ***argvp)
             else
                 jl_errorf("julia: invalid argument to --trace-eval={yes|no} (%s)", optarg);
             break;
+        case opt_hugepage_threshold:
+            if (!strcmp(optarg, "auto")) {
+                jl_options.hugepage_threshold = default_hugepage_threshold();
+            }
+            else if (!strcmp(optarg, "no")) {
+                jl_options.hugepage_threshold = -1; // disabled
+            }
+            else {
+                // parse size like heap-size options but without percent and allowing 0
+                long double value = 0.0;
+                char unit[4] = {0};
+                int nparsed = sscanf(optarg, "%Lf%3s", &value, unit);
+                if (nparsed == 0 || strlen(unit) > 2 || (strlen(unit) == 2 && ascii_tolower(unit[1]) != 'b'))
+                    jl_errorf("julia: invalid argument to --hugepage-threshold={auto|no|<size>[<unit>]} (%s)", optarg);
+                uint64_t multiplier = 1ull;
+                switch (ascii_tolower(unit[0])) {
+                    case '\0':
+                    case 'b':
+                        break;
+                    case 'k':
+                        multiplier <<= 10;
+                        break;
+                    case 'm':
+                        multiplier <<= 20;
+                        break;
+                    case 'g':
+                        multiplier <<= 30;
+                        break;
+                    case 't':
+                        multiplier <<= 40;
+                        break;
+                    default:
+                        jl_errorf("julia: invalid argument to --hugepage-threshold={auto|no|<size>[<unit>]} (%s)", optarg);
+                }
+                long double sz = value * multiplier;
+                if (isnan(sz) || sz < 0)
+                    jl_errorf("julia: invalid argument to --hugepage-threshold={auto|no|<size>[<unit>]} (%s)", optarg);
+                const long double limit = ldexpl(1.0, 63); // INT64_MAX + 1
+                int64_t parsed = (sz < limit) ? (int64_t)sz : INT64_MAX;
+                jl_options.hugepage_threshold = parsed; // allow 0 (means advise for all)
+            }
+            break;
         case opt_task_metrics:
             if (!strcmp(optarg, "no"))
                 jl_options.task_metrics = JL_OPTIONS_TASK_METRICS_OFF;

src/jloptions.h

@@ -74,6 +74,7 @@ typedef struct {
     int8_t gc_sweep_always_full;
     int8_t compress_sysimage;
     int8_t alert_on_critical_error;
+    int64_t hugepage_threshold;
 } jl_options_t;
 
 #endif

src/julia.h

@@ -2057,6 +2057,7 @@ JL_DLLEXPORT int jl_cpu_threads(void) JL_NOTSAFEPOINT;
 JL_DLLEXPORT int jl_effective_threads(void) JL_NOTSAFEPOINT;
 JL_DLLEXPORT long jl_getpagesize(void) JL_NOTSAFEPOINT;
 JL_DLLEXPORT long jl_getallocationgranularity(void) JL_NOTSAFEPOINT;
+JL_DLLEXPORT long jl_gethugepagesize(void) JL_NOTSAFEPOINT;
 JL_DLLEXPORT int jl_is_debugbuild(void) JL_NOTSAFEPOINT;
 JL_DLLEXPORT jl_sym_t *jl_get_UNAME(void) JL_NOTSAFEPOINT;
 JL_DLLEXPORT jl_sym_t *jl_get_ARCH(void) JL_NOTSAFEPOINT;

src/sys.c

@@ -10,6 +10,7 @@
 #include <errno.h>
 #include <signal.h>
 #include <fcntl.h>
+#include <stdio.h>
 
 #include "julia.h"
 #include "julia_internal.h"
@@ -610,6 +611,41 @@ JL_DLLEXPORT long jl_getallocationgranularity(void) JL_NOTSAFEPOINT
 }
 #endif
 
+JL_DLLEXPORT long jl_gethugepagesize(void) JL_NOTSAFEPOINT
+{
+#if defined(_OS_LINUX_)
+    long detected = 0;
+    FILE *f = fopen("/sys/kernel/mm/transparent_hugepage/hpage_pmd_size", "r");
+    if (f) {
+        unsigned long long size = 0;
+        if (fscanf(f, "%llu", &size) == 1 && size > 0) {
+            detected = (long)size;
+        }
+        fclose(f);
+    }
+    if (detected == 0) {
+        f = fopen("/proc/meminfo", "r");
+        if (f) {
+            char line[256];
+            while (fgets(line, sizeof(line), f)) {
+                unsigned long long kb = 0;
+                if (sscanf(line, "Hugepagesize:%llu kB", &kb) == 1 && kb > 0) {
+                    detected = (long)(kb * 1024ULL);
+                    break;
+                }
+            }
+            fclose(f);
+        }
+    }
+    if (detected == 0) {
+        detected = 2 * 1024 * 1024; // 2 MiB fallback
+    }
+    return detected;
+#else
+    return 0;
+#endif
+}
+
 JL_DLLEXPORT long jl_SC_CLK_TCK(void)
 {
 #ifndef _OS_WINDOWS_