From 71403bd4a843cc66bee28c6a11a279654b2b1857 Mon Sep 17 00:00:00 2001 From: David Laehnemann Date: Mon, 9 Jan 2023 11:58:38 +0100 Subject: [PATCH] fix: introduce default 20% deduction from resources.mem_mb and override via params.java_mem_overhead_mv (#24) * fix: introduce default 20% deduction from resources.mem_mb for specifying -Xmx, which can be overriden by specifying params.java_mem_overhead_mb * update with suggestions from https://github.com/snakemake/snakemake-wrapper-utils/pull/24#pullrequestreview-1236465733 --- snakemake_wrapper_utils/java.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/snakemake_wrapper_utils/java.py b/snakemake_wrapper_utils/java.py index d140021..dad83fd 100644 --- a/snakemake_wrapper_utils/java.py +++ b/snakemake_wrapper_utils/java.py @@ -1,37 +1,42 @@ import sys +def java_mem_xmx_error(unit, params_key): + return f"You have specified resources.mem_{unit} and provided `-Xmx` in params.{params_key}. For Java memory specifications, please only use resources.mem_mb (for total memory reserved for the rule) and params.java_mem_overhead_mb (to specify any required non-heap overhead that needs to be set aside before determining the -Xmx value)." -def get_java_opts(snakemake): +def get_java_opts(snakemake, java_mem_overhead_factor=0.2): """Obtain java_opts from params, and handle resource definitions in resources.""" java_opts = snakemake.params.get("java_opts", "") extra = snakemake.params.get("extra", "") + assert 0.0 <= java_mem_overhead_factor <= 1.0 # Getting memory in megabytes, if java opts is not filled with -Xmx parameter # By doing so, backward compatibility is preserved if "mem_mb" in snakemake.resources.keys(): if "-Xmx" in java_opts: sys.exit( - "You have specified resources.mem_mb and provided `-Xmx` in params.java_opts. For Java memory specifications, please only use resources.mem_mb." + java_mem_xmx_error("mb", "java_opts") ) if "-Xmx" in extra: sys.exit( - "You have specified resources.mem_mb and provided `-Xmx` in params.extra. For Java memory specifications, please only use resources.mem_mb." + java_mem_xmx_error("mb", "extra") ) - java_opts += " -Xmx{}M".format(snakemake.resources["mem_mb"]) + java_opts += " -Xmx{}M".format( round( snakemake.resources["mem_mb"] * (1.0 - java_mem_overhead_factor) ) ) + # Getting memory in gigabytes, for user convenience. Please prefer the use # of mem_mb over mem_gb as advised in documentation. elif "mem_gb" in snakemake.resources.keys(): if "-Xmx" in java_opts: sys.exit( - "You have specified resources.mem_gb and provided `-Xmx` in params.java_opts. For Java memory specifications, please only use resources.mem_mb." + java_mem_xmx_error("gb", "java_opts") ) if "-Xmx" in extra: sys.exit( - "You have specified resources.mem_gb and provided `-Xmx` in params.extra. For Java memory specifications, please only use resources.mem_mb." + java_mem_xmx_error("gb", "extra") ) - java_opts += " -Xmx{}G".format(snakemake.resources["mem_gb"]) + java_opts += " -Xmx{}G".format( round( snakemake.resources["mem_gb"] * (1.0 - java_mem_overhead_factor) ) ) + # Getting java temp directory from output files list, if -Djava.io.tmpdir # is not provided in java parameters. By doing so, backward compatibility is