From c00d214a43844ab23e556dd81028ab1f02de8a73 Mon Sep 17 00:00:00 2001 From: Diego Tavares Date: Tue, 19 Nov 2024 16:37:43 -0800 Subject: [PATCH] [rqd/cuebot] Hard and Soft memory limits (#1589) Currently, frames are created with minimal requirements, but limits are not enforced. This PR implements soft and hard limits on RQD when running on docker mode. For more information about soft and hard limits [read](https://docs.docker.com/engine/containers/resource_constraints/). Limits are calculated using the minimum memory defined for the layer and a multiplier that can be tuned at Dispatcher.java. Ideally, these values should be extracted to opencue.properties, but they are being used in a static context, and interpreted before the file is actually read. --------- Signed-off-by: Diego Tavares --- .gitignore | 5 +- VERSION.in | 2 +- .../com/imageworks/spcue/DispatchFrame.java | 22 ++++++- .../com/imageworks/spcue/VirtualProc.java | 8 +-- .../spcue/dao/postgres/FrameDaoJdbc.java | 2 +- .../spcue/dispatcher/CoreUnitDispatcher.java | 2 +- .../dispatcher/DispatchSupportService.java | 2 + .../spcue/dispatcher/Dispatcher.java | 6 ++ .../dispatcher/FrameCompleteHandler.java | 8 ++- .../spcue/dispatcher/LocalDispatcher.java | 6 +- .../dispatcher/FrameCompleteHandlerTests.java | 59 ++++++++++++++++++- .../spcue/test/util/CoreSaturationTests.java | 2 +- .../spcue/test/util/CoreSpanTests.java | 12 ++-- proto/rqd.proto | 2 + rqd/rqd/rqcore.py | 16 +++++ rqd/tests/rqcore_test.py | 47 ++++++++++++++- 16 files changed, 177 insertions(+), 24 deletions(-) diff --git a/.gitignore b/.gitignore index 337ab67da..c76c98d24 100644 --- a/.gitignore +++ b/.gitignore @@ -17,4 +17,7 @@ rest_gateway/*.tar\.gz .eggs/* /cuebot/bin/* /logs/* -/.gradle/* \ No newline at end of file +/.gradle/* +cuebot/.settings/* +cuebot/.classpath +cuebot/.project diff --git a/VERSION.in b/VERSION.in index 5625e59da..a58941b07 100644 --- a/VERSION.in +++ b/VERSION.in @@ -1 +1 @@ -1.2 +1.3 \ No newline at end of file diff --git a/cuebot/src/main/java/com/imageworks/spcue/DispatchFrame.java b/cuebot/src/main/java/com/imageworks/spcue/DispatchFrame.java index 2c60e9930..1bd3806a9 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/DispatchFrame.java +++ b/cuebot/src/main/java/com/imageworks/spcue/DispatchFrame.java @@ -20,6 +20,8 @@ package com.imageworks.spcue; import java.util.Optional; + +import com.imageworks.spcue.dispatcher.Dispatcher; import com.imageworks.spcue.grpc.job.FrameState; public class DispatchFrame extends FrameEntity implements FrameInterface { @@ -42,7 +44,6 @@ public class DispatchFrame extends FrameEntity implements FrameInterface { public int minCores; public int maxCores; public boolean threadable; - public long minMemory; public int minGpus; public int maxGpus; public long minGpuMemory; @@ -52,5 +53,24 @@ public class DispatchFrame extends FrameEntity implements FrameInterface { // The Operational System this frame is expected to run in public String os; + + // Memory requirement for this frame in bytes + private long minMemory; + + // Soft limit to be enforced for this frame in bytes + public long softMemoryLimit; + + // Hard limit to be enforced for this frame in bytes + public long hardMemoryLimit; + + public void setMinMemory(long minMemory) { + this.minMemory = minMemory; + this.softMemoryLimit = (long)(((double)minMemory) * Dispatcher.SOFT_MEMORY_MULTIPLIER); + this.hardMemoryLimit = (long)(((double)minMemory) * Dispatcher.HARD_MEMORY_MULTIPLIER); + } + + public long getMinMemory() { + return this.minMemory; + } } diff --git a/cuebot/src/main/java/com/imageworks/spcue/VirtualProc.java b/cuebot/src/main/java/com/imageworks/spcue/VirtualProc.java index 02ade6bb4..2954fe662 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/VirtualProc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/VirtualProc.java @@ -103,7 +103,7 @@ public static final VirtualProc build(DispatchHost host, proc.isLocalDispatch = host.isLocalDispatch; proc.coresReserved = frame.minCores; - proc.memoryReserved = frame.minMemory; + proc.memoryReserved = frame.getMinMemory(); proc.gpusReserved = frame.minGpus; proc.gpuMemoryReserved = frame.minGpuMemory; @@ -156,11 +156,11 @@ else if (proc.coresReserved >= 100) { proc.coresReserved = wholeCores * 100; } else { - if (host.idleMemory - frame.minMemory + if (host.idleMemory - frame.getMinMemory() <= Dispatcher.MEM_STRANDED_THRESHHOLD) { proc.coresReserved = wholeCores * 100; } else { - proc.coresReserved = getCoreSpan(host, frame.minMemory); + proc.coresReserved = getCoreSpan(host, frame.getMinMemory()); } } if (host.threadMode == ThreadMode.VARIABLE_VALUE @@ -247,7 +247,7 @@ public static final VirtualProc build(DispatchHost host, proc.isLocalDispatch = host.isLocalDispatch; proc.coresReserved = lja.getThreads() * 100; - proc.memoryReserved = frame.minMemory; + proc.memoryReserved = frame.getMinMemory(); proc.gpusReserved = frame.minGpus; proc.gpuMemoryReserved = frame.minGpuMemory; diff --git a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/FrameDaoJdbc.java b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/FrameDaoJdbc.java index 9e0f6f80c..1703664c7 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/FrameDaoJdbc.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dao/postgres/FrameDaoJdbc.java @@ -325,7 +325,7 @@ public DispatchFrame mapRow(ResultSet rs, int rowNum) throws SQLException { frame.minCores = rs.getInt("int_cores_min"); frame.maxCores = rs.getInt("int_cores_max"); frame.threadable = rs.getBoolean("b_threadable"); - frame.minMemory = rs.getLong("int_mem_min"); + frame.setMinMemory(rs.getLong("int_mem_min")); frame.minGpus = rs.getInt("int_gpus_min"); frame.maxGpus = rs.getInt("int_gpus_max"); frame.minGpuMemory = rs.getLong("int_gpu_mem_min"); diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/CoreUnitDispatcher.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/CoreUnitDispatcher.java index c5ea11cb6..b8abe83e0 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/CoreUnitDispatcher.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/CoreUnitDispatcher.java @@ -286,7 +286,7 @@ public List dispatchHost(DispatchHost host, JobInterface job) { } if (host.idleCores < host.handleNegativeCoresRequirement(frame.minCores) || - host.idleMemory < frame.minMemory || + host.idleMemory < frame.getMinMemory() || host.idleGpus < frame.minGpus || host.idleGpuMemory < frame.minGpuMemory) { logger.debug("Cannot dispatch, insufficient resources."); diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupportService.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupportService.java index 0779209b0..80a1ff362 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupportService.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/DispatchSupportService.java @@ -396,6 +396,8 @@ public RunFrame prepareRqdRunFrame(VirtualProc proc, DispatchFrame frame) { .setStartTime(System.currentTimeMillis()) .setIgnoreNimby(proc.isLocalDispatch) .setOs(proc.os) + .setSoftMemoryLimit(frame.softMemoryLimit) + .setHardMemoryLimit(frame.hardMemoryLimit) .putAllEnvironment(jobDao.getEnvironment(frame)) .putAllEnvironment(layerDao.getLayerEnvironment(frame)) .putEnvironment("CUE3", "1") diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/Dispatcher.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/Dispatcher.java index 3bb1ae105..f045a5ce5 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/Dispatcher.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/Dispatcher.java @@ -82,6 +82,9 @@ public interface Dispatcher { // Upgrade the memory on the layer by 1g and retry. public static final int EXIT_STATUS_MEMORY_FAILURE = 33; + // Upgrade the memory on the layer by 1g and retry. + public static final int DOCKER_EXIT_STATUS_MEMORY_FAILURE = 137; + // max retry time public static final int FRAME_TIME_NO_RETRY = 3600 * 8; @@ -112,6 +115,9 @@ public interface Dispatcher { // memory public static final long MINIMUM_MEMORY_INCREASE = CueUtil.GB2; + public static final double SOFT_MEMORY_MULTIPLIER = 1.1; + public static final double HARD_MEMORY_MULTIPLIER = 1.4; + /** * Dispatch a host to the facility. * diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/FrameCompleteHandler.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/FrameCompleteHandler.java index 04d0fa1a9..3e92158a0 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/FrameCompleteHandler.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/FrameCompleteHandler.java @@ -320,14 +320,15 @@ public void handlePostFrameCompleteOperations(VirtualProc proc, } /* - * An exit status of 33 indicates that the frame was killed by the + * Some exit statuses indicate that a frame was killed by the * application due to a memory issue and should be retried. In this * case, disable the optimizer and raise the memory by what is * specified in the show's service override, service or 2GB. */ if (report.getExitStatus() == Dispatcher.EXIT_STATUS_MEMORY_FAILURE || report.getExitSignal() == Dispatcher.EXIT_STATUS_MEMORY_FAILURE - || frameDetail.exitStatus == Dispatcher.EXIT_STATUS_MEMORY_FAILURE) { + || frameDetail.exitStatus == Dispatcher.EXIT_STATUS_MEMORY_FAILURE + || report.getExitStatus() == Dispatcher.DOCKER_EXIT_STATUS_MEMORY_FAILURE) { long increase = CueUtil.GB2; // since there can be multiple services, just going for the @@ -641,7 +642,8 @@ else if (frame.state.equals(FrameState.DEAD)) { newState = FrameState.DEAD; } else if (frame.retries >= job.maxRetries) { if (!(report.getExitStatus() == Dispatcher.EXIT_STATUS_MEMORY_FAILURE - || report.getExitSignal() == Dispatcher.EXIT_STATUS_MEMORY_FAILURE)) + || report.getExitSignal() == Dispatcher.EXIT_STATUS_MEMORY_FAILURE + || report.getExitStatus() == Dispatcher.DOCKER_EXIT_STATUS_MEMORY_FAILURE)) newState = FrameState.DEAD; } diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/LocalDispatcher.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/LocalDispatcher.java index ffd205d32..9c3754f69 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/LocalDispatcher.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/LocalDispatcher.java @@ -116,7 +116,7 @@ private List dispatchHost(DispatchHost host, JobInterface job, * not move on. */ if (!lha.hasAdditionalResources(lha.getThreads() * 100, - frame.minMemory, + frame.getMinMemory(), frame.minGpus, frame.minGpuMemory)) { continue; @@ -209,7 +209,7 @@ private List dispatchHost(DispatchHost host, LayerInterface layer, * not move on. */ if (!lha.hasAdditionalResources(lha.getThreads() * 100, - frame.minMemory, + frame.getMinMemory(), frame.minGpus, frame.minGpuMemory)) { continue; @@ -294,7 +294,7 @@ private List dispatchHost(DispatchHost host, FrameInterface frame, */ DispatchFrame dframe = jobManager.getDispatchFrame(frame.getId()); if (!lha.hasAdditionalResources(lha.getMaxCoreUnits(), - dframe.minMemory, + dframe.getMinMemory(), lha.getMaxGpuUnits(), dframe.minGpuMemory)) { return procs; diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/FrameCompleteHandlerTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/FrameCompleteHandlerTests.java index 1f452e92a..7d0901562 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/FrameCompleteHandlerTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/dispatcher/FrameCompleteHandlerTests.java @@ -272,7 +272,7 @@ public void testGpuReportOver() { (jobManager.isLayerComplete(layer1_0) ? 1 : 0) + (jobManager.isLayerComplete(layer2_0) ? 1 : 0)); assertEquals(1, - (jobManager.isJobComplete(job1) ? 1 : 0) + + (jobManager.isJobComplete(job1) ? 1 : 0) + (jobManager.isJobComplete(job2) ? 1 : 0)); } @@ -415,6 +415,63 @@ private void executeMinMemIncrease(int expected, boolean override) { assertEquals(expected, ulayer.getMinimumMemory()); } + + + private void executeMinMemIncreaseDocker(int expected, boolean override) { + if (override) { + ServiceOverrideEntity soe = new ServiceOverrideEntity(); + soe.showId = "00000000-0000-0000-0000-000000000000"; + soe.name = "apitest"; + soe.threadable = false; + soe.minCores = 10; + soe.minMemory = (int) CueUtil.GB2; + soe.tags = new LinkedHashSet<>(); + soe.tags.add("general"); + soe.minMemoryIncrease = (int) CueUtil.GB8; + + serviceManager.createService(soe); + } + + String jobName = "pipe-default-testuser_min_mem_test"; + JobDetail job = jobManager.findJobDetail(jobName); + LayerDetail layer = layerDao.findLayerDetail(job, "test_layer"); + FrameDetail frame = frameDao.findFrameDetail(job, "0000-test_layer"); + jobManager.setJobPaused(job, false); + + DispatchHost host = getHost(HOSTNAME2); + List procs = dispatcher.dispatchHost(host); + assertEquals(1, procs.size()); + VirtualProc proc = procs.get(0); + assertEquals(job.getId(), proc.getJobId()); + assertEquals(layer.getId(), proc.getLayerId()); + assertEquals(frame.getId(), proc.getFrameId()); + + RunningFrameInfo info = RunningFrameInfo.newBuilder() + .setJobId(proc.getJobId()) + .setLayerId(proc.getLayerId()) + .setFrameId(proc.getFrameId()) + .setResourceId(proc.getProcId()) + .build(); + FrameCompleteReport report = FrameCompleteReport.newBuilder() + .setFrame(info) + .setExitStatus(Dispatcher.DOCKER_EXIT_STATUS_MEMORY_FAILURE) + .build(); + + DispatchJob dispatchJob = jobManager.getDispatchJob(proc.getJobId()); + DispatchFrame dispatchFrame = jobManager.getDispatchFrame(report.getFrame().getFrameId()); + FrameDetail frameDetail = jobManager.getFrameDetail(report.getFrame().getFrameId()); + dispatchSupport.stopFrame(dispatchFrame, FrameState.DEAD, report.getExitStatus(), + report.getFrame().getMaxRss()); + frameCompleteHandler.handlePostFrameCompleteOperations(proc, + report, dispatchJob, dispatchFrame, FrameState.WAITING, frameDetail); + + assertFalse(jobManager.isLayerComplete(layer)); + + JobDetail ujob = jobManager.findJobDetail(jobName); + LayerDetail ulayer = layerDao.findLayerDetail(ujob, "test_layer"); + assertEquals(expected, ulayer.getMinimumMemory()); + } + @Test @Transactional @Rollback(true) diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/util/CoreSaturationTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/util/CoreSaturationTests.java index 9ecd9c2b0..d6796b76e 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/util/CoreSaturationTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/util/CoreSaturationTests.java @@ -47,7 +47,7 @@ public void testCoreAndMemorySaturation1() { DispatchFrame frame = new DispatchFrame(); frame.services = "NOTarnold"; frame.minCores = 100; - frame.minMemory = CueUtil.GB * 7; + frame.setMinMemory(CueUtil.GB * 7); frame.threadable = true; VirtualProc proc = VirtualProc.build(host, frame); diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/util/CoreSpanTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/util/CoreSpanTests.java index 5543cc698..d64f42cfc 100644 --- a/cuebot/src/test/java/com/imageworks/spcue/test/util/CoreSpanTests.java +++ b/cuebot/src/test/java/com/imageworks/spcue/test/util/CoreSpanTests.java @@ -53,7 +53,7 @@ public void testCoreSpan() { DispatchFrame frame = new DispatchFrame(); frame.minCores = 100; - frame.minMemory = CueUtil.GB * 7; + frame.setMinMemory(CueUtil.GB * 7); frame.threadable = true; VirtualProc proc = VirtualProc.build(host, frame); @@ -70,7 +70,7 @@ public void testCoreSpanTest1(){ DispatchFrame frame = new DispatchFrame(); frame.minCores = 100; - frame.minMemory = CueUtil.GB; + frame.setMinMemory(CueUtil.GB); VirtualProc proc = VirtualProc.build(host, frame); assertEquals(100, proc.coresReserved); @@ -84,7 +84,7 @@ public void testCoreSpanTest2() { DispatchFrame frame = new DispatchFrame(); frame.minCores = 100; - frame.minMemory = CueUtil.GB4; + frame.setMinMemory(CueUtil.GB4); frame.threadable = true; VirtualProc proc = VirtualProc.build(host, frame); @@ -102,7 +102,7 @@ public void testCoreSpanTest3() { DispatchFrame frame = new DispatchFrame(); frame.minCores = 100; - frame.minMemory = memReservedDefault; + frame.setMinMemory(memReservedDefault); frame.threadable = true; VirtualProc proc = VirtualProc.build(host, frame); @@ -117,7 +117,7 @@ public void testCoreSpanTest4() { DispatchFrame frame = new DispatchFrame(); frame.minCores = 100; - frame.minMemory = CueUtil.GB * 8; + frame.setMinMemory(CueUtil.GB * 8); frame.threadable = true; VirtualProc proc = VirtualProc.build(host, frame); @@ -141,7 +141,7 @@ public void testBuildVirtualProc() { DispatchFrame frame = new DispatchFrame(); frame.minCores = 100; - frame.minMemory = memReservedDefault; + frame.setMinMemory(memReservedDefault); frame.threadable = true; proc = VirtualProc.build(host, frame); diff --git a/proto/rqd.proto b/proto/rqd.proto index 73216c6da..327233bb0 100644 --- a/proto/rqd.proto +++ b/proto/rqd.proto @@ -115,6 +115,8 @@ message RunFrame { int32 num_gpus = 23; report.ChildrenProcStats children = 24; string os = 25; + int64 soft_memory_limit = 26; + int64 hard_memory_limit = 27; } message RunFrameSeq { diff --git a/rqd/rqd/rqcore.py b/rqd/rqd/rqcore.py index 5beb73b76..4c3a04199 100644 --- a/rqd/rqd/rqcore.py +++ b/rqd/rqd/rqcore.py @@ -1039,6 +1039,20 @@ def runDocker(self): command.replace(tempPassword, "[password]").replace(";", "\n"), prependTimestamp=rqd.rqconstants.RQD_PREPEND_TIMESTAMP) + # Handle memory limits. Cuebot users KB docker uses Bytes. + # Docker min requirement is 6MB, if request is bellow limit, give the frame a reasonable + # amount of memory. + soft_memory_limit = runFrame.soft_memory_limit * 1000 + if soft_memory_limit <= 6291456: + logging.warning("Frame requested %s bytes of soft_memory_limit, which is lower than " + "minimum required. Running with 1MB", soft_memory_limit) + soft_memory_limit = "1GB" + hard_memory_limit = runFrame.hard_memory_limit * 1000 + if hard_memory_limit <= 6291456: + logging.warning("Frame requested %s bytes of hard_memory_limit, which is lower than " + "minimum required. Running with 2MB", hard_memory_limit) + hard_memory_limit = "2GB" + # Write command to a file on the job tmpdir to simplify replaying a frame command = self._createCommandFile(command) docker_client = self.rqCore.docker.from_env() @@ -1058,6 +1072,8 @@ def runDocker(self): network="host", stderr=True, hostname=self.frameEnv["jobhost"], + mem_reservation=soft_memory_limit, + mem_limit=hard_memory_limit, entrypoint=command) log_stream = container.logs(stream=True) diff --git a/rqd/tests/rqcore_test.py b/rqd/tests/rqcore_test.py index 82e585147..16ea9426b 100644 --- a/rqd/tests/rqcore_test.py +++ b/rqd/tests/rqcore_test.py @@ -701,6 +701,8 @@ def test_runDocker(self, getTempDirMock, permsUser, timeMock, popenMock): frameUid = 928 frameUsername = 'my-random-user' returnCode = 0 + softLimit = 2000000000 + hardLimit = 5000000000 renderHost = rqd.compiled_proto.report_pb2.RenderHost(name='arbitrary-host-name') logFile = os.path.join(logDir, '%s.%s.rqlog' % (jobName, frameName)) @@ -731,6 +733,7 @@ def test_runDocker(self, getTempDirMock, permsUser, timeMock, popenMock): children = rqd.compiled_proto.report_pb2.ChildrenProcStats() + # Test Valid memory limit runFrame = rqd.compiled_proto.rqd_pb2.RunFrame( frame_id=frameId, job_name=jobName, @@ -740,7 +743,9 @@ def test_runDocker(self, getTempDirMock, permsUser, timeMock, popenMock): log_dir=logDir, children=children, environment={"ENVVAR": "env_value"}, - os="centos7") + os="centos7", + soft_memory_limit=softLimit, + hard_memory_limit=hardLimit) frameInfo = rqd.rqnetwork.RunningFrame(rqCore, runFrame) # when @@ -761,6 +766,8 @@ def test_runDocker(self, getTempDirMock, permsUser, timeMock, popenMock): network="host", stderr=True, hostname=mock.ANY, + mem_reservation=softLimit*1000, + mem_limit=hardLimit*1000, entrypoint=cmd_file ) @@ -771,6 +778,44 @@ def test_runDocker(self, getTempDirMock, permsUser, timeMock, popenMock): frameInfo ) + ### Test minimum memory limit + runFrame = rqd.compiled_proto.rqd_pb2.RunFrame( + frame_id=frameId, + job_name=jobName, + frame_name=frameName, + uid=frameUid, + user_name=frameUsername, + log_dir=logDir, + children=children, + environment={"ENVVAR": "env_value"}, + os="centos7", + soft_memory_limit=1, + hard_memory_limit=2) + frameInfo = rqd.rqnetwork.RunningFrame(rqCore, runFrame) + + # when + attendantThread = rqd.rqcore.FrameAttendantThread(rqCore, runFrame, frameInfo) + attendantThread.start() + attendantThread.join() + + # then + cmd_file = os.path.join(tempDir, 'rqd-cmd-%s-%s' % (runFrame.frame_id, currentTime)) + rqCore.docker.from_env.return_value.containers.run.assert_called_with( + image="centos7_image", + detach=True, + environment=mock.ANY, + working_dir=jobTempPath, + mounts=rqCore.docker_mounts, + privileged=True, + pid_mode="host", + network="host", + stderr=True, + hostname=mock.ANY, + mem_reservation="1GB", + mem_limit="2GB", + entrypoint=cmd_file + ) + # TODO(bcipriano) Re-enable this test once Windows is supported. The main sticking point here # is that the log directory is always overridden on Windows which makes mocking difficult.