Issue in running AutoRunner #1251
Description
What could be the reason for the auto3dseg not printing any loss results when running for a custom dataset, and only showing results when interrupting the program with "ctrl+c"?
if I give "ctrl+c"
with an error;
Traceback (most recent call last):
File "/home/jovyan/work/dints_0/scripts/search.py", line 647, in
fire.Fire()
File "/home/jovyan/conda/lib/python3.8/site-packages/fire/core.py", line 141, in Fire
component_trace = _Fire(component, args, parsed_flag_args, context, name)
File "/home/jovyan/conda/lib/python3.8/site-packages/fire/core.py", line 475, in _Fire
component, remaining_args = _CallAndUpdateTrace(
File "/home/jovyan/conda/lib/python3.8/site-packages/fire/core.py", line 691, in _CallAndUpdateTrace
component = fn(*varargs, **kwargs)
File "/home/jovyan/work/dints_0/scripts/search.py", line 496, in run
val_outputs = sliding_window_inference(
File "/home/jovyan/MONAI/monai/inferers/utils.py", line 190, in sliding_window_inference
seg_prob_out = predictor(window_data, *args, **kwargs) # batched patch segmentation
File "/home/jovyan/conda/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1130, in _call_impl
return forward_call(*input, **kwargs)
File "/home/jovyan/MONAI/monai/networks/nets/dints.py", line 497, in forward
outputs = self.dints_space(inputs)
File "/home/jovyan/conda/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1130, in _call_impl
return forward_call(*input, **kwargs)
File "/home/jovyan/MONAI/monai/networks/nets/dints.py", line 1033, in forward
for res_idx, activation in enumerate(self.arch_code_a[blk_idx].data.cpu().numpy()):
KeyboardInterrupt
KeyboardInterrupt Traceback (most recent call last)
Input In [26], in <cell line: 1>()
----> 1 runner.run()
File ~/MONAI/monai/apps/auto3dseg/auto_runner.py:682, in AutoRunner.run(self)
677 raise ValueError(
678 f"Could not find training scripts in {self.work_dir}. "
679 "Possibly the required algorithms generation step was not completed."
680 )
681 if not self.hpo:
--> 682 self._train_algo_in_sequence(history)
683 else:
684 self._train_algo_in_nni(history)
File ~/MONAI/monai/apps/auto3dseg/auto_runner.py:556, in AutoRunner._train_algo_in_sequence(self, history)
554 for task in history:
555 for _, algo in task.items():
--> 556 algo.train(self.train_params)
557 acc = algo.get_score()
558 algo_to_pickle(algo, template_path=algo.template_path, best_metrics=acc)
File ~/work/algorithm_templates/dints/scripts/algo.py:426, in DintsAlgo.train(self, train_params)
424 cmd, devices_info = self._create_cmd(dints_search_params)
425 cmd_search = cmd.replace("train.py", "search.py")
--> 426 self._run_cmd(cmd_search, devices_info)
428 # training
429 dints_train_params = {}
File ~/MONAI/monai/apps/auto3dseg/bundle_gen.py:191, in BundleAlgo._run_cmd(self, cmd, devices_info)
189 if devices_info:
190 ps_environ["CUDA_VISIBLE_DEVICES"] = devices_info
--> 191 normal_out = subprocess.run(cmd.split(), env=ps_environ, check=True)
193 return normal_out
File ~/conda/lib/python3.8/subprocess.py:495, in run(input, capture_output, timeout, check, *popenargs, **kwargs)
493 with Popen(*popenargs, **kwargs) as process:
494 try:
--> 495 stdout, stderr = process.communicate(input, timeout=timeout)
496 except TimeoutExpired as exc:
497 process.kill()
File ~/conda/lib/python3.8/subprocess.py:1020, in Popen.communicate(self, input, timeout)
1018 stderr = self.stderr.read()
1019 self.stderr.close()
-> 1020 self.wait()
1021 else:
1022 if timeout is not None:
File ~/conda/lib/python3.8/subprocess.py:1083, in Popen.wait(self, timeout)
1081 endtime = _time() + timeout
1082 try:
-> 1083 return self._wait(timeout=timeout)
1084 except KeyboardInterrupt:
1085 # https://bugs.python.org/issue25942
1086 # The first keyboard interrupt waits briefly for the child to
1087 # exit under the common assumption that it also received the ^C
1088 # generated SIGINT and will exit rapidly.
1089 if timeout is not None:
File ~/conda/lib/python3.8/subprocess.py:1808, in Popen._wait(self, timeout)
1806 if self.returncode is not None:
1807 break # Another thread waited.
-> 1808 (pid, sts) = self._try_wait(0)
1809 # Check the pid and loop as waitpid has been known to
1810 # return 0 even without WNOHANG in odd situations.
1811 # http://bugs.python.org/issue14396.
1812 if pid == self.pid:
File ~/conda/lib/python3.8/subprocess.py:1766, in Popen._try_wait(self, wait_flags)
1764 """All callers to this function MUST hold self._waitpid_lock."""
1765 try:
-> 1766 (pid, sts) = os.waitpid(self.pid, wait_flags)
1767 except ChildProcessError:
1768 # This happens if SIGCLD is set to be ignored or waiting
1769 # for child processes has otherwise been disabled for our
1770 # process. This child is dead, we can't get the status.
1771 pid = self.pid
KeyboardInterrupt: