Merge pull request ericaltendorf#234 from wisherhxl/to_push_tmp_dir

src/plotman/_tests/configuration_test.py

@@ -54,3 +54,33 @@ def test_loads_without_user_interface(config_text):
     reloaded_yaml = configuration.get_validated_configs(stripped_config_text, '')
 
     assert reloaded_yaml.user_interface == configuration.UserInterface()
+
+
+def test_get_dst_directories_gets_dst():
+    tmp = ['/tmp']
+    dst = ['/dst0', '/dst1']
+    directories = configuration.Directories(log='', tmp=tmp, dst=dst)
+
+    assert directories.get_dst_directories() == dst
+
+
+def test_get_dst_directories_gets_tmp():
+    tmp = ['/tmp']
+    directories = configuration.Directories(log='', tmp=tmp)
+
+    assert directories.get_dst_directories() == tmp
+
+
+def test_dst_is_dst():
+    tmp = ['/tmp']
+    dst = ['/dst0', '/dst1']
+    directories = configuration.Directories(log='', tmp=tmp, dst=dst)
+
+    assert not directories.dst_is_tmp()
+
+
+def test_dst_is_tmp():
+    tmp = ['/tmp']
+    directories = configuration.Directories(log='', tmp=tmp)
+
+    assert directories.dst_is_tmp()

src/plotman/archive.py

@@ -134,8 +134,8 @@ def archive(dir_cfg, all_jobs):
     dir2ph = manager.dstdirs_to_furthest_phase(all_jobs)
     best_priority = -100000000
     chosen_plot = None
-
-    for d in dir_cfg.dst:
+    dst_dir = dir_cfg.get_dst_directories()
+    for d in dst_dir:
         ph = dir2ph.get(d, job.Phase(0, 0))
         dir_plots = plot_util.list_k32_plots(d)
         gb_free = plot_util.df_b(d) / plot_util.GB

src/plotman/configuration.py

@@ -65,11 +65,24 @@ class TmpOverrides:
 class Directories:
     log: str
     tmp: List[str]
-    dst: List[str]
+    dst: Optional[List[str]] = None
     tmp2: Optional[str] = None
     tmp_overrides: Optional[Dict[str, TmpOverrides]] = None
     archive: Optional[Archive] = None
 
+    def dst_is_tmp(self):
+        return self.dst is None
+
+    def get_dst_directories(self):
+        """Returns either <Directories.dst> or <Directories.tmp>. If
+        Directories.dst is None, Use Directories.tmp as dst directory.
+        """
+        if self.dst_is_tmp():
+            return self.tmp
+
+        return self.dst
+
+
 @attr.frozen
 class Scheduling:
     global_max_jobs: int

src/plotman/interactive.py

@@ -168,7 +168,8 @@ def curses_main(stdscr):
 
         # Directory prefixes, for abbreviation
         tmp_prefix = os.path.commonpath(cfg.directories.tmp)
-        dst_prefix = os.path.commonpath(cfg.directories.dst)
+        dst_dir = cfg.directories.get_dst_directories()
+        dst_prefix = os.path.commonpath(dst_dir)
         if archiving_configured:
             arch_prefix = cfg.directories.archive.rsyncd_path
 
@@ -178,7 +179,7 @@ def curses_main(stdscr):
         tmp_report = reporting.tmp_dir_report(
             jobs, cfg.directories, cfg.scheduling, n_cols, 0, n_tmpdirs, tmp_prefix)
         dst_report = reporting.dst_dir_report(
-            jobs, cfg.directories.dst, n_cols, dst_prefix)
+            jobs, dst_dir, n_cols, dst_prefix)
         if archiving_configured:
             arch_report = reporting.arch_dir_report(archdir_freebytes, n_cols, arch_prefix)
             if not arch_report:

src/plotman/manager.py

@@ -97,14 +97,18 @@ def maybe_start_new_plot(dir_cfg, sched_cfg, plotting_cfg):
             tmpdir = max(rankable, key=operator.itemgetter(1))[0]
 
             # Select the dst dir least recently selected
-            dir2ph = { d:ph for (d, ph) in dstdirs_to_youngest_phase(jobs).items()
-                      if d in dir_cfg.dst and ph is not None}
-            unused_dirs = [d for d in dir_cfg.dst if d not in dir2ph.keys()]
-            dstdir = ''
-            if unused_dirs:
-                dstdir = random.choice(unused_dirs)
+            dst_dir = dir_cfg.get_dst_directories()
+            if dir_cfg.dst_is_tmp():
+                dstdir = tmpdir
             else:
-                dstdir = max(dir2ph, key=dir2ph.get)
+                dir2ph = { d:ph for (d, ph) in dstdirs_to_youngest_phase(jobs).items()
+                        if d in dst_dir and ph is not None}
+                unused_dirs = [d for d in dst_dir if d not in dir2ph.keys()]
+                dstdir = ''
+                if unused_dirs:
+                    dstdir = random.choice(unused_dirs)
+                else:
+                    dstdir = max(dir2ph, key=dir2ph.get)
 
             logfile = os.path.join(
                 dir_cfg.log, pendulum.now().isoformat(timespec='microseconds').replace(':', '_') + '.log'

src/plotman/reporting.py

@@ -208,9 +208,10 @@ def arch_dir_report(archdir_freebytes, width, prefix=''):
 
 # TODO: remove this
 def dirs_report(jobs, dir_cfg, sched_cfg, width):
+    dst_dir = dir_cfg.get_dst_directories()
     reports = [
         tmp_dir_report(jobs, dir_cfg, sched_cfg, width),
-        dst_dir_report(jobs, dir_cfg.dst, width),
+        dst_dir_report(jobs, dst_dir, width),
     ]
     if dir_cfg.archive is not None:
         reports.extend([

src/plotman/resources/plotman.yaml

@@ -47,10 +47,13 @@ directories:
         # chia plots create as -2.  Only one tmp2 directory is supported.
         # tmp2: /mnt/tmp/a
 
-        # One or more directories; the scheduler will use all of them.
-        # These again are presumed to be on independent physical devices,
-        # so writes (plot jobs) and reads (archivals) can be scheduled
-        # to minimize IO contention.
+        # Optional: A list of one or more directories; the scheduler will
+        # use all of them.  These again are presumed to be on independent
+        # physical devices so writes (plot jobs) and reads (archivals) can
+        # be scheduled to minimize IO contention.
+        # 
+        # If dst is commented out, the tmp directories will be used as the
+        # buffer.
         dst:
                 - /mnt/dst/00
                 - /mnt/dst/01