diff --git a/src/plotman/_tests/configuration_test.py b/src/plotman/_tests/configuration_test.py index f0a548b6..327c66e4 100644 --- a/src/plotman/_tests/configuration_test.py +++ b/src/plotman/_tests/configuration_test.py @@ -54,3 +54,33 @@ def test_loads_without_user_interface(config_text): reloaded_yaml = configuration.get_validated_configs(stripped_config_text, '') assert reloaded_yaml.user_interface == configuration.UserInterface() + + +def test_get_dst_directories_gets_dst(): + tmp = ['/tmp'] + dst = ['/dst0', '/dst1'] + directories = configuration.Directories(log='', tmp=tmp, dst=dst) + + assert directories.get_dst_directories() == dst + + +def test_get_dst_directories_gets_tmp(): + tmp = ['/tmp'] + directories = configuration.Directories(log='', tmp=tmp) + + assert directories.get_dst_directories() == tmp + + +def test_dst_is_dst(): + tmp = ['/tmp'] + dst = ['/dst0', '/dst1'] + directories = configuration.Directories(log='', tmp=tmp, dst=dst) + + assert not directories.dst_is_tmp() + + +def test_dst_is_tmp(): + tmp = ['/tmp'] + directories = configuration.Directories(log='', tmp=tmp) + + assert directories.dst_is_tmp() diff --git a/src/plotman/archive.py b/src/plotman/archive.py index d3bb61db..641ae1d1 100644 --- a/src/plotman/archive.py +++ b/src/plotman/archive.py @@ -134,8 +134,8 @@ def archive(dir_cfg, all_jobs): dir2ph = manager.dstdirs_to_furthest_phase(all_jobs) best_priority = -100000000 chosen_plot = None - - for d in dir_cfg.dst: + dst_dir = dir_cfg.get_dst_directories() + for d in dst_dir: ph = dir2ph.get(d, job.Phase(0, 0)) dir_plots = plot_util.list_k32_plots(d) gb_free = plot_util.df_b(d) / plot_util.GB diff --git a/src/plotman/configuration.py b/src/plotman/configuration.py index 14b40d70..f081e75d 100644 --- a/src/plotman/configuration.py +++ b/src/plotman/configuration.py @@ -65,11 +65,24 @@ class TmpOverrides: class Directories: log: str tmp: List[str] - dst: List[str] + dst: Optional[List[str]] = None tmp2: Optional[str] = None tmp_overrides: Optional[Dict[str, TmpOverrides]] = None archive: Optional[Archive] = None + def dst_is_tmp(self): + return self.dst is None + + def get_dst_directories(self): + """Returns either or . If + Directories.dst is None, Use Directories.tmp as dst directory. + """ + if self.dst_is_tmp(): + return self.tmp + + return self.dst + + @attr.frozen class Scheduling: global_max_jobs: int diff --git a/src/plotman/interactive.py b/src/plotman/interactive.py index d8b7e300..c60d7236 100644 --- a/src/plotman/interactive.py +++ b/src/plotman/interactive.py @@ -168,7 +168,8 @@ def curses_main(stdscr): # Directory prefixes, for abbreviation tmp_prefix = os.path.commonpath(cfg.directories.tmp) - dst_prefix = os.path.commonpath(cfg.directories.dst) + dst_dir = cfg.directories.get_dst_directories() + dst_prefix = os.path.commonpath(dst_dir) if archiving_configured: arch_prefix = cfg.directories.archive.rsyncd_path @@ -178,7 +179,7 @@ def curses_main(stdscr): tmp_report = reporting.tmp_dir_report( jobs, cfg.directories, cfg.scheduling, n_cols, 0, n_tmpdirs, tmp_prefix) dst_report = reporting.dst_dir_report( - jobs, cfg.directories.dst, n_cols, dst_prefix) + jobs, dst_dir, n_cols, dst_prefix) if archiving_configured: arch_report = reporting.arch_dir_report(archdir_freebytes, n_cols, arch_prefix) if not arch_report: diff --git a/src/plotman/manager.py b/src/plotman/manager.py index 78da9608..8c418f35 100644 --- a/src/plotman/manager.py +++ b/src/plotman/manager.py @@ -97,14 +97,18 @@ def maybe_start_new_plot(dir_cfg, sched_cfg, plotting_cfg): tmpdir = max(rankable, key=operator.itemgetter(1))[0] # Select the dst dir least recently selected - dir2ph = { d:ph for (d, ph) in dstdirs_to_youngest_phase(jobs).items() - if d in dir_cfg.dst and ph is not None} - unused_dirs = [d for d in dir_cfg.dst if d not in dir2ph.keys()] - dstdir = '' - if unused_dirs: - dstdir = random.choice(unused_dirs) + dst_dir = dir_cfg.get_dst_directories() + if dir_cfg.dst_is_tmp(): + dstdir = tmpdir else: - dstdir = max(dir2ph, key=dir2ph.get) + dir2ph = { d:ph for (d, ph) in dstdirs_to_youngest_phase(jobs).items() + if d in dst_dir and ph is not None} + unused_dirs = [d for d in dst_dir if d not in dir2ph.keys()] + dstdir = '' + if unused_dirs: + dstdir = random.choice(unused_dirs) + else: + dstdir = max(dir2ph, key=dir2ph.get) logfile = os.path.join( dir_cfg.log, pendulum.now().isoformat(timespec='microseconds').replace(':', '_') + '.log' diff --git a/src/plotman/reporting.py b/src/plotman/reporting.py index 4149d913..50cc9a9a 100644 --- a/src/plotman/reporting.py +++ b/src/plotman/reporting.py @@ -208,9 +208,10 @@ def arch_dir_report(archdir_freebytes, width, prefix=''): # TODO: remove this def dirs_report(jobs, dir_cfg, sched_cfg, width): + dst_dir = dir_cfg.get_dst_directories() reports = [ tmp_dir_report(jobs, dir_cfg, sched_cfg, width), - dst_dir_report(jobs, dir_cfg.dst, width), + dst_dir_report(jobs, dst_dir, width), ] if dir_cfg.archive is not None: reports.extend([ diff --git a/src/plotman/resources/plotman.yaml b/src/plotman/resources/plotman.yaml index fe8bd187..060989b8 100644 --- a/src/plotman/resources/plotman.yaml +++ b/src/plotman/resources/plotman.yaml @@ -47,10 +47,13 @@ directories: # chia plots create as -2. Only one tmp2 directory is supported. # tmp2: /mnt/tmp/a - # One or more directories; the scheduler will use all of them. - # These again are presumed to be on independent physical devices, - # so writes (plot jobs) and reads (archivals) can be scheduled - # to minimize IO contention. + # Optional: A list of one or more directories; the scheduler will + # use all of them. These again are presumed to be on independent + # physical devices so writes (plot jobs) and reads (archivals) can + # be scheduled to minimize IO contention. + # + # If dst is commented out, the tmp directories will be used as the + # buffer. dst: - /mnt/dst/00 - /mnt/dst/01