Various documentation improvements (#912)

# Pull Request

## Title

Various documentation improvements

---

## Description

- [x] Some spelling fixups
- [x] sidebar navigation depth tweaks
- [x] Tunables improvements
- [x] Split type related things out from tunable.py to tunable_types.py
  - [x] Associated import fixups
  - [x] Add Examples to Tunable class
- [x] Optimizer documentation improvements
  - [x] Add more examples
  - [x] Add more cross references
- [x] "See Also" rendering mixups.
- [x] mlos_viz dabl doc fixups
- [x] LlamaTune doc fixups
- [ ] ~Environments~ (future PR)
- [ ] ~Services~ (future PR)
- [x] Disable Azure intersphinx resolution:
Azure/azure-sdk-for-python#39316,
Quansight-Labs/intersphinx_registry#57

See Also: #891

---

## Type of Change

- Documentation update
- Refactor

---

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

Author: bpkroth

.cspell.json

@@ -35,6 +35,7 @@
         "iloc",
         "ipykernel",
         "iterrows",
+        "jsons",
         "jsonschema",
         "jupyterlab",
         "keepalive",

doc/nginx-docker.sh

@@ -12,6 +12,8 @@ scriptpath=$(readlink -f "$0")
 scriptdir=$(dirname "$scriptpath")
 cd "$scriptdir"
 
+SKIP_NGINX_BUILD=${SKIP_NGINX_BUILD:-false}
+
 if [ -f ../.devcontainer/.env ]; then
     source ../.devcontainer/.env
 fi
@@ -28,13 +30,15 @@ cmd="${1:-}"
 if [ "$cmd" == 'start' ]; then
     set -x
     tmpdir=$(mktemp -d)
-    docker build --progress=plain -t mlos-doc-nginx \
-        --build-arg http_proxy=${http_proxy:-} \
-        --build-arg https_proxy=${https_proxy:-} \
-        --build-arg no_proxy=${no_proxy:-} \
-        --build-arg NGINX_PORT=$NGINX_PORT \
-        -f Dockerfile "$tmpdir"
-    rmdir "$tmpdir"
+    if ! $SKIP_NGINX_BUILD; then
+        docker build --progress=plain -t mlos-doc-nginx \
+            --build-arg http_proxy=${http_proxy:-} \
+            --build-arg https_proxy=${https_proxy:-} \
+            --build-arg no_proxy=${no_proxy:-} \
+            --build-arg NGINX_PORT=$NGINX_PORT \
+            -f Dockerfile "$tmpdir"
+        rmdir "$tmpdir"
+    fi
     docker run -d --name mlos-doc-nginx \
         -v "$repo_root/doc/nginx-default.conf":/etc/nginx/templates/default.conf.template \
         -v "$repo_root/doc":/doc \

doc/source/conf.py

@@ -136,8 +136,10 @@ def is_on_github_actions():
 intersphinx_mapping = get_intersphinx_mapping(
     packages={
         "asyncssh",
-        "azure-core",
-        "azure-identity",
+        # Azure SDKs removed their intersphinx publishing.
+        # https://github.com/Azure/azure-sdk-for-python/issues/39316
+        # "azure-core",
+        # "azure-identity",
         "configspace",
         "matplotlib",
         "numpy",
@@ -166,12 +168,15 @@ def is_on_github_actions():
     ("ConcreteOptimizer", "class"): "data",
     ("ConcreteSpaceAdapter", "class"): "data",
     ("DistributionName", "class"): "data",
+    ("mlos_bench.tunables.tunable_types.DistributionName", "class"): "data",
     ("FlamlDomain", "class"): "data",
     ("mlos_core.spaces.converters.flaml.FlamlDomain", "class"): "data",
     ("TunableValue", "class"): "data",
-    ("mlos_bench.tunables.tunable.TunableValue", "class"): "data",
+    ("mlos_bench.tunables.tunable_types.TunableValue", "class"): "data",
     ("TunableValueType", "class"): "data",
+    ("mlos_bench.tunables.tunable_types.TunableValueType", "class"): "data",
     ("TunableValueTypeName", "class"): "data",
+    ("mlos_bench.tunables.tunable_types.TunableValueTypeName", "class"): "data",
     ("T_co", "class"): "data",
     ("CoroReturnType", "class"): "data",
     ("FutureReturnType", "class"): "data",
@@ -230,6 +235,9 @@ def setup(app: SphinxApp) -> None:
 nitpick_ignore_regex = [
     # Ignore some external references that don't use sphinx for their docs.
     (r"py:.*", r"flaml\..*"),
+    # Azure SDKs removed their intersphinx publishing.
+    # https://github.com/Azure/azure-sdk-for-python/issues/39316
+    (r"py:.*", r"azure\..*"),
 ]
 
 # Which documents to include in the build.
@@ -295,6 +303,11 @@ def setup(app: SphinxApp) -> None:
 #
 html_theme = "sphinx_rtd_theme"
 
+html_theme_options = {
+    "canonical_url": "https://microsoft.github.io/MLOS/",
+    "navigation_depth": -1,
+}
+
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".

doc/source/index.rst

@@ -10,6 +10,8 @@ MLOS Documentation
 
 Documentation is generated from both the `source tree markdown <source_tree_docs/index.html>`_ and the Python docstrings for each of the packages with navigation links on the side.
 
+MLOS also has a fairly large test suite, so when in doubt, check the `code <https://github.com/microsoft/MLOS>`_ for additional examples or reach out to the maintainers on `Github <https://github.com/microsoft/MLOS>`_!
+
 .. toctree::
    :caption: Source Tree Documentation
    :maxdepth: 4

mlos_bench/mlos_bench/__init__.py

@@ -15,7 +15,9 @@
 ``mlos_bench`` can be installed from `pypi <https://pypi.org/project/mlos-bench>`_
 via ``pip install mlos-bench`` and executed using the ``mlos_bench`` `command
 <../../mlos_bench.run.usage.html>`_ using a collection of `json` `configs
-<https://github.com/microsoft/MLOS/tree/main/mlos_bench/mlos_bench/config/>`_.
+<https://github.com/microsoft/MLOS/tree/main/mlos_bench/mlos_bench/config/>`_
+which are managed and documented in the :py:mod:`~mlos_bench.config` module and
+elsewhere in this package documentation.
 
 It is intended to be used with :py:mod:`mlos_core` via
 :py:class:`~mlos_bench.optimizers.mlos_core_optimizer.MlosCoreOptimizer` to help
@@ -59,8 +61,9 @@
 benchmarking environments or autotuning experiments.
 
 Where possible, the framework will provide common configs for reuse (e.g., deploying
-a VM on Azure, or run benchbase against a database system) to allow users to focus
-on the specifics of their experiments.
+a VM on Azure, or run `benchbase <https://github.com/cmu-db/benchbase>`_ against
+a database system) to allow users to focus on the specifics of their
+experiments.
 
 Where none are currently available, one can create them external to MLOS, however
 users are also encouraged to `submit PRs or Issues
@@ -125,18 +128,20 @@
 harder to scale to many users.
 
 With these features as a part of the MLOS ecosystem, benchmarking can become a
-*service* that any developer, admin, research, etc. can use and adapt.
+*service* that any developer, admin, researcher, etc. can use and adapt.
 
 See below for more information on the classes in this package.
 
 Notes
 -----
-Note that while the docstrings in this package are generated from the source code
-and hence sometimes more focused on the implementation details, most user
-interactions with the package will be through the `json configs
-<https://github.com/microsoft/MLOS/tree/main/mlos_bench/mlos_bench/config/>`_. Even
-so it may be useful to look at the source code to understand how those are
-interpreted.
+Note that while the docstrings in this package are generated from the source
+code and hence sometimes more focused on the implementation details, we do try
+to provide some (testable) examples here, but most user interactions with the
+package will be through the `json configs
+<https://github.com/microsoft/MLOS/tree/main/mlos_bench/mlos_bench/config/>`_.
+Even so it may be useful to look at the documentation here (perhaps especially
+starting with :py:mod:`mlos_bench.config`) and, if necessary, the `source code
+<https://github.com/microsoft/MLOS>`_ to understand how those are interpreted.
 
 Examples
 --------

mlos_bench/mlos_bench/config/__init__.py

@@ -208,12 +208,14 @@
 Here is a list of some well known variables that are provided or required by the
 system and may be used in the config files:
 
-- ``$experiment_id``: A unique identifier for the experiment.
+- ``$experiment_id``: A unique identifier for the ``Experiment``.
     Typically provided in globals.
-- ``$trial_id``: A unique identifier for the trial currently being executed.
+- ``$trial_id``: A unique identifier for the ``Trial`` currently being executed.
     This can be useful in the configs for :py:mod:`mlos_bench.environments` for
     instance (e.g., when writing scripts).
-- TODO: Document more variables here.
+- ``$trial_runner_id``: A unique identifier for the ``TrialRunner``.
+    This can be useful when running multiple trials in parallel (e.g., to
+    provision a numbered VM per worker).
 
 Tunable Configs
 ^^^^^^^^^^^^^^^

mlos_bench/mlos_bench/config/schemas/__init__.py

@@ -7,8 +7,8 @@
 
 See Also
 --------
-mlos_bench.config.schemas.config_schemas : The module handling the actual schema
-    definitions and validation.
+mlos_bench.config.schemas.config_schemas :
+    The module handling the actual schema definitions and validation.
 """
 
 from mlos_bench.config.schemas.config_schemas import CONFIG_SCHEMA_DIR, ConfigSchema

mlos_bench/mlos_bench/environments/__init__.py

@@ -116,11 +116,12 @@
 
 See Also
 --------
-:py:mod:`mlos_bench.config` : Overview of the configuration system.
-:py:mod:`mlos_bench.services` : Overview of the Services available to the
-    Environments and their configurations.
-:py:mod:`mlos_bench.tunables` : Overview of the Tunables available to the
-    Environments and their configurations.
+:py:mod:`mlos_bench.config` :
+    Overview of the configuration system.
+:py:mod:`mlos_bench.services` :
+    Overview of the Services available to the Environments and their configurations.
+:py:mod:`mlos_bench.tunables` :
+    Overview of the Tunables available to the Environments and their configurations.
 """
 
 from mlos_bench.environments.base_environment import Environment

mlos_bench/mlos_bench/environments/base_environment.py

@@ -19,8 +19,8 @@
 from mlos_bench.dict_templater import DictTemplater
 from mlos_bench.environments.status import Status
 from mlos_bench.services.base_service import Service
-from mlos_bench.tunables.tunable import TunableValue
 from mlos_bench.tunables.tunable_groups import TunableGroups
+from mlos_bench.tunables.tunable_types import TunableValue
 from mlos_bench.util import instantiate_from_config, merge_parameters
 
 if TYPE_CHECKING:

mlos_bench/mlos_bench/environments/composite_env.py

@@ -12,8 +12,8 @@
 from mlos_bench.environments.base_environment import Environment
 from mlos_bench.environments.status import Status
 from mlos_bench.services.base_service import Service
-from mlos_bench.tunables.tunable import TunableValue
 from mlos_bench.tunables.tunable_groups import TunableGroups
+from mlos_bench.tunables.tunable_types import TunableValue
 
 _LOG = logging.getLogger(__name__)
 

mlos_bench/mlos_bench/environments/local/local_env.py

@@ -25,8 +25,8 @@
 from mlos_bench.environments.status import Status
 from mlos_bench.services.base_service import Service
 from mlos_bench.services.types.local_exec_type import SupportsLocalExec
-from mlos_bench.tunables.tunable import TunableValue
 from mlos_bench.tunables.tunable_groups import TunableGroups
+from mlos_bench.tunables.tunable_types import TunableValue
 from mlos_bench.util import datetime_parser, path_join
 
 _LOG = logging.getLogger(__name__)

mlos_bench/mlos_bench/environments/local/local_fileshare_env.py

@@ -17,8 +17,8 @@
 from mlos_bench.services.base_service import Service
 from mlos_bench.services.types.fileshare_type import SupportsFileShareOps
 from mlos_bench.services.types.local_exec_type import SupportsLocalExec
-from mlos_bench.tunables.tunable import TunableValue
 from mlos_bench.tunables.tunable_groups import TunableGroups
+from mlos_bench.tunables.tunable_types import TunableValue
 
 _LOG = logging.getLogger(__name__)
 

mlos_bench/mlos_bench/environments/mock_env.py

@@ -14,8 +14,9 @@
 from mlos_bench.environments.base_environment import Environment
 from mlos_bench.environments.status import Status
 from mlos_bench.services.base_service import Service
-from mlos_bench.tunables.tunable import Tunable, TunableValue
+from mlos_bench.tunables.tunable import Tunable
 from mlos_bench.tunables.tunable_groups import TunableGroups
+from mlos_bench.tunables.tunable_types import TunableValue
 
 _LOG = logging.getLogger(__name__)
 

mlos_bench/mlos_bench/environments/remote/remote_env.py

@@ -22,8 +22,8 @@
 from mlos_bench.services.base_service import Service
 from mlos_bench.services.types.host_ops_type import SupportsHostOps
 from mlos_bench.services.types.remote_exec_type import SupportsRemoteExec
-from mlos_bench.tunables.tunable import TunableValue
 from mlos_bench.tunables.tunable_groups import TunableGroups
+from mlos_bench.tunables.tunable_types import TunableValue
 
 _LOG = logging.getLogger(__name__)
 

mlos_bench/mlos_bench/environments/script_env.py

@@ -16,8 +16,8 @@
 
 from mlos_bench.environments.base_environment import Environment
 from mlos_bench.services.base_service import Service
-from mlos_bench.tunables.tunable import TunableValue
 from mlos_bench.tunables.tunable_groups import TunableGroups
+from mlos_bench.tunables.tunable_types import TunableValue
 from mlos_bench.util import try_parse_val
 
 _LOG = logging.getLogger(__name__)

mlos_bench/mlos_bench/launcher.py

@@ -29,8 +29,8 @@
 from mlos_bench.services.local.local_exec import LocalExecService
 from mlos_bench.services.types.config_loader_type import SupportsConfigLoading
 from mlos_bench.storage.base_storage import Storage
-from mlos_bench.tunables.tunable import TunableValue
 from mlos_bench.tunables.tunable_groups import TunableGroups
+from mlos_bench.tunables.tunable_types import TunableValue
 from mlos_bench.util import try_parse_val
 
 _LOG_LEVEL = logging.INFO