Move examples under torchx module (#224)

Summary:
Pull Request resolved: #224

The diff moves `examples` under `torchx` namespace, also removes examples Dockerfile, and makes torchx image to use dev-requirements

Reviewed By: kiukchung

Differential Revision: D31464358

fbshipit-source-id: c56a92ce1fbf5b2881156e57152aa9d40af9eac8

Author: aivanou

.coveragerc

@@ -4,3 +4,4 @@ source =
 
 omit =
     *test*
+    torchx/examples/*

dev-requirements.txt

@@ -14,4 +14,5 @@ torchserve>=0.4.0
 captum>=0.3.1
 importlib-metadata
 ax-platform[mysql]>=0.2.2
-fsspec>=2021.09.0
+fsspec[s3]>=2021.09.0
+torch-model-archiver>=0.4.2

docs/source/conf.py

@@ -324,8 +324,8 @@ def handle_item(fieldarg, content):
 
 sphinx_gallery_conf = {
     "examples_dirs": [
-        "../../examples/apps",
-        "../../examples/pipelines",
+        "../../torchx/examples/apps",
+        "../../torchx/examples/pipelines",
     ],
     "gallery_dirs": [
         "examples_apps",

examples/apps/Dockerfile

@@ -52,12 +52,10 @@ def get_local_docker_sched_info(image: str) -> SchedulerInfo:
 def main() -> None:
     print("Starting components integration tests")
     torchx_image = "dummy_image"
-    examples_image = "dummy_image"
     dryrun: bool = False
     try:
         build = build_and_push_image()
         torchx_image = build.torchx_image
-        examples_image = build.examples_image
     except MissingEnvError:
         dryrun = True
         print("Skip runnig tests, executed only docker buid step")
@@ -75,8 +73,8 @@ def main() -> None:
     test_suite.run_components(
         examples_app_defs_providers,
         scheduler_infos=[
-            get_local_docker_sched_info(examples_image),
-            get_k8s_sched_info(examples_image),
+            get_local_docker_sched_info(torchx_image),
+            get_k8s_sched_info(torchx_image),
         ],
         dryrun=dryrun,
     )

scripts/component_integration_tests.py

@@ -10,9 +10,9 @@
 """
 
 
-import examples.apps.datapreproc.component as dp_component
-import examples.apps.dist_cifar.component as dist_cifar_component
-import examples.apps.lightning_classy_vision.component as cv_component
+import torchx.examples.apps.datapreproc.component as dp_component
+import torchx.examples.apps.dist_cifar.component as dist_cifar_component
+import torchx.examples.apps.lightning_classy_vision.component as cv_component
 from torchx.components.integration_tests.component_provider import ComponentProvider
 from torchx.specs import AppDef
 

scripts/example_app_defs.py

@@ -17,7 +17,6 @@
 class BuildInfo:
     id: str
     torchx_image: str
-    examples_image: str
 
 
 class MissingEnvError(AssertionError):
@@ -41,15 +40,6 @@ def run_in_bg(*args: str) -> "subprocess.Popen[str]":
     return subprocess.Popen(args)
 
 
-def build_examples_canary(id: str) -> str:
-    examples_tag = "torchx_examples_canary"
-
-    print(f"building {examples_tag}")
-    run("docker", "build", "-t", examples_tag, "examples/apps/")
-
-    return examples_tag
-
-
 def build_torchx_canary(id: str) -> str:
     torchx_tag = "torchx_canary"
 
@@ -65,30 +55,18 @@ def torchx_container_tag(id: str) -> str:
     return f"{CONTAINER_REPO}:canary_{id}_torchx"
 
 
-def examples_container_tag(id: str) -> str:
-    CONTAINER_REPO = getenv_asserts("CONTAINER_REPO")
-    return f"{CONTAINER_REPO}:canary_{id}_examples"
-
-
 def build_images() -> BuildInfo:
     id = f"{getuser()}_{random_id()}"
-    examples_image = build_examples_canary(id)
     torchx_image = build_torchx_canary(id)
     return BuildInfo(
         id=id,
         torchx_image=torchx_image,
-        examples_image=examples_image,
     )
 
 
 def push_images(build: BuildInfo) -> None:
-    examples_tag = examples_container_tag(build.id)
-    run("docker", "tag", build.examples_image, examples_tag)
-    build.examples_image = examples_tag
-
     torchx_tag = torchx_container_tag(build.id)
     run("docker", "tag", build.torchx_image, torchx_tag)
     build.torchx_image = torchx_tag
 
-    run("docker", "push", examples_tag)
     run("docker", "push", torchx_tag)

scripts/integ_test_utils.py

@@ -24,7 +24,6 @@
 ```
 export KFP_NAMESPACE=<kfp namespace>
 export INTEGRATION_TEST_STORAGE=<cloud storage path>
-export EXAMPLES_CONTAINER_REPO=<docker repo>
 export TORCHX_CONTAINER_REPO=<docker repo>
 ```
 
@@ -141,7 +140,6 @@ def save_advanced_pipeline_spec(path: str, build: BuildInfo) -> None:
 
     id = build.id
     torchx_image = build.torchx_image
-    examples_image = build.examples_image
 
     STORAGE_PATH = os.getenv("INTEGRATION_TEST_STORAGE", "/tmp/storage")
     root = os.path.join(STORAGE_PATH, id)
@@ -153,8 +151,6 @@ def save_advanced_pipeline_spec(path: str, build: BuildInfo) -> None:
         "--output_path",
         output,
         "--image",
-        examples_image,
-        "--torchx_image",
         torchx_image,
         "--model_name",
         f"tiny_image_net_{id}",
@@ -165,7 +161,7 @@ def save_pipeline_spec(path: str, pipeline_file: str, *args: str) -> None:
     print(f"generating pipeline spec for {pipeline_file}")
 
     with tempfile.TemporaryDirectory() as tmpdir:
-        run(os.path.join("examples/pipelines/kfp", pipeline_file), *args)
+        run(os.path.join("torchx/examples/pipelines/kfp", pipeline_file), *args)
         shutil.copy("pipeline.yaml", path)
 
 
@@ -202,7 +198,6 @@ def run_pipeline(build: BuildInfo, pipeline_file: str) -> object:
     except MaxRetryError:
         print(_connection_error_message())
         raise
-    namespace = getenv_asserts("KFP_NAMESPACE")
     resp = client.create_run_from_pipeline_package(
         pipeline_file,
         arguments={},

scripts/kfpint.py

@@ -12,8 +12,6 @@
 import argparse
 import os
 
-from examples.apps.dist_cifar.component import trainer
-
 # pyre-ignore-all-errors[21] # Cannot find module utils
 # pyre-ignore-all-errors[11]
 from integ_test_utils import (
@@ -23,6 +21,7 @@
     BuildInfo,
 )
 from pyre_extensions import none_throws
+from torchx.examples.apps.dist_cifar.component import trainer
 from torchx.runner import get_runner
 from torchx.specs import Resource, named_resources, RunConfig, AppState
 
@@ -48,7 +47,7 @@ def build_and_push_image() -> BuildInfo:
 def run_job(dryrun: bool = False) -> None:
     register_gpu_resource()
     build = build_and_push_image()
-    image = build.examples_image
+    image = build.torchx_image
     runner = get_runner("kubeflow-dist-runner")
 
     storage_path = os.getenv("INTEGRATION_TEST_STORAGE", "/tmp/storage")

scripts/kube_dist_trainer.py

@@ -140,6 +140,8 @@ def _run(self, runner: Runner, args: argparse.Namespace) -> Optional[str]:
             return
         else:
             app_handle = cast(specs.AppHandle, result)
+            # do not delete this line. It is used by slurm tests to retrieve the app id
+            print(app_handle)
 
             if args.scheduler.startswith("local"):
                 self._wait_and_exit(runner, app_handle)