[CI] : Add customizations for ci build/test scripts.

Author: sahas3

build_tools/ci/build_posix.sh

@@ -2,19 +2,40 @@
 
 set -eu -o errtrace
 
-this_dir="$(cd $(dirname $0) && pwd)"
-repo_root="$(cd $this_dir/../.. && pwd)"
-build_dir="$repo_root/build"
+this_dir="$(cd "$(dirname "$0")" && pwd)"
+repo_root="$(cd "$this_dir"/../.. && pwd)"
 install_dir="$repo_root/install"
-mkdir -p "$build_dir"
-build_dir="$(cd $build_dir && pwd)"
 cache_dir="${cache_dir:-}"
+build_dir="${build_dir:-}"
+torch_version="${1:-nightly}"
+
+case $torch_version in
+    nightly)
+    build_dir_name=build
+    ;;
+    stable)
+    build_dir_name=build_stable
+    ;;
+    *)
+    echo "Unrecognized torch version '$torch_version' (specify 'nightly' or 'stable' with cl arg)"
+    exit 1
+    ;;
+esac
+
+# Setup build dir
+if [ -z "${build_dir}" ]; then
+  build_dir="${repo_root}/${build_dir_name}"
+  mkdir -p "$build_dir"
+  build_dir="$(cd "${build_dir}" && pwd)"
+fi
+
+echo "Building in ${build_dir}"
 
 # Setup cache dir.
 if [ -z "${cache_dir}" ]; then
   cache_dir="${repo_root}/.build-cache"
   mkdir -p "${cache_dir}"
-  cache_dir="$(cd ${cache_dir} && pwd)"
+  cache_dir="$(cd "${cache_dir}" && pwd)"
 fi
 echo "Caching to ${cache_dir}"
 mkdir -p "${cache_dir}/ccache"
@@ -34,7 +55,7 @@ export CMAKE_CXX_COMPILER_LAUNCHER=ccache
 # Clear ccache stats.
 ccache -z
 
-cd $repo_root
+cd "$repo_root"
 
 echo "::group::CMake configure"
 cmake -S "$repo_root/externals/llvm-project/llvm" -B "$build_dir" \
@@ -59,7 +80,7 @@ cmake --build "$build_dir" --target tools/torch-mlir/all -- -k 0
 echo "::endgroup::"
 
 echo "::group::Unit tests"
-cmake --build $repo_root/build --target check-torch-mlir-all
+cmake --build "$build_dir" --target check-torch-mlir-all
 echo "::endgroup::"
 
 # Show ccache stats.

build_tools/ci/check_generated_sources.sh

@@ -2,15 +2,15 @@
 
 set -eu -o errtrace
 
-this_dir="$(cd $(dirname $0) && pwd)"
-repo_root="$(cd $this_dir/../.. && pwd)"
+this_dir="$(cd "$(dirname "$0")" && pwd)"
+repo_root="$(cd "$this_dir"/../.. && pwd)"
 
 function _check_file_not_changed_by() {
   # _check_file_not_changed_by <cmd> <file>
   cmd="$1"
   file="$2"
-  file_backup="$PWD/$(basename $file)"
-  file_new="$PWD/$(basename $file).new"
+  file_backup="$PWD/$(basename "$file")"
+  file_new="$PWD/$(basename "$file").new"
   # Save the original file.
   cp "$file" "$file_backup"
   # Run the command to regenerate it.
@@ -37,9 +37,9 @@ function _check_file_not_changed_by() {
 }
 
 echo "::group:: Check that update_abstract_interp_lib.sh has been run"
-_check_file_not_changed_by $repo_root/build_tools/update_abstract_interp_lib.sh $repo_root/lib/Dialect/Torch/Transforms/AbstractInterpLibrary.cpp
+_check_file_not_changed_by "$repo_root"/build_tools/update_abstract_interp_lib.sh "$repo_root"/lib/Dialect/Torch/Transforms/AbstractInterpLibrary.cpp
 echo "::endgroup::"
 
 echo "::group:: Check that update_torch_ods.sh has been run"
-_check_file_not_changed_by $repo_root/build_tools/update_torch_ods.sh $repo_root/include/torch-mlir/Dialect/Torch/IR/GeneratedTorchOps.td
+_check_file_not_changed_by "$repo_root"/build_tools/update_torch_ods.sh "$repo_root"/include/torch-mlir/Dialect/Torch/IR/GeneratedTorchOps.td
 echo "::endgroup::"

build_tools/ci/install_python_deps.sh

@@ -2,21 +2,30 @@
 
 set -eu -o errtrace
 
-this_dir="$(cd $(dirname $0) && pwd)"
-repo_root="$(cd $this_dir/../.. && pwd)"
+this_dir="$(cd "$(dirname "$0")" && pwd)"
+repo_root="$(cd "$this_dir"/../.. && pwd)"
 torch_version="${1:-unknown}"
 
+
+echo "::group::upgrading pip"
+python3 -m pip install --upgrade pip
+echo "::endgroup::"
+
+echo "::group::installing llvm python deps"
+python3 -m pip install --no-cache-dir -r "$repo_root"/externals/llvm-project/mlir/python/requirements.txt
+echo "::endgroup::"
+
 case $torch_version in
   nightly)
     echo "::group::installing nightly torch"
-    python3 -m pip install --no-cache-dir -r $repo_root/requirements.txt
-    python3 -m pip install --no-cache-dir -r $repo_root/torchvision-requirements.txt
+    python3 -m pip install --no-cache-dir -r "$repo_root"/requirements.txt
+    python3 -m pip install --no-cache-dir -r "$repo_root"/torchvision-requirements.txt
     echo "::endgroup::"
     ;;
   stable)
     echo "::group::installing stable torch"
     python3 -m pip install --no-cache-dir torch torchvision --index-url https://download.pytorch.org/whl/cpu
-    python3 -m pip install --no-cache-dir -r $repo_root/build-requirements.txt
+    python3 -m pip install --no-cache-dir -r "$repo_root"/build-requirements.txt
     echo "::endgroup::"
     ;;
   *)
@@ -26,5 +35,5 @@ case $torch_version in
 esac
 
 echo "::group::installing test requirements"
-python3 -m pip install --no-cache-dir -r $repo_root/test-requirements.txt
+python3 -m pip install --no-cache-dir -r "$repo_root"/test-requirements.txt
 echo "::endgroup::"

build_tools/ci/test_posix.sh

@@ -2,18 +2,36 @@
 
 set -eu -o errtrace
 
-this_dir="$(cd $(dirname $0) && pwd)"
-repo_root="$(cd $this_dir/../.. && pwd)"
-torch_version="${1:-unknown}"
+this_dir="$(cd "$(dirname "$0")" && pwd)"
+repo_root="$(cd "$this_dir"/../.. && pwd)"
+torch_version="${1:-nightly}"
+build_dir="${build_dir:-}"
 
-export PYTHONPATH="$repo_root/build/tools/torch-mlir/python_packages/torch_mlir:$repo_root/projects/pt1"
+case $torch_version in
+    nightly)
+    build_dir_name=build
+    ;;
+    stable)
+    build_dir_name=build_stable
+    ;;
+    *)
+    echo "Unrecognized torch version '$torch_version' (specify 'nightly' or 'stable' with cl arg)"
+    exit 1
+    ;;
+esac
+
+if [ -z "${build_dir}" ]; then
+  build_dir="${repo_root}/${build_dir_name}"
+  build_dir="$(cd "${build_dir}" && pwd)"
+fi
+
+export PYTHONPATH="$build_dir/tools/torch-mlir/python_packages/torch_mlir:$repo_root/projects/pt1"
 
 echo "::group::Run ONNX e2e integration tests"
 python3 -m e2e_testing.main --config=onnx -v
 echo "::endgroup::"
 
-case $torch_version in
-  nightly)
+if [ "$torch_version" = "nightly" ]; then
     # Failing with: NotImplementedError:
     #   Could not run 'aten::empty.memory_format' with arguments from the 'Lazy' backend.
     # As of 2024-01-07
@@ -35,11 +53,4 @@ case $torch_version in
     echo "::group::Run FxImporter TOSA e2e integration tests"
     python3 -m e2e_testing.main --config=fx_importer_tosa -v
     echo "::endgroup::"
-    ;;
-  stable)
-    ;;
-  *)
-    echo "Unrecognized torch version '$torch_version' (specify 'nightly' or 'stable' with cl arg)"
-    exit 1
-    ;;
-esac
+fi