Skip to content

Commit 1a91eba

Browse files
fschlimbfschlimb
authored
Conda build (#91)
* ading conda recipe * using conda build in ci * fixing compiler errors and warnings
1 parent 9e8bafa commit 1a91eba

28 files changed

+315
-214
lines changed

‎.github/workflows/ci.yml

Lines changed: 51 additions & 109 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,15 @@
11
name: GitHub Actions Demo
22
run-name: ${{ github.actor }} CI for sharpy
3-
on: [push, pull_request]
3+
on:
4+
push:
5+
branches:
6+
- main
7+
pull_request:
48
defaults:
59
run:
610
shell: bash
11+
env:
12+
IMEX_CACHE_NUMBER: 4 # Increase to reset cache
713
jobs:
814
build_and_test:
915
runs-on: tpi-ubuntu-latest
@@ -14,122 +20,61 @@ jobs:
1420
uses: actions/checkout@v3
1521
with:
1622
submodules: recursive
17-
- name: Setup IMEX Cache Var
23+
- name: Setup env
1824
run: |
25+
INSTALLED_DIR="$(realpath $GITHUB_WORKSPACE)/third_party/install"
26+
echo "INSTALLED_DIR=$INSTALLED_DIR" >> "$GITHUB_ENV"
27+
CONDA_ROOT="$(realpath $GITHUB_WORKSPACE/..)/install/miniconda3"
28+
echo "CONDA_ROOT=$CONDA_ROOT" >> "$GITHUB_ENV"
1929
echo 'IMEX_SHA<<EOF' >> $GITHUB_ENV
2030
cat imex_version.txt >> $GITHUB_ENV
2131
echo 'EOF' >> $GITHUB_ENV
22-
- name: Checkout IMEX
23-
uses: actions/checkout@v3
32+
- name: Restore IMEX cache
33+
id: cache-imex
34+
uses: actions/cache/restore@v3
2435
with:
25-
repository: intel-innersource/frameworks.ai.mlir.mlir-extensions
26-
token: ${{ secrets.PATIMEX }}
27-
path: third_party/imex
28-
ref: ${{ env.IMEX_SHA }}
29-
# - name: Cache miniconda
30-
# id: cache-miniconda
31-
# uses: actions/cache@v3
32-
# env:
33-
# MINICONDA_CACHE_NUMBER: 2 # Increase to reset cache
34-
# with:
35-
# path: third_party/install/miniconda/**
36-
# key: ${{ runner.os }}-miniconda-${{ env.MINICONDA_CACHE_NUMBER }}-${{ hashFiles('conda-env.txt') }}
36+
path: third_party/install/**
37+
key: ${{ runner.os }}-build-imex-${{ env.IMEX_CACHE_NUMBER }}-${{ env.IMEX_SHA }}
3738
- name: Miniconda
38-
# if: steps.cache-miniconda.outputs.cache-hit != 'true'
39-
run: |
40-
rm -rf "$GITHUB_WORKSPACE"/third_party/install/miniconda
41-
mkdir -p "$GITHUB_WORKSPACE"/third_party/install
42-
cd "$GITHUB_WORKSPACE"/third_party
43-
wget -q https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh
44-
bash Miniconda3-latest-Linux-x86_64.sh -u -b -f -p $GITHUB_WORKSPACE/third_party/install/miniconda
45-
echo "$GITHUB_WORKSPACE/third_party/install/miniconda/bin" >> $GITHUB_PATH
46-
echo "$GITHUB_WORKSPACE/third_party/install/miniconda/condabin" >> $GITHUB_PATH
47-
export PATH=$GITHUB_WORKSPACE/third_party/install/miniconda/bin:${PATH}
48-
. $GITHUB_WORKSPACE/third_party/install/miniconda/etc/profile.d/conda.sh
49-
. $GITHUB_WORKSPACE/third_party/install/miniconda/bin/activate
50-
cd -
51-
conda create --file conda-env.txt --name sharpy
52-
conda clean -a -y
53-
# - name: Setup miniconda
54-
# if: steps.cache-miniconda.outputs.cache-hit == 'true'
55-
# run: |
56-
# echo "$GITHUB_WORKSPACE/third_party/install/miniconda/bin" >> $GITHUB_PATH
57-
# echo "$GITHUB_WORKSPACE/third_party/install/miniconda/condabin" >> $GITHUB_PATH
58-
# export PATH=$GITHUB_WORKSPACE/third_party/install/miniconda/bin:${PATH}
59-
- name: Setup LLVM Cache Var
6039
run: |
61-
echo 'LLVM_SHA<<EOF' >> $GITHUB_ENV
62-
cat third_party/imex/build_tools/llvm_version.txt >> $GITHUB_ENV
63-
echo 'EOF' >> $GITHUB_ENV
64-
- name: Cache LLVM-MLIR
65-
id: cache-llvm-mlir
66-
uses: actions/cache@v3
67-
env:
68-
LLVM_CACHE_NUMBER: 1 # Increase to reset cache
69-
with:
70-
path: third_party/install/llvm-mlir/**
71-
key: ${{ runner.os }}-build-llvm-${{ env.LLVM_CACHE_NUMBER }}-${{ env.LLVM_SHA }}
72-
- name: Build LLVM MLIR
73-
if: steps.cache-llvm-mlir.outputs.cache-hit != 'true'
40+
env
41+
rm -rf $CONDA_ROOT
42+
cd $GITHUB_WORKSPACE/..
43+
rm -f Miniconda3-*.sh
44+
CPKG=Miniconda3-latest-Linux-x86_64.sh
45+
wget -q https://repo.anaconda.com/miniconda/$CPKG
46+
bash $CPKG -u -b -f -p $CONDA_ROOT
47+
export PATH=$CONDA_ROOT/condabin:$CONDA_ROOT/bin:${PATH}
48+
eval "$($CONDA_ROOT/bin/python -m conda shell.bash hook)"
49+
conda install -c conda-forge --override-channels python git-lfs conda-build
50+
# aahhhh bug in conda somewhere
51+
sed -i "s,\#\!/usr/bin/env python,#\!$CONDA_ROOT/bin/python," $CONDA_ROOT/*bin/conda
52+
conda clean --all -y
53+
- name: Build SHARPY
7454
run: |
75-
echo "INFO: Need to rebuild LLVM-MLIR. Previous installation for MLIR not found"
76-
. $GITHUB_WORKSPACE/third_party/install/miniconda/etc/profile.d/conda.sh
77-
. $GITHUB_WORKSPACE/third_party/install/miniconda/bin/activate sharpy
78-
np=`nproc`
79-
pushd third_party
80-
if [ -d "llvm-project" ]; then rm -rf llvm-project; fi
81-
mkdir llvm-project || exit 1
82-
cd llvm-project || exit 1
83-
git init
84-
git remote add origin https://github.com/llvm/llvm-project || exit 1
85-
git fetch origin ${{ env.LLVM_SHA }} || exit 1
86-
git reset --hard FETCH_HEAD || exit 1
87-
if [ -d "$GITHUB_WORKSPACE/third_party/imex/build_tools/patches" ]; then git apply $GITHUB_WORKSPACE/third_party/imex/build_tools/patches/*.patch; fi
88-
cd -
89-
mkdir -p build/llvm-mlir || exit 1
90-
cd build/llvm-mlir || exit 1
91-
cmake ../../llvm-project/llvm -GNinja -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_PROJECTS=mlir -DLLVM_ENABLE_ASSERTIONS=ON -DLLVM_ENABLE_RTTI=ON -DLLVM_INSTALL_UTILS=ON -DLLVM_TARGETS_TO_BUILD=X86 -DLLVM_ENABLE_BINDINGS=OFF -DLLVM_ENABLE_ZSTD=OFF -DLLVM_DISTRIBUTION_COMPONENTS="llvm-headers;llvm-libraries;cmake-exports;FileCheck;count;not;mlir-headers;mlir-libraries;mlir-cmake-exports;mlir-tblgen;mlir-cpu-runner" -DCMAKE_INSTALL_PREFIX="$GITHUB_WORKSPACE"/third_party/install/llvm-mlir
92-
cmake --build . -j ${np} || exit 1
93-
cmake --install . || exit 1
94-
cp bin/FileCheck $GITHUB_WORKSPACE/third_party/install/llvm-mlir/bin/
95-
cp bin/count $GITHUB_WORKSPACE/third_party/install/llvm-mlir/bin/
96-
cp bin/not $GITHUB_WORKSPACE/third_party/install/llvm-mlir/bin/
97-
popd
98-
- name: Cache IMEX
99-
id: cache-imex-mlir
100-
uses: actions/cache@v3
101-
env:
102-
IMEX_CACHE_NUMBER: 1 # Increase to reset cache
55+
env
56+
export PATH=$CONDA_ROOT/condabin:$CONDA_ROOT/bin:${PATH}
57+
eval "$($CONDA_ROOT/bin/python -m conda shell.bash hook)"
58+
cd conda-recipe
59+
conda build --numpy 1.26 -c conda-forge --override-channels --no-anaconda-upload --no-test .
60+
- name: Save IMEX cache
61+
uses: actions/cache/save@v3
62+
if: steps.cache-imex.outputs.cache-hit != 'true'
10363
with:
104-
path: third_party/install/imex/**
64+
path: third_party/install/**
10565
key: ${{ runner.os }}-build-imex-${{ env.IMEX_CACHE_NUMBER }}-${{ env.IMEX_SHA }}
106-
- name: Build IMEX
107-
if: steps.cache-imex-mlir.outputs.cache-hit != 'true'
66+
- name: Install SHARPY
10867
run: |
109-
. $GITHUB_WORKSPACE/third_party/install/miniconda/etc/profile.d/conda.sh
110-
. $GITHUB_WORKSPACE/third_party/install/miniconda/bin/activate sharpy
111-
mkdir -p third_party/build/imex
112-
pushd third_party/build/imex
113-
MLIR_DIR=$GITHUB_WORKSPACE/third_party/install/llvm-mlir cmake $GITHUB_WORKSPACE/third_party/imex -DCMAKE_BUILD_TYPE=debug -G Ninja
114-
cmake --build . -j ${np} || exit 1
115-
cmake --install . --prefix=$GITHUB_WORKSPACE/third_party/install/imex || exit 1
116-
popd
117-
- name: Build SHARPY
118-
run: |
119-
. $GITHUB_WORKSPACE/third_party/install/miniconda/etc/profile.d/conda.sh
120-
. $GITHUB_WORKSPACE/third_party/install/miniconda/bin/activate sharpy
121-
MLIRROOT=$GITHUB_WORKSPACE/third_party/install/llvm-mlir IMEXROOT=$GITHUB_WORKSPACE/third_party/install/imex python -m pip install . -v
122-
libidtr=$(pip show -f sharpy | grep Location | awk '{print $2}' | tr -d '\n')
123-
libidtr=${libidtr}/$(pip show -f sharpy | grep libidtr | awk '{print $1}' | tr -d '\n')
124-
echo "SHARPY_IDTR_SO=${libidtr}" >> $GITHUB_ENV
125-
echo $SHARPY_IDTR_SO
68+
export PATH=$CONDA_ROOT/condabin:$CONDA_ROOT/bin:${PATH}
69+
eval "$($CONDA_ROOT/bin/python -m conda shell.bash hook)"
70+
conda create --use-local -c conda-forge --override-channels -n sharpy -v sharpy pytest
71+
conda activate sharpy
12672
- name: Test SHARPY (pytest)
12773
run: |
128-
. $GITHUB_WORKSPACE/third_party/install/miniconda/etc/profile.d/conda.sh
129-
. $GITHUB_WORKSPACE/third_party/install/miniconda/bin/activate sharpy
74+
export PATH=$CONDA_ROOT/condabin:$CONDA_ROOT/bin:${PATH}
75+
eval "$($CONDA_ROOT/bin/python -m conda shell.bash hook)"
76+
conda activate sharpy
13077
cd test
131-
echo $SHARPY_IDTR_SO
132-
ldd $SHARPY_IDTR_SO
13378
pytest . -v -s
13479
SHARPY_FORCE_DIST=1 pytest . -v -s
13580
mpirun -n 2 pytest . -v -s
@@ -138,8 +83,9 @@ jobs:
13883
cd -
13984
- name: Run examples
14085
run: |
141-
. $GITHUB_WORKSPACE/third_party/install/miniconda/etc/profile.d/conda.sh
142-
. $GITHUB_WORKSPACE/third_party/install/miniconda/bin/activate sharpy
86+
export PATH=$CONDA_ROOT/condabin:$CONDA_ROOT/bin:${PATH}
87+
eval "$($CONDA_ROOT/bin/python -m conda shell.bash hook)"
88+
conda activate sharpy
14389
cd examples
14490
SHARPY_FALLBACK=numpy python -u ./stencil-2d.py 5 1024 star 4
14591
SHARPY_FALLBACK=numpy SHARPY_FORCE_DIST=1 python -u ./stencil-2d.py 5 1024 star 4
@@ -167,7 +113,3 @@ jobs:
167113
mpirun -n 3 python -u ./black_scholes.py
168114
mpirun -n 4 python -u ./black_scholes.py
169115
cd -
170-
- name: Cleanup
171-
run: |
172-
pip list
173-
pip uninstall -y sharpy

‎.pre-commit-config.yaml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ repos:
1717
- id: fix-byte-order-marker
1818
- id: mixed-line-ending
1919
- id: trailing-whitespace
20+
exclude: '.*\.patch'
2021
- repo: https://github.com/psf/black
2122
rev: 23.12.0
2223
hooks:

‎CMakeLists.txt

Lines changed: 13 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -6,17 +6,6 @@ if(DEFINED ENV{CONDA_PREFIX})
66
else()
77
set(CONDA_PREFIX UNSET)
88
endif()
9-
10-
if(DEFINED ENV{MKLROOT})
11-
set(MKL_ROOT $ENV{MKLROOT})
12-
else()
13-
set(MKL_ROOT ${CONDA_PREFIX})
14-
endif()
15-
if(DEFINED ENV{TBBROOT})
16-
set(TBB_ROOT $ENV{TBBROOT})
17-
else()
18-
set(TBB_ROOT ${CONDA_PREFIX})
19-
endif()
209
if(DEFINED ENV{MLIRROOT})
2110
set(MLIR_ROOT $ENV{MLIRROOT})
2211
else()
@@ -28,11 +17,7 @@ if(DEFINED ENV{IMEXROOT})
2817
else()
2918
set(IMEX_ROOT ${CONDA_PREFIX})
3019
endif()
31-
if(MKLROOT STREQUAL UNSET OR TBBROOT STREQUAL UNSET)
32-
message(FATAL_ERROR "MKLROOT and TBBROOT not set (nor CONDA_PREFIX)")
33-
endif()
3420

35-
set(ZLIB_LIBRARY /usr/lib/x86_64-linux-gnu/libz.a)
3621
# C++ standard
3722
set(CMAKE_CXX_STANDARD 17)
3823
set(CMAKE_C_EXTENSIONS OFF)
@@ -50,18 +35,19 @@ message(STATUS "Expected IMEX sha: \"${EXPECTED_IMEX_SHA}\"")
5035
# Deps
5136
# ===============
5237
# Find dependent packages like IMEX, Python3 and NumPy
53-
find_package(ZLIB)
38+
find_package(ZLIB REQUIRED)
39+
find_package(TBB REQUIRED)
5440
find_package(Python3 COMPONENTS Interpreter Development.Module NumPy REQUIRED)
55-
find_package(pybind11 CONFIG)
41+
find_package(pybind11 CONFIG REQUIRED)
5642
# FindMPI does not work with iMPI conda packages
5743
set(MPI_INCLUDE_PATH $ENV{I_MPI_ROOT}/include)
5844
set(MPI_CXX_LIBRARIES $ENV{I_MPI_ROOT}/lib/libmpi.so $ENV{I_MPI_ROOT}/lib/libmpicxx.so)
5945
# find_package(MPI REQUIRED CXX)
60-
find_package(LLVM REQUIRED CONFIG NO_SYSTEM_ENVIRONMENT_PATH NO_CMAKE_SYSTEM_PATH)
46+
find_package(LLVM REQUIRED CONFIG NO_SYSTEM_ENVIRONMENT_PATH NO_CMAKE_SYSTEM_PATH REQUIRED)
6147
message(STATUS "Using LLVMConfig.cmake in: ${LLVM_DIR}")
62-
find_package(MLIR REQUIRED CONFIG NO_SYSTEM_ENVIRONMENT_PATH NO_CMAKE_SYSTEM_PATH)
48+
find_package(MLIR REQUIRED CONFIG NO_SYSTEM_ENVIRONMENT_PATH NO_CMAKE_SYSTEM_PATH REQUIRED)
6349
message(STATUS "Using MLIRConfig.cmake in: ${MLIR_DIR}")
64-
find_package(IMEX REQUIRED CONFIG)
50+
find_package(IMEX REQUIRED CONFIG REQUIRED)
6551
message(STATUS "Using IMEXConfig.cmake in: ${IMEX_DIR}")
6652

6753
list(APPEND CMAKE_MODULE_PATH "${IMEX_CMAKE_DIR}")
@@ -72,10 +58,10 @@ list(APPEND CMAKE_MODULE_PATH "${LLVM_CMAKE_DIR}")
7258
#include(AddIMEX)
7359

7460
# macro for mlir root directory
75-
add_compile_definitions(CMAKE_MLIR_ROOT="${MLIR_ROOT}" CMAKE_IMEX_ROOT="${IMEX_ROOT}" FORTIFY_SOURCE=2 USE_MKL=1)
61+
add_compile_definitions(CMAKE_MLIR_ROOT="${MLIR_ROOT}" CMAKE_IMEX_ROOT="${IMEX_ROOT}" FORTIFY_SOURCE=2)
7662
add_compile_options(
7763
"-ftemplate-backtrace-limit=0"
78-
"-flarge-source-files"
64+
"$<$<STREQUAL:${CMAKE_CXX_COMPILER_ID},GNU>:-flarge-source-files>"
7965
"-Wall" "-Wformat" "-fPIC" "-Wno-comment"
8066
"$<$<CONFIG:DEBUG>:-fstack-protector>"
8167
"$<$<CONFIG:RELEASE>:-flto>"
@@ -93,8 +79,7 @@ SET(CMAKE_INSTALL_RPATH "$ORIGIN;$ORIGIN/../../..")
9379

9480
#find_package(OpenMP)
9581

96-
set(MKL_LIBRARIES -L${MKLROOT}/lib -lmkl_intel_lp64 -lmkl_tbb_thread -lmkl_core -ltbb -lpthread -lrt -ldl -lm)
97-
#set(CMAKE_INSTALL_RPATH $ENV{MKLROOT}/lib)
82+
# set(MKL_LIBRARIES -L${MKLROOT}/lib -lmkl_intel_lp64 -lmkl_tbb_thread -lmkl_core -ltbb -lpthread -lrt -ldl -lm)
9883
# Use -fPIC even if statically compiled
9984
set(CMAKE_POSITION_INDEPENDENT_CODE ON)
10085

@@ -152,8 +137,6 @@ include_directories(
152137
${PROJECT_SOURCE_DIR}/src/include
153138
${PROJECT_SOURCE_DIR}/third_party/bitsery/include
154139
${MPI_INCLUDE_PATH}
155-
$ENV{MKLROOT}/include
156-
$ENV{TBBROOT}/include
157140
${pybind11_INCLUDE_DIRS}
158141
${MLIR_INCLUDE_DIRS}
159142
${IMEX_INCLUDE_DIRS})
@@ -163,21 +146,17 @@ if (CMAKE_SYSTEM_NAME STREQUAL Linux)
163146
# target_link_options(idtr PRIVATE "LINKER:-fvisibility=hidden" "LINKER:--exclude-libs,All")
164147
endif()
165148

166-
#compile_options(_sharpy PRIVATE -fopenmp)
167149
get_property(mlir_dialect_libs GLOBAL PROPERTY MLIR_DIALECT_LIBS)
168150
get_property(mlir_conversion_libs GLOBAL PROPERTY MLIR_CONVERSION_LIBS)
169151
get_property(mlir_extension_libs GLOBAL PROPERTY MLIR_EXTENSION_LIBS)
170152
get_property(mlir_translation_libs GLOBAL PROPERTY MLIR_TRANSLATION_LIBS)
171153
get_property(imex_dialect_libs GLOBAL PROPERTY IMEX_DIALECT_LIBS)
172154
get_property(imex_conversion_libs GLOBAL PROPERTY IMEX_CONVERSION_LIBS)
173155

174-
#llvm_update_compile_flags(_sharpy)
175-
target_link_directories(_sharpy PRIVATE ${CONDA_PREFIX}/lib ${IMEX_ROOT}/lib)
156+
target_link_directories(_sharpy PRIVATE ${CONDA_PREFIX}/lib ${IMEX_ROOT}/lib )
176157
target_link_directories(idtr PRIVATE ${CONDA_PREFIX}/lib)
177158

178159
target_link_libraries(_sharpy PRIVATE
179-
# ${MKL_LIBRARIES}
180-
# tbb
181160
${mlir_dialect_libs}
182161
${mlir_conversion_libs}
183162
${mlir_extension_libs}
@@ -191,11 +170,10 @@ target_link_libraries(_sharpy PRIVATE
191170
LLVMX86CodeGen
192171
LLVMX86AsmParser
193172
idtr
194-
tbb
195-
${ZLIB_LIBRARIES}
173+
TBB::tbb
174+
ZLIB::ZLIB
196175
)
197176
target_link_libraries(idtr PRIVATE
198177
${MPI_CXX_LIBRARIES}
199-
# ${MKL_LIBRARIES}
200-
tbb
178+
TBB::tbb
201179
)

‎README.md

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
[![.github/workflows/ci.yml](https://github.com/intel-sandbox/sharpy/actions/workflows/ci.yml/badge.svg)](https://github.com/intel-sandbox/sharpy/actions/workflows/ci.yml)
22

3+
***This software package is not ready for production use and and merely a proof of concept implementation.***
4+
35
# Distributed Python Array
46

57
A array implementation following the [array API as defined by the data-API consortium](https://data-apis.org/array-api/latest/index.html).

0 commit comments

Comments
 (0)