Skip to content

Commit 6247382

Browse files
fschlimbfschlimb
authored
Sharpy (#69)
Completing shift to new name sharpy * renaming files ddptensor (and alike) -> sharpy * also renaming variables, namespaces etc (also shortcuts like ddpt)
1 parent de11eed commit 6247382

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

100 files changed

+1098
-1510
lines changed

Diff for: ‎.github/workflows/ci.yml

+23-23
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
name: GitHub Actions Demo
2-
run-name: ${{ github.actor }} CI for ddpt
2+
run-name: ${{ github.actor }} CI for sharpy
33
on: [push, pull_request]
44
defaults:
55
run:
@@ -49,7 +49,7 @@ jobs:
4949
cd -
5050
conda env create -f conda-env.yml
5151
conda clean -a -y
52-
# conda create -q -n ddpt -c intel -c defaults pybind11 numpy cmake ninja impi-devel libstdcxx-ng mpi4py pytest scipy tbb-devel
52+
# conda create -q -n sharpy -c intel -c defaults pybind11 numpy cmake ninja impi-devel libstdcxx-ng mpi4py pytest scipy tbb-devel
5353
- name: Setup miniconda
5454
if: steps.cache-miniconda.outputs.cache-hit == 'true'
5555
run: |
@@ -74,7 +74,7 @@ jobs:
7474
run: |
7575
echo "INFO: Need to rebuild LLVM-MLIR. Previous installation for MLIR not found"
7676
. $GITHUB_WORKSPACE/third_party/install/miniconda/etc/profile.d/conda.sh
77-
. $GITHUB_WORKSPACE/third_party/install/miniconda/bin/activate ddpt
77+
. $GITHUB_WORKSPACE/third_party/install/miniconda/bin/activate sharpy
7878
np=`nproc`
7979
pushd third_party
8080
if [ -d "llvm-project" ]; then rm -rf llvm-project; fi
@@ -107,49 +107,49 @@ jobs:
107107
if: steps.cache-imex-mlir.outputs.cache-hit != 'true'
108108
run: |
109109
. $GITHUB_WORKSPACE/third_party/install/miniconda/etc/profile.d/conda.sh
110-
. $GITHUB_WORKSPACE/third_party/install/miniconda/bin/activate ddpt
110+
. $GITHUB_WORKSPACE/third_party/install/miniconda/bin/activate sharpy
111111
mkdir -p third_party/build/imex
112112
pushd third_party/build/imex
113113
MLIR_DIR=$GITHUB_WORKSPACE/third_party/install/llvm-mlir cmake $GITHUB_WORKSPACE/third_party/imex -DCMAKE_BUILD_TYPE=debug -G Ninja
114114
cmake --build . -j ${np} || exit 1
115115
cmake --install . --prefix=$GITHUB_WORKSPACE/third_party/install/imex || exit 1
116116
popd
117-
- name: Build DDPT
117+
- name: Build SHARPY
118118
run: |
119119
. $GITHUB_WORKSPACE/third_party/install/miniconda/etc/profile.d/conda.sh
120-
. $GITHUB_WORKSPACE/third_party/install/miniconda/bin/activate ddpt
120+
. $GITHUB_WORKSPACE/third_party/install/miniconda/bin/activate sharpy
121121
MLIRROOT=$GITHUB_WORKSPACE/third_party/install/llvm-mlir IMEXROOT=$GITHUB_WORKSPACE/third_party/install/imex python setup.py install
122-
# DDPT_ROOT=`pip show -f ddptensor | grep Location | awk '{print $2}'`
122+
# SHARPY_ROOT=`pip show -f sharpy | grep Location | awk '{print $2}'`
123123
# for some reason we cannot use the idtr from the install, not even a copy works
124-
libidtr=`ls ${PWD}/build/lib.linux-x86_64-cpython-*/ddptensor/libidtr.so`
125-
echo "DDPT_IDTR_SO=${libidtr}" >> $GITHUB_ENV
126-
echo $DDPT_IDTR_SO
127-
- name: Test DDPT (pytest)
124+
libidtr=`ls ${PWD}/build/lib.linux-x86_64-cpython-*/sharpy/libidtr.so`
125+
echo "SHARPY_IDTR_SO=${libidtr}" >> $GITHUB_ENV
126+
echo $SHARPY_IDTR_SO
127+
- name: Test SHARPY (pytest)
128128
run: |
129129
. $GITHUB_WORKSPACE/third_party/install/miniconda/etc/profile.d/conda.sh
130-
. $GITHUB_WORKSPACE/third_party/install/miniconda/bin/activate ddpt
130+
. $GITHUB_WORKSPACE/third_party/install/miniconda/bin/activate sharpy
131131
cd test
132-
echo $DDPT_IDTR_SO
133-
ldd $DDPT_IDTR_SO
132+
echo $SHARPY_IDTR_SO
133+
ldd $SHARPY_IDTR_SO
134134
pytest . -v -s
135-
DDPT_FORCE_DIST=1 pytest . -v -s
135+
SHARPY_FORCE_DIST=1 pytest . -v -s
136136
mpirun -n 2 pytest . -v -s
137137
mpirun -n 3 pytest . -v -s
138138
mpirun -n 4 pytest . -v -s
139139
cd -
140140
- name: Run examples
141141
run: |
142142
. $GITHUB_WORKSPACE/third_party/install/miniconda/etc/profile.d/conda.sh
143-
. $GITHUB_WORKSPACE/third_party/install/miniconda/bin/activate ddpt
143+
. $GITHUB_WORKSPACE/third_party/install/miniconda/bin/activate sharpy
144144
cd examples
145-
DDPT_FALLBACK=numpy python -u ./stencil-2d.py 5 1024 star 4
146-
DDPT_FALLBACK=numpy DDPT_FORCE_DIST=1 python -u ./stencil-2d.py 5 1024 star 4
147-
mpirun -n 2 -genv DDPT_FALLBACK=numpy python -u ./stencil-2d.py 5 2048 star 2
148-
mpirun -n 3 -genv DDPT_FALLBACK=numpy python -u ./stencil-2d.py 5 2048 star 2
149-
mpirun -n 4 -genv DDPT_FALLBACK=numpy python -u ./stencil-2d.py 5 2048 star 2
145+
SHARPY_FALLBACK=numpy python -u ./stencil-2d.py 5 1024 star 4
146+
SHARPY_FALLBACK=numpy SHARPY_FORCE_DIST=1 python -u ./stencil-2d.py 5 1024 star 4
147+
mpirun -n 2 -genv SHARPY_FALLBACK=numpy python -u ./stencil-2d.py 5 2048 star 2
148+
mpirun -n 3 -genv SHARPY_FALLBACK=numpy python -u ./stencil-2d.py 5 2048 star 2
149+
mpirun -n 4 -genv SHARPY_FALLBACK=numpy python -u ./stencil-2d.py 5 2048 star 2
150150
python -u ./wave_equation.py -d f32
151151
python -u ./wave_equation.py -d f64
152-
DDPT_FORCE_DIST=1 python -u ./wave_equation.py
152+
SHARPY_FORCE_DIST=1 python -u ./wave_equation.py
153153
mpirun -n 2 python -u ./wave_equation.py
154154
mpirun -n 3 python -u ./wave_equation.py
155155
mpirun -n 4 python -u ./wave_equation.py
@@ -159,4 +159,4 @@ jobs:
159159
- name: Cleanup
160160
run: |
161161
pip list
162-
pip uninstall -y ddptensor
162+
pip uninstall -y sharpy

Diff for: ‎CMakeLists.txt

+21-21
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
cmake_minimum_required(VERSION 3.18.2)
2-
project(ddptensor VERSION 1.0)
2+
project(sharpy VERSION 1.0)
33

44
if(DEFINED ENV{CONDA_PREFIX})
55
set(CONDA_PREFIX $ENV{CONDA_PREFIX})
@@ -77,11 +77,11 @@ set(MKL_LIBRARIES -L${MKLROOT}/lib -lmkl_intel_lp64 -lmkl_tbb_thread -lmkl_core
7777
# Use -fPIC even if statically compiled
7878
set(CMAKE_POSITION_INDEPENDENT_CODE ON)
7979

80-
set(P2C_HPP ${PROJECT_SOURCE_DIR}/src/include/ddptensor/p2c_ids.hpp)
80+
set(P2C_HPP ${PROJECT_SOURCE_DIR}/src/include/sharpy/p2c_ids.hpp)
8181
# Generate enums
8282
add_custom_command(
83-
COMMAND python ${PROJECT_SOURCE_DIR}/scripts/code_gen.py ${PROJECT_SOURCE_DIR}/ddptensor/array_api.py ${P2C_HPP}
84-
DEPENDS ${PROJECT_SOURCE_DIR}/scripts/code_gen.py ${PROJECT_SOURCE_DIR}/ddptensor/array_api.py
83+
COMMAND python ${PROJECT_SOURCE_DIR}/scripts/code_gen.py ${PROJECT_SOURCE_DIR}/sharpy/array_api.py ${P2C_HPP}
84+
DEPENDS ${PROJECT_SOURCE_DIR}/scripts/code_gen.py ${PROJECT_SOURCE_DIR}/sharpy/array_api.py
8585
OUTPUT ${P2C_HPP}
8686
COMMENT "Generating ${P2C_HPP}."
8787
)
@@ -90,12 +90,12 @@ add_custom_command(
9090
# Target
9191
# ============
9292

93-
FILE(GLOB Hpps ${PROJECT_SOURCE_DIR}/src/include/ddptensor/*.hpp)
94-
FILE(GLOB JitHpps ${PROJECT_SOURCE_DIR}/src/include/ddptensor/jit/*.hpp)
93+
FILE(GLOB Hpps ${PROJECT_SOURCE_DIR}/src/include/sharpy/*.hpp)
94+
FILE(GLOB JitHpps ${PROJECT_SOURCE_DIR}/src/include/sharpy/jit/*.hpp)
9595
set(Hpps ${Hpps} ${JitHpps} ${P2C_HPP})
9696

97-
set(DDPTSrcs
98-
${PROJECT_SOURCE_DIR}/src/ddptensor.cpp
97+
set(SHARPYSrcs
98+
${PROJECT_SOURCE_DIR}/src/_sharpy.cpp
9999
${PROJECT_SOURCE_DIR}/src/Creator.cpp
100100
${PROJECT_SOURCE_DIR}/src/EWBinOp.cpp
101101
${PROJECT_SOURCE_DIR}/src/EWUnyOp.cpp
@@ -114,7 +114,7 @@ set(RTSrcs
114114
${PROJECT_SOURCE_DIR}/src/Mediator.cpp
115115
${PROJECT_SOURCE_DIR}/src/MPIMediator.cpp
116116
${PROJECT_SOURCE_DIR}/src/CollComm.cpp
117-
${PROJECT_SOURCE_DIR}/src/DDPTensorImpl.cpp
117+
${PROJECT_SOURCE_DIR}/src/NDArray.cpp
118118
${PROJECT_SOURCE_DIR}/src/Factory.cpp
119119
${PROJECT_SOURCE_DIR}/src/Registry.cpp
120120
${PROJECT_SOURCE_DIR}/src/_deferred.cpp
@@ -125,10 +125,10 @@ set(IDTRSrcs
125125
${PROJECT_SOURCE_DIR}/src/Transceiver.cpp
126126
)
127127

128-
pybind11_add_module(_ddptensor MODULE ${DDPTSrcs} ${Hpps})
129-
add_library(_ddpt_rt SHARED ${RTSrcs} ${Hpps})
128+
pybind11_add_module(_sharpy MODULE ${SHARPYSrcs} ${Hpps})
129+
add_library(_sharpy_rt SHARED ${RTSrcs} ${Hpps})
130130
add_library(idtr SHARED ${IDTRSrcs} ${Hpps})
131-
set(AllTargets _ddptensor _ddpt_rt idtr)
131+
set(AllTargets _sharpy _sharpy_rt idtr)
132132

133133
add_compile_definitions(USE_MKL=1)
134134
add_compile_options("-ftemplate-backtrace-limit=0")
@@ -143,25 +143,25 @@ include_directories(
143143
${IMEX_INCLUDE_DIRS})
144144

145145
if (CMAKE_SYSTEM_NAME STREQUAL Linux)
146-
target_link_options(_ddptensor PRIVATE "LINKER:--version-script=${CMAKE_CURRENT_SOURCE_DIR}/export.txt")
147-
target_link_options(_ddpt_rt PRIVATE "LINKER:--version-script=${CMAKE_CURRENT_SOURCE_DIR}/export-ddpt_rt.txt")
146+
target_link_options(_sharpy PRIVATE "LINKER:--version-script=${CMAKE_CURRENT_SOURCE_DIR}/export.txt")
147+
target_link_options(_sharpy_rt PRIVATE "LINKER:--version-script=${CMAKE_CURRENT_SOURCE_DIR}/export-sharpy_rt.txt")
148148
# target_link_options(idtr PRIVATE "LINKER:-fvisibility=hidden" "LINKER:--exclude-libs,All")
149149
endif()
150150

151-
#compile_options(_ddptensor PRIVATE -fopenmp)
151+
#compile_options(_sharpy PRIVATE -fopenmp)
152152
get_property(mlir_dialect_libs GLOBAL PROPERTY MLIR_DIALECT_LIBS)
153153
get_property(mlir_conversion_libs GLOBAL PROPERTY MLIR_CONVERSION_LIBS)
154154
get_property(mlir_extension_libs GLOBAL PROPERTY MLIR_EXTENSION_LIBS)
155155
get_property(mlir_translation_libs GLOBAL PROPERTY MLIR_TRANSLATION_LIBS)
156156
get_property(imex_dialect_libs GLOBAL PROPERTY IMEX_DIALECT_LIBS)
157157
get_property(imex_conversion_libs GLOBAL PROPERTY IMEX_CONVERSION_LIBS)
158158

159-
#llvm_update_compile_flags(_ddpttensor)
160-
target_link_directories(_ddptensor PRIVATE ${CONDA_PREFIX}/lib)
161-
target_link_directories(_ddpt_rt PRIVATE ${CONDA_PREFIX}/lib) # ${IMEX_INSTALL_PREFIX}/lib)
159+
#llvm_update_compile_flags(_sharpy)
160+
target_link_directories(_sharpy PRIVATE ${CONDA_PREFIX}/lib)
161+
target_link_directories(_sharpy_rt PRIVATE ${CONDA_PREFIX}/lib) # ${IMEX_INSTALL_PREFIX}/lib)
162162
target_link_directories(idtr PRIVATE ${CONDA_PREFIX}/lib)
163163

164-
target_link_libraries(_ddptensor PRIVATE
164+
target_link_libraries(_sharpy PRIVATE
165165
# ${MKL_LIBRARIES}
166166
# tbb
167167
${mlir_dialect_libs}
@@ -176,15 +176,15 @@ target_link_libraries(_ddptensor PRIVATE
176176
IMEXUtil
177177
LLVMX86CodeGen
178178
LLVMX86AsmParser
179-
_ddpt_rt
179+
_sharpy_rt
180180
idtr
181181
)
182182
target_link_libraries(idtr PRIVATE
183183
${MPI_C_LIBRARIES}
184184
# ${MKL_LIBRARIES}
185185
tbb
186186
)
187-
target_link_libraries(_ddpt_rt PRIVATE
187+
target_link_libraries(_sharpy_rt PRIVATE
188188
${MPI_C_LIBRARIES}
189189
# ${MKL_LIBRARIES}
190190
tbb

Diff for: ‎README.md

+27-27
Original file line numberDiff line numberDiff line change
@@ -1,21 +1,21 @@
1-
[![.github/workflows/ci.yml](https://github.com/intel-sandbox/personal.fschlimb.ddptensor/actions/workflows/ci.yml/badge.svg)](https://github.com/intel-sandbox/personal.fschlimb.ddptensor/actions/workflows/ci.yml)
2-
# Distributed Data-Parallel Python Tensor
3-
A tensor implementation following the [array API as defined by the data-API consortium](https://data-apis.org/array-api/latest/index.html).
1+
[![.github/workflows/ci.yml](https://github.com/intel-sandbox/personal.fschlimb.sharpy/actions/workflows/ci.yml/badge.svg)](https://github.com/intel-sandbox/personal.fschlimb.sharpy/actions/workflows/ci.yml)
2+
# Distributed Data-Parallel Python Array
3+
A array implementation following the [array API as defined by the data-API consortium](https://data-apis.org/array-api/latest/index.html).
44
It supports a controller-worker execution model as well as a CSP-like execution.
55

66
## Setting up build environment
77
Install MLIR/LLVM and IMEX from branch dist-ndarray (see https://github.com/intel-innersource/frameworks.ai.mlir.mlir-extensions/tree/dist-ndarray).
88
```bash
9-
git --recurse-submodules clone https://github.com/intel-sandbox/personal.fschlimb.ddptensor
10-
cd personal.fschlimb.ddptensor
9+
git --recurse-submodules clone https://github.com/intel-sandbox/personal.fschlimb.sharpy
10+
cd personal.fschlimb.sharpy
1111
git checkout jit
12-
conda env create -f conda-env.yml -n ddpt
13-
conda activate ddpt
12+
conda env create -f conda-env.yml -n sharpy
13+
conda activate sharpy
1414
export MPIROOT=$CONDA_PREFIX
1515
export MLIRROOT=<your-MLIR-install-dir>
1616
export IMEXROOT=<your-IMEX-install-dir>
1717
```
18-
## Building ddptensor
18+
## Building sharpy
1919
```bash
2020
python setup.py develop
2121
```
@@ -28,41 +28,41 @@ __Test are currently not operational on this branch.__
2828
# single rank
2929
pytest test
3030
# multiple ranks, controller-worker, controller spawns ranks
31-
DDPT_MPI_SPAWN=$NoW PYTHON_EXE=`which python` pytest test
31+
SHARPY_MPI_SPAWN=$NoW PYTHON_EXE=`which python` pytest test
3232
# multiple ranks, controller-worker, mpirun
3333
mpirun -n $N python -m pytest test
3434
# multiple ranks, CSP
35-
DDPT_CW=0 mpirun -n $N python -m pytest test
35+
SHARPY_CW=0 mpirun -n $N python -m pytest test
3636
```
3737

38-
If DDPT_MPI_SPAWN is set it spawns the provided number of MPI processes.
38+
If SHARPY_MPI_SPAWN is set it spawns the provided number of MPI processes.
3939
By default new processes launch python executing a worker loop.
4040
This requires setting PYTHON_EXE.
41-
Alternatively DDPT_MPI_EXECUTABLE and DDPT_MPI_EXE_ARGS are used.
42-
Additionally DDPT_MPI_HOSTS can be used to control the host to use for spawning processes.
41+
Alternatively SHARPY_MPI_EXECUTABLE and SHARPY_MPI_EXE_ARGS are used.
42+
Additionally SHARPY_MPI_HOSTS can be used to control the host to use for spawning processes.
4343

4444
## Running
4545
```python
46-
import ddptensor as dt
47-
dt.init(False)
48-
a = dt.arange(0, 10, 1, dt.int64)
46+
import sharpy as sp
47+
sp.init(False)
48+
a = sp.arange(0, 10, 1, sp.int64)
4949
#print(a) # should trigger compilation
50-
b = dt.arange(0, 100, 10, dt.int64)
50+
b = sp.arange(0, 100, 10, sp.int64)
5151
#print(b.dtype) # should _not_ trigger compilation
5252
c = a * b
5353
#print(c)
54-
d = dt.sum(c, [0])
54+
d = sp.sum(c, [0])
5555
#del b # generated function should _not_ return b
5656
print(a, c, d) # printing of c (not a!) should trigger compilation
57-
dt.fini()
57+
sp.fini()
5858
```
5959
Assuming the above is in file `simple.py` a single-process run is executed like
6060
```bash
61-
DDPT_IDTR_SO=`pwd`/ddptensor/libidtr.so python simple.py
61+
SHARPY_IDTR_SO=`pwd`/sharpy/libidtr.so python simple.py
6262
```
6363
and multi-process run is executed like
6464
```bash
65-
DDPT_IDTR_SO=`pwd`/ddptensor/libidtr.so mpirun -n 5 python simple.py
65+
SHARPY_IDTR_SO=`pwd`/sharpy/libidtr.so mpirun -n 5 python simple.py
6666
```
6767

6868
## Contributing
@@ -74,23 +74,23 @@ pre-commit autoupdate
7474

7575
## Overview
7676
### Deferred Execution
77-
Typically, ddptensor operations do not get executed immediately. Instead, the function returns a transparent object (a future) only.
77+
Typically, sharpy operations do not get executed immediately. Instead, the function returns a transparent object (a future) only.
7878
the actual computation gets deferred by creating a promise/deferred object and queuing it for later. This is not visible to users, they can use it as any other numpy-like library.
7979

8080
Only when actual data is needed, computation will happen; that is when
81-
- the values of tensor elements are casted to bool int or float
82-
- the tensor is printed
81+
- the values of array elements are casted to bool int or float
82+
- the array is printed
8383

8484
In the background a worker thread handles deferred objects. Until computation is needed it dequeues deferred objects from the FIFO queue and asks them to generate MLIR.
8585
Objects can either generate MLIR or instead provide a run() function to immediately execute. For the latter case the current MLIR function gets executed before calling run() to make sure potential dependences are met.
8686

8787
### Distribution
88-
Tensors and operations on them get transparently distributed across multiple processes. Respective functionality is partly handled by this library and partly IMEX dist dialect.
88+
Arrays and operations on them get transparently distributed across multiple processes. Respective functionality is partly handled by this library and partly IMEX dist dialect.
8989
IMEX relies on a runtime library for complex communication tasks and for inspecting runtime configuration, such as number of processes and process id (MPI rank).
90-
ddptensor provides this library functionality in a separate dynamic library "idtr".
90+
sharpy provides this library functionality in a separate dynamic library "idtr".
9191

9292
Right now, data is split in the first dimension (only). Each process knows the partition it owns. For optimization partitions can actually overlap.
9393

94-
ddptensor supports to execution modes:
94+
sharpy supports to execution modes:
9595
1. CSP/SPMD/explicitly-distributed execution, meaning all processes execute the same program, execution is replicated on all processes. Data is typically not replicated but distributed among processes.
9696
2. Controller-Worker/implicitly-distributed execution, meaning only a single process executes the program and it distributes data and work to worker processes.

Diff for: ‎conda-env.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
name: ddpt
1+
name: sharpy
22
channels:
33
- intel
44
- defaults

Diff for: ‎ddptensor/random.py

-11
This file was deleted.

Diff for: ‎ddptensor/spmd.py

-19
This file was deleted.

Diff for: ‎ddptensor/torch/__init__.py

-20
This file was deleted.

0 commit comments

Comments
 (0)