Skip to content

Commit c0ebdd4

Browse files
nmanderynmandery
committed
release the GIL more often to allow other threads to run
1 parent ae3d0c9 commit c0ebdd4

9 files changed

+228
-153
lines changed

‎h3ronpy/CHANGES.rst

+2-1
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,8 @@ Versioning <https://semver.org/spec/v2.0.0.html>`__.
1212
Unreleased
1313
----------
1414

15-
- Upgrade to h3o 0.7
15+
- Upgrade to h3o 0.7.
16+
- Release the GIL more often to allow other threads to run.
1617
- The minimum supported python version is now 3.9.
1718

1819
0.21.1 - 2024-10-04

‎h3ronpy/src/array.rs

+27-1
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,9 @@ use pyo3::prelude::*;
66
use pyo3::types::{PyCapsule, PyTuple};
77
use pyo3_arrow::ffi::to_array_pycapsules;
88

9-
use crate::arrow_interop::pyarray_to_cellindexarray;
9+
use crate::arrow_interop::{
10+
pyarray_to_cellindexarray, pyarray_to_directededgeindexarray, pyarray_to_vertexindexarray,
11+
};
1012
use crate::resolution::PyResolution;
1113

1214
#[pyclass(name = "CellArray")]
@@ -89,6 +91,18 @@ impl PyDirectedEdgeArray {
8991
}
9092
}
9193

94+
impl AsRef<DirectedEdgeIndexArray> for PyDirectedEdgeArray {
95+
fn as_ref(&self) -> &DirectedEdgeIndexArray {
96+
&self.0
97+
}
98+
}
99+
100+
impl<'py> FromPyObject<'py> for PyDirectedEdgeArray {
101+
fn extract_bound(ob: &Bound<'py, PyAny>) -> PyResult<Self> {
102+
Ok(Self(pyarray_to_directededgeindexarray(ob)?))
103+
}
104+
}
105+
92106
#[pyclass(name = "VertexArray")]
93107
pub struct PyVertexArray(VertexIndexArray);
94108

@@ -117,3 +131,15 @@ impl PyVertexArray {
117131
Self(self.0.slice(offset, length))
118132
}
119133
}
134+
135+
impl AsRef<VertexIndexArray> for PyVertexArray {
136+
fn as_ref(&self) -> &VertexIndexArray {
137+
&self.0
138+
}
139+
}
140+
141+
impl<'py> FromPyObject<'py> for PyVertexArray {
142+
fn extract_bound(ob: &Bound<'py, PyAny>) -> PyResult<Self> {
143+
Ok(Self(pyarray_to_vertexindexarray(ob)?))
144+
}
145+
}

‎h3ronpy/src/op/compact.rs

+23-11
Original file line numberDiff line numberDiff line change
@@ -8,22 +8,34 @@ use crate::error::IntoPyResult;
88

99
#[pyfunction]
1010
#[pyo3(signature = (cellarray, mixed_resolutions = false))]
11-
pub(crate) fn compact(cellarray: PyCellArray, mixed_resolutions: bool) -> PyResult<PyObject> {
11+
pub(crate) fn compact(
12+
py: Python<'_>,
13+
cellarray: PyCellArray,
14+
mixed_resolutions: bool,
15+
) -> PyResult<PyObject> {
1216
let cellindexarray = cellarray.into_inner();
13-
let compacted = if mixed_resolutions {
14-
cellindexarray.compact_mixed_resolutions()
15-
} else {
16-
cellindexarray.compact()
17-
}
18-
.into_pyresult()?;
17+
let compacted = py
18+
.allow_threads(|| {
19+
if mixed_resolutions {
20+
cellindexarray.compact_mixed_resolutions()
21+
} else {
22+
cellindexarray.compact()
23+
}
24+
})
25+
.into_pyresult()?;
1926

20-
Python::with_gil(|py| h3array_to_pyarray(compacted, py))
27+
h3array_to_pyarray(compacted, py)
2128
}
2229

2330
#[pyfunction]
2431
#[pyo3(signature = (cellarray, target_resolution))]
25-
pub(crate) fn uncompact(cellarray: PyCellArray, target_resolution: u8) -> PyResult<PyObject> {
32+
pub(crate) fn uncompact(
33+
py: Python<'_>,
34+
cellarray: PyCellArray,
35+
target_resolution: u8,
36+
) -> PyResult<PyObject> {
2637
let target_resolution = Resolution::try_from(target_resolution).into_pyresult()?;
27-
let out = cellarray.into_inner().uncompact(target_resolution);
28-
Python::with_gil(|py| h3array_to_pyarray(out, py))
38+
let cellarray = cellarray.into_inner();
39+
let out = py.allow_threads(|| cellarray.uncompact(target_resolution));
40+
h3array_to_pyarray(out, py)
2941
}

‎h3ronpy/src/op/localij.rs

+11-9
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@ pub(crate) fn cells_to_localij(
5050
#[pyfunction]
5151
#[pyo3(signature = (anchor, i_array, j_array, set_failing_to_invalid = false))]
5252
pub(crate) fn localij_to_cells(
53+
py: Python<'_>,
5354
anchor: &Bound<PyAny>,
5455
i_array: &Bound<PyAny>,
5556
j_array: &Bound<PyAny>,
@@ -59,17 +60,18 @@ pub(crate) fn localij_to_cells(
5960
let j_array = pyarray_to_native::<Int32Array>(j_array)?;
6061
let anchorarray = get_anchor_array(anchor, i_array.len())?;
6162

62-
let localij_arrays = LocalIJArrays::try_new(anchorarray, i_array, j_array).into_pyresult()?;
63+
let cellarray = py.allow_threads(|| {
64+
let localij_arrays =
65+
LocalIJArrays::try_new(anchorarray, i_array, j_array).into_pyresult()?;
6366

64-
let cellarray = if set_failing_to_invalid {
65-
localij_arrays
66-
.to_cells_failing_to_invalid()
67-
.into_pyresult()?
68-
} else {
69-
localij_arrays.to_cells().into_pyresult()?
70-
};
67+
if set_failing_to_invalid {
68+
localij_arrays.to_cells_failing_to_invalid().into_pyresult()
69+
} else {
70+
localij_arrays.to_cells().into_pyresult()
71+
}
72+
})?;
7173

72-
Python::with_gil(|py| h3array_to_pyarray(cellarray, py))
74+
h3array_to_pyarray(cellarray, py)
7375
}
7476

7577
fn get_anchor_array(anchor: &Bound<PyAny>, len: usize) -> PyResult<CellIndexArray> {

‎h3ronpy/src/op/resolution.rs

+11-5
Original file line numberDiff line numberDiff line change
@@ -14,14 +14,20 @@ use crate::error::IntoPyResult;
1414
use crate::DEFAULT_CELL_COLUMN_NAME;
1515

1616
#[pyfunction]
17-
pub(crate) fn change_resolution(cellarray: PyCellArray, h3_resolution: u8) -> PyResult<PyObject> {
17+
pub(crate) fn change_resolution(
18+
py: Python<'_>,
19+
cellarray: PyCellArray,
20+
h3_resolution: u8,
21+
) -> PyResult<PyObject> {
1822
let cellindexarray = cellarray.into_inner();
1923
let h3_resolution = Resolution::try_from(h3_resolution).into_pyresult()?;
20-
let out = cellindexarray
21-
.change_resolution(h3_resolution)
22-
.into_pyresult()?;
24+
let out = py.allow_threads(|| {
25+
cellindexarray
26+
.change_resolution(h3_resolution)
27+
.into_pyresult()
28+
})?;
2329

24-
Python::with_gil(|py| h3array_to_pyarray(out, py))
30+
h3array_to_pyarray(out, py)
2531
}
2632

2733
#[pyfunction]

‎h3ronpy/src/op/string.rs

+45-36
Original file line numberDiff line numberDiff line change
@@ -14,67 +14,76 @@ use crate::error::IntoPyResult;
1414
#[pyfunction]
1515
#[pyo3(signature = (stringarray, set_failing_to_invalid = false))]
1616
pub(crate) fn cells_parse(
17+
py: Python<'_>,
1718
stringarray: PyArray,
1819
set_failing_to_invalid: bool,
1920
) -> PyResult<PyObject> {
2021
let (boxed_array, _field) = stringarray.into_inner();
21-
let cells = if let Some(stringarray) = boxed_array.as_any().downcast_ref::<StringArray>() {
22-
CellIndexArray::parse_genericstringarray(stringarray, set_failing_to_invalid)
23-
.into_pyresult()?
24-
} else if let Some(stringarray) = boxed_array.as_any().downcast_ref::<LargeStringArray>() {
25-
CellIndexArray::parse_genericstringarray(stringarray, set_failing_to_invalid)
26-
.into_pyresult()?
27-
} else {
28-
return Err(PyValueError::new_err(
29-
"unsupported array type to parse cells from",
30-
));
31-
};
22+
let cells = py.allow_threads(|| {
23+
if let Some(stringarray) = boxed_array.as_any().downcast_ref::<StringArray>() {
24+
CellIndexArray::parse_genericstringarray(stringarray, set_failing_to_invalid)
25+
.into_pyresult()
26+
} else if let Some(stringarray) = boxed_array.as_any().downcast_ref::<LargeStringArray>() {
27+
CellIndexArray::parse_genericstringarray(stringarray, set_failing_to_invalid)
28+
.into_pyresult()
29+
} else {
30+
Err(PyValueError::new_err(
31+
"unsupported array type to parse cells from",
32+
))
33+
}
34+
})?;
3235

33-
Python::with_gil(|py| h3array_to_pyarray(cells, py))
36+
h3array_to_pyarray(cells, py)
3437
}
3538

3639
#[pyfunction]
3740
#[pyo3(signature = (stringarray, set_failing_to_invalid = false))]
3841
pub(crate) fn vertexes_parse(
42+
py: Python<'_>,
3943
stringarray: PyArray,
4044
set_failing_to_invalid: bool,
4145
) -> PyResult<PyObject> {
4246
let (boxed_array, _field) = stringarray.into_inner();
43-
let vertexes = if let Some(utf8array) = boxed_array.as_any().downcast_ref::<StringArray>() {
44-
VertexIndexArray::parse_genericstringarray(utf8array, set_failing_to_invalid)
45-
.into_pyresult()?
46-
} else if let Some(utf8array) = boxed_array.as_any().downcast_ref::<LargeStringArray>() {
47-
VertexIndexArray::parse_genericstringarray(utf8array, set_failing_to_invalid)
48-
.into_pyresult()?
49-
} else {
50-
return Err(PyValueError::new_err(
51-
"unsupported array type to parse vertexes from",
52-
));
53-
};
47+
let vertexes = py.allow_threads(|| {
48+
if let Some(utf8array) = boxed_array.as_any().downcast_ref::<StringArray>() {
49+
VertexIndexArray::parse_genericstringarray(utf8array, set_failing_to_invalid)
50+
.into_pyresult()
51+
} else if let Some(utf8array) = boxed_array.as_any().downcast_ref::<LargeStringArray>() {
52+
VertexIndexArray::parse_genericstringarray(utf8array, set_failing_to_invalid)
53+
.into_pyresult()
54+
} else {
55+
Err(PyValueError::new_err(
56+
"unsupported array type to parse vertexes from",
57+
))
58+
}
59+
})?;
5460

55-
Python::with_gil(|py| h3array_to_pyarray(vertexes, py))
61+
h3array_to_pyarray(vertexes, py)
5662
}
5763

5864
#[pyfunction]
5965
#[pyo3(signature = (stringarray, set_failing_to_invalid = false))]
6066
pub(crate) fn directededges_parse(
67+
py: Python<'_>,
6168
stringarray: PyArray,
6269
set_failing_to_invalid: bool,
6370
) -> PyResult<PyObject> {
6471
let (boxed_array, _field) = stringarray.into_inner();
65-
let edges = if let Some(stringarray) = boxed_array.as_any().downcast_ref::<StringArray>() {
66-
DirectedEdgeIndexArray::parse_genericstringarray(stringarray, set_failing_to_invalid)
67-
.into_pyresult()?
68-
} else if let Some(stringarray) = boxed_array.as_any().downcast_ref::<LargeStringArray>() {
69-
DirectedEdgeIndexArray::parse_genericstringarray(stringarray, set_failing_to_invalid)
70-
.into_pyresult()?
71-
} else {
72-
return Err(PyValueError::new_err(
73-
"unsupported array type to parse directededges from",
74-
));
75-
};
72+
let edges = py.allow_threads(|| {
73+
if let Some(stringarray) = boxed_array.as_any().downcast_ref::<StringArray>() {
74+
DirectedEdgeIndexArray::parse_genericstringarray(stringarray, set_failing_to_invalid)
75+
.into_pyresult()
76+
} else if let Some(stringarray) = boxed_array.as_any().downcast_ref::<LargeStringArray>() {
77+
DirectedEdgeIndexArray::parse_genericstringarray(stringarray, set_failing_to_invalid)
78+
.into_pyresult()
79+
} else {
80+
Err(PyValueError::new_err(
81+
"unsupported array type to parse directededges from",
82+
))
83+
}
84+
})?;
7685

77-
Python::with_gil(|py| h3array_to_pyarray(edges, py))
86+
h3array_to_pyarray(edges, py)
7887
}
7988

8089
#[pyfunction]

‎h3ronpy/src/op/valid.rs

+2-2
Original file line numberDiff line numberDiff line change
@@ -12,10 +12,10 @@ use crate::arrow_interop::*;
1212

1313
fn h3index_valid<IX>(py: Python, arr: &Bound<PyAny>, booleanarray: bool) -> PyResult<PyObject>
1414
where
15-
IX: H3IndexArrayValue,
15+
IX: H3IndexArrayValue + Send,
1616
{
1717
let u64array = pyarray_to_uint64array(arr)?;
18-
let validated = H3Array::<IX>::from_iter_with_validity(u64array.iter());
18+
let validated = py.allow_threads(|| H3Array::<IX>::from_iter_with_validity(u64array.iter()));
1919

2020
if booleanarray {
2121
let nullbuffer = validated

‎h3ronpy/src/raster.rs

+6-7
Original file line numberDiff line numberDiff line change
@@ -153,16 +153,15 @@ macro_rules! make_raster_to_h3_variant {
153153
nodata_value: Option<$dtype>,
154154
) -> PyResult<(PyObject, PyObject)> {
155155
let arr = np_array.as_array();
156-
let (values, cells) = raster_to_h3(
156+
let (values, cells) = py.allow_threads(|| raster_to_h3(
157157
&arr,
158158
transform,
159159
&nodata_value,
160160
h3_resolution,
161161
axis_order_str,
162162
compact,
163-
)?;
163+
).map(|(values, cells)| (<$array_dtype>::from(values), cells)))?;
164164

165-
let values = <$array_dtype>::from(values);
166165
let values = PyArray::from_array_ref(Arc::new(values)).to_arro3(py)?;
167166
let cells = h3array_to_pyarray(CellIndexArray::from(cells), py)?;
168167

@@ -188,17 +187,17 @@ macro_rules! make_raster_to_h3_float_variant {
188187
// create a copy with the values wrapped in ordered floats to
189188
// support the internal hashing
190189
let of_arr = arr.map(|v| OrderedFloat::from(*v));
191-
let (values, cells) = raster_to_h3(
190+
let (values, cells) = py.allow_threads(|| raster_to_h3(
192191
&of_arr.view(),
193192
transform,
194193
&nodata_value.map(OrderedFloat::from),
195194
h3_resolution,
196195
axis_order_str,
197196
compact,
198-
)?;
197+
).map(|(values, cells)| (
198+
<$array_dtype>::from(values.into_iter().map(|v| v.into_inner()).collect::<Vec<$dtype>>()),
199+
cells)))?;
199200

200-
let values: Vec<$dtype> = values.into_iter().map(|v| v.into_inner()).collect();
201-
let values = <$array_dtype>::from(values);
202201
let values = PyArray::from_array_ref(Arc::new(values)).to_arro3(py)?;
203202
let cells = h3array_to_pyarray(CellIndexArray::from(cells), py)?;
204203

0 commit comments

Comments
 (0)